Scraper Duplication

docker-compose.yml

@@ -7,7 +7,7 @@ services:
       MYSQL_USER: ${MYSQL_USER}
       MYSQL_PASSWORD: ${MYSQL_PASSWORD}
     volumes:
-      - db_data:/var/lib/mysql
+      - tas_councils_db_data:/var/lib/mysql
     restart: unless-stopped
     ports:
       - "3306:3306"
@@ -72,4 +72,5 @@ services:
     restart: unless-stopped
 
 volumes:
-  db_data:
+  tas_councils_db_data:
+    external: true

lib/enrich.rb

@@ -42,6 +42,7 @@ def ensure_extra_columns!(table)
     "lng"                => "DOUBLE NULL",
     "property_id"        => "VARCHAR(50) NULL",
     "title_reference"    => "VARCHAR(80) NULL",
+    "document_url"       => "TEXT NULL",
     "local_document_url" => "TEXT NULL",
     "on_notice_to"       => "DATE NULL",
     "on_notice_to_raw"   => "VARCHAR(80) NULL"

lib/scraper_helpers.rb

@@ -0,0 +1,58 @@
+# lib/scraper_helpers.rb
+# Shared top-level helpers required by individual DA scrapers.
+#
+# Usage — at the top of a scraper, after other requires:
+#
+#   require_relative "../lib/scraper_helpers"
+#
+# This file requires db and enrich so scrapers don't need separate
+# require lines for those two libs.
+
+require "uri"
+require_relative "./db"
+require_relative "./enrich"
+
+# Resolve a possibly-relative href against a base URL.
+# Returns the href string unchanged if URI.join raises.
+def abs_url(base, href)
+  return "" if href.to_s.strip.empty?
+  URI.join(base, href).to_s
+rescue URI::InvalidURIError
+  href.to_s
+end
+
+# Return node.text.strip, or default when node is nil.
+def text_or(node, default = "")
+  node ? node.text.strip : default
+end
+
+# Upsert a DA row, run enrichment, and optionally UPDATE extra columns.
+#
+# table  - validated DA table name (e.g. "da_glamorgan")
+# row    - hash passed to DB.upsert; must include :council_reference and :address
+# extras - optional hash of { column_name => value } pairs to UPDATE after upsert
+#          e.g. { document_url: "https://...", on_notice_to: Date.new(2025,6,1) }
+#
+# Prints "Upserted <ref> -> <address>" on success.
+def upsert_and_enrich!(table:, row:, extras: {})
+  DB.upsert(table, row)
+  enrich_after_upsert!(
+    table: table,
+    council_reference: row[:council_reference],
+    address: row[:address]
+  )
+  unless extras.empty?
+    begin
+      esc = DB.client.escape(table)
+      set_clause = extras.keys.map { |k| "`#{k}` = ?" }.join(", ")
+      vals = extras.values + [row[:council_reference], row[:address]]
+      upd = DB.client.prepare(
+        "UPDATE `#{esc}` SET #{set_clause} WHERE council_reference = ? AND address = ?"
+      )
+      upd.execute(*vals)
+    rescue Mysql2::Error => e
+      warn "[scraper_helpers] extras update skipped for #{row[:council_reference]}: #{e.message}"
+    end
+  end
+  puts "Upserted #{row[:council_reference]} -> #{row[:address]}"
+end

scrapers/centralhighlands.rb

@@ -3,27 +3,14 @@
 require "nokogiri"
 require "cgi"
 require_relative "../lib/http"
-require_relative "../lib/db"
 require_relative "../lib/util"
-require_relative "../lib/enrich"
+require_relative "../lib/scraper_helpers"
 
 TABLE = ENV.fetch("TABLE_NAME") # run_all.sh -> da_centralhighlands
 URL   = "https://centralhighlands.tas.gov.au/development-applications/"
 
 DB.ensure_table!(TABLE)
 
-# Optional: store the PDF URL
-begin
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS document_url VARCHAR(1024) NULL")
-rescue => e
-  warn "document_url add skipped: #{e.class} #{e.message}"
-end
-
-def abs_url(base, href)
-  return "" if href.to_s.strip.empty?
-  URI.join(base, href).to_s rescue href.to_s
-end
-
 # DA 2025/42, DA2025/00042, etc.
 REF_RX = %r{\bDA\s*(20\d{2})\s*/\s*([A-Za-z0-9\-_.]+)}i
 def extract_ref(str)
@@ -99,30 +86,19 @@ links.each_with_index do |a, idx|
 
   next if address.empty? || ref.nil?
 
-  DB.upsert(TABLE, {
-    description: description.empty? ? "Development Application" : description,
-    date_received: on_notice,            # store close date here for consistency with others
-    date_received_raw: on_notice_raw,
-    address: address,
-    council_reference: ref,
-    applicant: "",
-    owner: ""
-  })
-  
-  enrich_after_upsert!(
+  upsert_and_enrich!(
     table: TABLE,
-    council_reference: ref,
-    address: address
+    row: {
+      description: description.empty? ? "Development Application" : description,
+      date_received: on_notice,
+      date_received_raw: on_notice_raw,
+      address: address,
+      council_reference: ref,
+      applicant: "",
+      owner: ""
+    },
+    extras: { document_url: pdf }
   )
-
-  begin
-    upd = DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET document_url = ? WHERE council_reference = ? AND address = ?")
-    upd.execute(pdf, ref, address)
-  rescue Mysql2::Error => e
-    warn "[centralhighlands] db update skipped for #{ref}: #{e.message}"
-  end
-
-  puts "Upserted #{ref} -> #{address}"
   saved += 1
 end
 

scrapers/derwentvalley.rb

@@ -4,9 +4,8 @@
 
 require "nokogiri"
 require_relative "../lib/http"
-require_relative "../lib/db"
 require_relative "../lib/util"
-require_relative "../lib/enrich"
+require_relative "../lib/scraper_helpers"
 
 TABLE        = ENV.fetch("TABLE_NAME")  # run_all.sh -> da_derwentvalley
 LIST_URL     = "https://www.derwentvalley.tas.gov.au/home/card-listing/development-applications"
@@ -14,20 +13,6 @@ NEWS_URL     = "https://www.derwentvalley.tas.gov.au/home/latest-news?f.News+cat
 
 DB.ensure_table!(TABLE)
 
-begin
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS document_url TEXT NULL")
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS on_notice_to DATE NULL")
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS on_notice_to_raw VARCHAR(80) NULL")
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS title_reference TEXT NULL")
-rescue => e
-  warn "Optional column add skipped: #{e.class} #{e.message}"
-end
-
-def abs_url(base, href)
-  return "" if href.to_s.strip.empty?
-  URI.join(base, href).to_s rescue href.to_s
-end
-
 # Common reference forms: "DA 2025/097"
 REF_RX = %r{\bDA\s*(20\d{2})\s*/\s*([A-Za-z0-9\-_.]+)}i
 def extract_ref(s)
@@ -155,30 +140,24 @@ links.each do |u|
   end
   next unless item
 
-  DB.upsert(TABLE, {
-    description: item[:description],
-    date_received: item[:date_received],
-    date_received_raw: item[:date_received_raw],
-    address: item[:address],
-    council_reference: item[:council_reference],
-    applicant: "",
-    owner: ""
-  })
-  
-  enrich_after_upsert!(
+  upsert_and_enrich!(
     table: TABLE,
-    council_reference: council_reference,
-    address: address
+    row: {
+      description: item[:description],
+      date_received: item[:date_received],
+      date_received_raw: item[:date_received_raw],
+      address: item[:address],
+      council_reference: item[:council_reference],
+      applicant: "",
+      owner: ""
+    },
+    extras: {
+      document_url:     item[:document_url],
+      on_notice_to:     item[:date_received],
+      on_notice_to_raw: item[:date_received_raw],
+      title_reference:  item[:title_reference]
+    }
   )
-
-  begin
-    upd = DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET document_url = ?, on_notice_to = ?, on_notice_to_raw = ?, title_reference = ? WHERE council_reference = ? AND address = ?")
-    upd.execute(item[:document_url], item[:date_received], item[:date_received_raw], item[:title_reference], item[:council_reference], item[:address])
-  rescue => e
-    warn "Extras update skipped for #{item[:council_reference]}: #{e.class} #{e.message}"
-  end
-
-  puts "Upserted #{item[:council_reference]} -> #{item[:address]}"
   saved += 1
 end
 

scrapers/enrich.rb

@@ -42,6 +42,7 @@ def ensure_extra_columns!(table)
     "lng"                => "DOUBLE NULL",
     "property_id"        => "VARCHAR(50) NULL",
     "title_reference"    => "VARCHAR(80) NULL",
+    "document_url"       => "TEXT NULL",
     "local_document_url" => "TEXT NULL",
     "on_notice_to"       => "DATE NULL",
     "on_notice_to_raw"   => "VARCHAR(80) NULL"

scrapers/georgetown.rb

@@ -2,31 +2,14 @@
 
 require "nokogiri"
 require_relative "../lib/http"
-require_relative "../lib/db"
 require_relative "../lib/util"
-require_relative "../lib/enrich"
+require_relative "../lib/scraper_helpers"
 
 TABLE = ENV.fetch("TABLE_NAME")  # run_all.sh sets from filename: da_georgetown
 URL   = "https://georgetown.tas.gov.au/development-applications/"
 
 DB.ensure_table!(TABLE)
 
-# Optional: keep the document link with each row
-begin
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS document_url VARCHAR(1024) NULL")
-rescue => e
-  warn "document_url add skipped: #{e.class} #{e.message}"
-end
-
-def text_or(node, fallback = "")
-  node ? node.text.strip : fallback
-end
-
-def abs_url(base, href)
-  return "" if href.to_s.strip.empty?
-  URI.join(base, href).to_s rescue href.to_s
-end
-
 # Try to pull a council reference from common places
 REF_RX = %r{(DA|APP|APPLICATION|PLA)\s*([0-9]{4})\s*[/\-]?\s*([A-Za-z0-9\-_.]{2,})}i
 def extract_reference(str)
@@ -115,31 +98,19 @@ end
 puts "Found #{items.length} items for #{TABLE}"
 
 items.each do |row|
-  DB.upsert(TABLE, {
-    description: row[:description],
-    date_received: row[:date_received],
-    date_received_raw: row[:date_received_raw],
-    address: row[:address],
-    council_reference: row[:council_reference],
-    applicant: "",
-    owner: ""
-  })
-  
-  enrich_after_upsert!(
+  upsert_and_enrich!(
     table: TABLE,
-    council_reference: council_reference,
-    address: address
+    row: {
+      description: row[:description],
+      date_received: row[:date_received],
+      date_received_raw: row[:date_received_raw],
+      address: row[:address],
+      council_reference: row[:council_reference],
+      applicant: "",
+      owner: ""
+    },
+    extras: { document_url: row[:document_url] }
   )
-
-  # keep the PDF link if the column exists
-  begin
-    upd = DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET document_url = ? WHERE council_reference = ? AND address = ?")
-    upd.execute(row[:document_url], row[:council_reference], row[:address])
-  rescue Mysql2::Error => e
-    warn "[georgetown] db update skipped for #{row[:council_reference]}: #{e.message}"
-  end
-
-  puts "Upserted #{row[:council_reference]} -> #{row[:address]}"
 end
 
 puts "Done #{TABLE}."

scrapers/glamorgan.rb

@@ -2,26 +2,14 @@
 
 require "nokogiri"
 require_relative "../lib/http"
-require_relative "../lib/db"
 require_relative "../lib/util"
-require_relative "../lib/enrich"
+require_relative "../lib/scraper_helpers"
 
 TABLE = ENV.fetch("TABLE_NAME")  # run_all.sh sets from filename: da_glamorgan
 URL   = "https://gsbc.tas.gov.au/services-facilities/public-notices/"
 
 DB.ensure_table!(TABLE)
 
-# Optional column to store the PDF or document link
-begin
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS document_url VARCHAR(1024) NULL")
-rescue => e
-  warn "document_url add skipped: #{e.class} #{e.message}"
-end
-
-def text_or(node, default = "")
-  node ? node.text.strip : default
-end
-
 # Try to extract a reference from visible text or file names
 REF_RX = %r{(DA|PLA|APP|APPLICATION)\s*([0-9]{4})\s*/\s*([A-Za-z0-9\-_.]+)}i
 def extract_reference(str)
@@ -32,13 +20,6 @@ def extract_reference(str)
   nil
 end
 
-def safe_abs(base, href)
-  return "" if href.to_s.strip.empty?
-  URI.join(base, href).to_s
-rescue URI::InvalidURIError
-  href.to_s
-end
-
 html = Http.get(URL)
 doc  = Nokogiri::HTML(html)
 
@@ -70,7 +51,7 @@ rows.each_with_index do |row, idx|
   application_raw    = text_or(tds[2])
   on_notice_to_raw   = text_or(tds[3])
   link_el            = tds[4]&.at_css("a")
-  document_url 	     = link_el ? safe_abs(URL, link_el["href"]) : ""
+  document_url 	     = link_el ? abs_url(URL, link_el["href"]) : ""
 
   date_received      = Util.parse_aus_date(application_raw)
   on_notice_to       = Util.parse_aus_date(on_notice_to_raw)
@@ -87,30 +68,19 @@ rows.each_with_index do |row, idx|
 
   next if address.empty? || council_reference.empty?
 
-  DB.upsert(TABLE, {
-    description: description,
-    date_received: date_received,
-    date_received_raw: application_raw,
-    address: address,
-    council_reference: council_reference,
-    applicant: "",
-    owner: ""
-  })
-  
-  enrich_after_upsert!(
+  upsert_and_enrich!(
     table: TABLE,
-    council_reference: council_reference,
-    address: address
+    row: {
+      description: description,
+      date_received: date_received,
+      date_received_raw: application_raw,
+      address: address,
+      council_reference: council_reference,
+      applicant: "",
+      owner: ""
+    },
+    extras: { document_url: document_url }
   )
-
-  begin
-    upd = DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET document_url = ? WHERE council_reference = ? AND address = ?")
-    upd.execute(document_url, council_reference, address)
-  rescue Mysql2::Error => e
-    warn "[glamorgan] db update skipped for #{council_reference}: #{e.message}"
-  end
-
-  puts "Upserted #{council_reference} -> #{address}"
   saved += 1
 end
 

scrapers/kingborough.rb

@@ -2,29 +2,14 @@
 
 require "nokogiri"
 require_relative "../lib/http"
-require_relative "../lib/db"
 require_relative "../lib/util"
-require_relative "../lib/enrich"
+require_relative "../lib/scraper_helpers"
 
 TABLE = ENV.fetch("TABLE_NAME")  # run_all.sh sets this from filename: da_kingborough
 URL   = "https://www.kingborough.tas.gov.au/development/planning-notices/"
 
 DB.ensure_table!(TABLE)
 
-# Optional columns for extras used on this site
-begin
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS document_url TEXT NULL")
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS on_notice_to DATE NULL")
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS on_notice_to_raw VARCHAR(50) NULL")
-rescue => e
-  warn "Optional column add skipped: #{e.class} #{e.message}"
-end
-
-def abs_url(base, href)
-  return "" if href.to_s.strip.empty?
-  URI.join(base, href).to_s rescue href.to_s
-end
-
 html = Http.get(URL)
 doc  = Nokogiri::HTML(html)
 
@@ -53,31 +38,23 @@ rows.each_with_index do |row, idx|
   # Need a reference and an address to store a row
   next if council_reference.empty? || address.empty?
 
-  DB.upsert(TABLE, {
-    description: description,
-    date_received: date_received,
-    date_received_raw: date_received_raw,
-    address: address,
-    council_reference: council_reference,
-    applicant: "",
-    owner: ""
-  })
-  
-  enrich_after_upsert!(
+  upsert_and_enrich!(
     table: TABLE,
-    council_reference: council_reference,
-    address: address
+    row: {
+      description: description,
+      date_received: date_received,
+      date_received_raw: date_received_raw,
+      address: address,
+      council_reference: council_reference,
+      applicant: "",
+      owner: ""
+    },
+    extras: {
+      document_url:     document_url,
+      on_notice_to:     on_notice_to,
+      on_notice_to_raw: on_notice_to_raw
+    }
   )
-
-  # Save the extras
-  begin
-    upd = DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET document_url = ?, on_notice_to = ?, on_notice_to_raw = ? WHERE council_reference = ? AND address = ?")
-    upd.execute(document_url, on_notice_to, on_notice_to_raw, council_reference, address)
-  rescue => e
-    warn "Extra field update skipped for #{council_reference}: #{e.class} #{e.message}"
-  end
-
-  puts "Upserted #{council_reference} -> #{address}"
   saved += 1
 end
 

scrapers/northernmidlands.rb

@@ -4,30 +4,14 @@ require "nokogiri"
 require "uri"
 require "cgi"
 require_relative "../lib/http"
-require_relative "../lib/db"
 require_relative "../lib/util"
-require_relative "../lib/enrich"
+require_relative "../lib/scraper_helpers"
 
 TABLE = ENV.fetch("TABLE_NAME")                    # run_all.sh -> da_northernmidlands
 URL   = "https://northernmidlands.tas.gov.au/planning/development-in-the-northern-midlands/development-applications-2"
 
 DB.ensure_table!(TABLE)
 
-# Optional extras
-begin
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS document_url TEXT NULL")
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS on_notice_to DATE NULL")
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS on_notice_to_raw VARCHAR(80) NULL")
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS title_reference TEXT NULL")
-rescue => e
-  warn "Optional column add skipped: #{e.class} #{e.message}"
-end
-
-def abs_url(base, href)
-  return "" if href.to_s.strip.empty?
-  URI.join(base, href).to_s rescue href.to_s
-end
-
 # “DA 2025/00123”, “DA2025/00123”, “Application No. DA 2025/123”
 REF_RX1 = %r{\bDA\s*(20\d{2})\s*/\s*([A-Za-z0-9\-_.]+)}i
 REF_RX2 = %r{\bDA(20\d{2})\s*[-\/]?\s*([0-9]{3,})\b}i
@@ -179,30 +163,24 @@ items = parse_items(doc, URL)
 puts "Found #{items.length} item(s) for #{TABLE}"
 
 items.each do |r|
-  DB.upsert(TABLE, {
-    description: r[:description],
-    date_received: r[:date_received],
-    date_received_raw: r[:date_received_raw],
-    address: r[:address],
-    council_reference: r[:council_reference],
-    applicant: "",
-    owner: ""
-  })
-  
-  enrich_after_upsert!(
+  upsert_and_enrich!(
     table: TABLE,
-    council_reference: council_reference,
-    address: address
+    row: {
+      description: r[:description],
+      date_received: r[:date_received],
+      date_received_raw: r[:date_received_raw],
+      address: r[:address],
+      council_reference: r[:council_reference],
+      applicant: "",
+      owner: ""
+    },
+    extras: {
+      document_url:     r[:document_url],
+      on_notice_to:     r[:date_received],
+      on_notice_to_raw: r[:date_received_raw],
+      title_reference:  r[:title_reference]
+    }
   )
-
-  begin
-    upd = DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET document_url = ?, on_notice_to = ?, on_notice_to_raw = ?, title_reference = ? WHERE council_reference = ? AND address = ?")
-    upd.execute(r[:document_url], r[:date_received], r[:date_received_raw], r[:title_reference], r[:council_reference], r[:address])
-  rescue => e
-    warn "Extras update skipped for #{r[:council_reference]}: #{e.class} #{e.message}"
-  end
-
-  puts "Upserted #{r[:council_reference]} -> #{r[:address]}"
 end
 
 puts "Done #{TABLE}."

scrapers/tasman.rb

@@ -3,27 +3,14 @@
 require "nokogiri"
 require "date"
 require_relative "../lib/http"
-require_relative "../lib/db"
 require_relative "../lib/util"
-require_relative "../lib/enrich"
+require_relative "../lib/scraper_helpers"
 
 TABLE = ENV.fetch("TABLE_NAME")  # run_all.sh sets from filename: da_tasman
 URL   = "https://tasman.tas.gov.au/advertised-applications/"
 
 DB.ensure_table!(TABLE)
 
-# Optional: store the PDF link
-begin
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS document_url VARCHAR(1024) NULL")
-rescue => e
-  warn "document_url add skipped: #{e.class} #{e.message}"
-end
-
-def abs_url(base, href)
-  return "" if href.to_s.strip.empty?
-  URI.join(base, href).to_s rescue href.to_s
-end
-
 # Nokogiri CSS :contains(...) is a bit special. Use a safer find for the Date row.
 def find_date_from_details(row)
   details = row.at_css(".details")
@@ -82,31 +69,19 @@ items.each_with_index do |row, idx|
   # Require core fields
   next if council_reference.empty? || address.empty?
 
-  DB.upsert(TABLE, {
-    description: description,
-    date_received: date_received,
-    date_received_raw: date_received_raw,
-    address: address,
-    council_reference: council_reference,
-    applicant: "",
-    owner: ""
-  })
-  
-  enrich_after_upsert!(
+  upsert_and_enrich!(
     table: TABLE,
-    council_reference: council_reference,
-    address: address
+    row: {
+      description: description,
+      date_received: date_received,
+      date_received_raw: date_received_raw,
+      address: address,
+      council_reference: council_reference,
+      applicant: "",
+      owner: ""
+    },
+    extras: { document_url: document_url }
   )
-
-  # Save the PDF link if the column exists
-  begin
-    upd = DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET document_url = ? WHERE council_reference = ? AND address = ?")
-    upd.execute(document_url, council_reference, address)
-  rescue Mysql2::Error => e
-    warn "[tasman] db update skipped for #{council_reference}: #{e.message}"
-  end
-
-  puts "Upserted #{council_reference} -> #{address}"
   saved += 1
 end
 

scrapers/westtamar.rb

@@ -2,29 +2,14 @@
 
 require "nokogiri"
 require_relative "../lib/http"
-require_relative "../lib/db"
 require_relative "../lib/util"
-require_relative "../lib/enrich"
+require_relative "../lib/scraper_helpers"
 
 TABLE    = ENV.fetch("TABLE_NAME")   # run_all.sh -> da_westtamar
 URL = "https://www.wtc.tas.gov.au/advertised-planning-applications/"
 
 DB.ensure_table!(TABLE)
 
-begin
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS document_url TEXT NULL")
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS on_notice_to DATE NULL")
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS on_notice_to_raw VARCHAR(80) NULL")
-  DB.client.query("ALTER TABLE `#{DB.client.escape(TABLE)}` ADD COLUMN IF NOT EXISTS title_reference TEXT NULL")
-rescue => e
-  warn "Optional column add skipped: #{e.class} #{e.message}"
-end
-
-def abs_url(base, href)
-  return "" if href.to_s.strip.empty?
-  URI.join(base, href).to_s rescue href.to_s
-end
-
 REF_RX_SLASH  = %r{\bDA\s*(20\d{2})\s*/\s*([A-Za-z0-9\-_.]+)}i
 REF_RX_HYPHEN = %r{\bDA\s*(\d{1,4})\s*-\s*(20\d{2})\b}i
 def extract_ref(text)
@@ -135,30 +120,24 @@ detail_links.each do |u|
   end
   next unless item
 
-  DB.upsert(TABLE, {
-    description: item[:description],
-    date_received: item[:date_received],
-    date_received_raw: item[:date_received_raw],
-    address: item[:address],
-    council_reference: item[:council_reference],
-    applicant: "",
-    owner: ""
-  })
-  
-  enrich_after_upsert!(
+  upsert_and_enrich!(
     table: TABLE,
-    council_reference: council_reference,
-    address: address
+    row: {
+      description: item[:description],
+      date_received: item[:date_received],
+      date_received_raw: item[:date_received_raw],
+      address: item[:address],
+      council_reference: item[:council_reference],
+      applicant: "",
+      owner: ""
+    },
+    extras: {
+      document_url:    item[:document_url],
+      on_notice_to:    item[:date_received],
+      on_notice_to_raw: item[:date_received_raw],
+      title_reference: item[:title_reference]
+    }
   )
-
-  begin
-    upd = DB.client.prepare("UPDATE `#{DB.client.escape(TABLE)}` SET document_url = ?, on_notice_to = ?, on_notice_to_raw = ?, title_reference = ? WHERE council_reference = ? AND address = ?")
-    upd.execute(item[:document_url], item[:date_received], item[:date_received_raw], item[:title_reference], item[:council_reference], item[:address])
-  rescue Mysql2::Error => e
-    warn "[westtamar] db update skipped for #{item[:council_reference]}: #{e.message}"
-  end
-
-  puts "Upserted #{item[:council_reference]} -> #{item[:address]}"
   saved += 1
 end