DB updates

lib/db.rb

@@ -55,6 +55,15 @@ module DB
             document_url TEXT NULL,
             local_document_url TEXT NULL,
             documents_json MEDIUMTEXT NULL,
+            status VARCHAR(100) NULL,
+            assigned_officer VARCHAR(255) NULL,
+            `group` VARCHAR(100) NULL,
+            category VARCHAR(100) NULL,
+            application_valid DATE NULL,
+            application_valid_raw VARCHAR(80) NULL,
+            advertised_on DATE NULL,
+            advertised_on_raw VARCHAR(80) NULL,
+            property_legal_description TEXT NULL,
             lat DECIMAL(10,7) NULL,
             lng DECIMAL(10,7) NULL,
             PRIMARY KEY (id),

lib/migrate.rb

@@ -95,6 +95,33 @@ module Migrate
           Log.warn "migrate", "skipped #{table}.documents_json: #{e.message}"
         end
       }
+    },
+    {
+      version: 4,
+      description: "Add application detail columns (status, assigned_officer, group, category, application_valid, advertised_on, property_legal_description)",
+      up: -> {
+        cols = {
+          "status"                      => "VARCHAR(100) NULL",
+          "assigned_officer"            => "VARCHAR(255) NULL",
+          "`group`"                     => "VARCHAR(100) NULL",
+          "category"                    => "VARCHAR(100) NULL",
+          "application_valid"           => "DATE NULL",
+          "application_valid_raw"       => "VARCHAR(80) NULL",
+          "advertised_on"               => "DATE NULL",
+          "advertised_on_raw"           => "VARCHAR(80) NULL",
+          "property_legal_description"  => "TEXT NULL"
+        }
+        Migrate.da_tables.each do |table|
+          esc = DB.client.escape(table)
+          cols.each do |col, defn|
+            DB.client.query(
+              "ALTER TABLE `#{esc}` ADD COLUMN IF NOT EXISTS #{col} #{defn}"
+            )
+          rescue Mysql2::Error => e
+            Log.warn "migrate", "skipped #{table}.#{col}: #{e.message}"
+          end
+        end
+      }
     }
   ].freeze
 

scrapers/launcestoncity.rb

@@ -438,14 +438,23 @@ tables.each do |t|
 		puts "Found #{documents.size} docs for #{council_reference}" if doc_list_url
 
 		DB.upsert(TABLE, {
-		  council_reference:  council_reference,
-		  address:            address,
-		  applicant:          applicant_name,
-		  date_received:      received_date,
-		  date_received_raw:  received_raw,
-		  document_url:       first_doc_url,
-		  local_document_url: first_local,
-		  documents_json:     documents.empty? ? nil : JSON.generate(documents)
+		  council_reference:          council_reference,
+		  address:                    address,
+		  applicant:                  applicant_name,
+		  date_received:              received_date,
+		  date_received_raw:          received_raw,
+		  document_url:               first_doc_url,
+		  local_document_url:         first_local,
+		  documents_json:             documents.empty? ? nil : JSON.generate(documents),
+		  status:                     status_text,
+		  assigned_officer:           assigned_off,
+		  group:                      group_text,
+		  category:                   category_text,
+		  application_valid:          valid_date,
+		  application_valid_raw:      valid_raw,
+		  advertised_on:              advertised_date,
+		  advertised_on_raw:          advertised_raw,
+		  property_legal_description: legal_desc
 		})
 
     rescue StandardError => e