Knowledge Base

.claude/settings.local.json

@@ -2,7 +2,12 @@
   "permissions": {
     "allow": [
       "WebFetch(domain:raw.githubusercontent.com)",
-      "Bash(find /f/GIT_REPO/crop_monitor -name .env* -o -name *.env -o -name config.php)"
+      "Bash(find /f/GIT_REPO/crop_monitor -name .env* -o -name *.env -o -name config.php)",
+      "Bash(composer require:*)",
+      "Bash(where composer:*)",
+      "Read(//c/ProgramData/ComposerSetup/**)",
+      "Read(//c/Users/lumion/AppData/Roaming/Composer/vendor/**)",
+      "Bash(find /c -name composer.phar)"
     ]
   }
 }

composer.json

@@ -1,5 +1,6 @@
 {
     "require": {
-        "phpmailer/phpmailer": "^6.9"
+        "phpmailer/phpmailer": "^6.9",
+        "smalot/pdfparser": "^2.0"
     }
 }

controllers/ollamaGenerate.php

@@ -2,17 +2,22 @@
 /**
  * controllers/ollamaGenerate.php
  *
- * AJAX POST handler: sends soil test data to a local Ollama instance and
- * returns an AI-generated agronomic interpretation for the requested section.
+ * AJAX POST handler: generates AI agronomic text using Ollama, grounded
+ * with relevant passages retrieved from the soil science knowledge base
+ * (William A. Albrecht et al.) via RAG (Retrieval-Augmented Generation).
  *
- * Expected POST params:
- *   csrf_token  string  CSRF token
+ * Flow:
+ *  1. Load full soil record + specification ranges
+ *  2. Build a structured data summary covering ALL measured elements
+ *  3. Embed that summary via nomic-embed-text → retrieve top-K book passages
+ *  4. Inject retrieved passages + data into a section-specific prompt
+ *  5. Send to llama3.1 and return the generated text
+ *
+ * POST params:
+ *   csrf_token  string
  *   rid         int     soil_records.id
- *   rand        string  soil_records.rand (ownership token)
+ *   rand        string  soil_records.rand
  *   section     string  overview | ai_interpretation | foliar | microbial
- *
- * Requires Ollama running on http://localhost:11434
- * Default model: llama3.2 — change OLLAMA_MODEL below to suit your setup.
  */
 
 if (session_status() === PHP_SESSION_NONE) {
@@ -25,7 +30,14 @@ require_once __DIR__ . '/../lib/csrf.php';
 
 header('Content-Type: application/json');
 
-// ── Auth + CSRF ─────────────────────────────────────────────────────────────
+// ── Config ───────────────────────────────────────────────────────────────────
+define('OLLAMA_HOST',      'http://192.168.8.73:11434');
+define('OLLAMA_MODEL',     'llama3.1:8b-instruct-q4_K_M');
+define('EMBED_MODEL',      'nomic-embed-text');
+define('RAG_TOP_K',        6);    // number of knowledge chunks to inject per request
+define('OLLAMA_TIMEOUT',   180);  // seconds
+
+// ── Auth + CSRF ───────────────────────────────────────────────────────────────
 if (!isLoggedIn()) {
     http_response_code(401);
     echo json_encode(['success' => false, 'error' => 'Not authenticated']);
@@ -44,10 +56,9 @@ if (!verifyCsrfToken($_POST['csrf_token'] ?? '')) {
     exit;
 }
 
-// ── Input validation ────────────────────────────────────────────────────────
-$recordId = (int)trim($_POST['rid']  ?? '');
-$randId   = trim($_POST['rand']      ?? '');
-$section  = trim($_POST['section']   ?? '');
+$recordId = (int)trim($_POST['rid']   ?? '');
+$randId   = trim($_POST['rand']       ?? '');
+$section  = trim($_POST['section']    ?? '');
 
 $validSections = ['overview', 'ai_interpretation', 'foliar', 'microbial'];
 if (!$recordId || $randId === '' || !in_array($section, $validSections, true)) {
@@ -56,10 +67,9 @@ if (!$recordId || $randId === '' || !in_array($section, $validSections, true)) {
     exit;
 }
 
-// ── Load data ───────────────────────────────────────────────────────────────
+// ── Load soil record + spec ───────────────────────────────────────────────────
 try {
-    $pdo    = getDBConnection();
-    $userId = getCurrentUserId();
+    $pdo = getDBConnection();
 
     $stmt = $pdo->prepare('SELECT * FROM soil_records WHERE id = ? AND rand = ?');
     $stmt->execute([$recordId, $randId]);
@@ -85,141 +95,251 @@ try {
     exit;
 }
 
-// ── Build soil data summary for the prompt ──────────────────────────────────
-
-/** Helper: format a numeric value safely, '' → 'N/A' */
+// ── Helper: safe float format ────────────────────────────────────────────────
 function fv(mixed $v, int $dp = 2): string
 {
     if ($v === null || $v === '') return 'N/A';
     return is_numeric($v) ? number_format((float)$v, $dp) : (string)$v;
 }
 
-/** Compare a value to a min/max range, return status string */
+// ── Helper: status vs spec range ─────────────────────────────────────────────
 function rangeStatus(mixed $value, mixed $min, mixed $max): string
 {
     if (!is_numeric($value)) return '';
     $v  = (float)$value;
     $lo = is_numeric($min) ? (float)$min : null;
     $hi = is_numeric($max) ? (float)$max : null;
-    if ($lo !== null && $v < $lo) return 'DEFICIENT';
-    if ($hi !== null && $v > $hi) return 'EXCESS';
-    return 'IDEAL';
+    if ($lo !== null && $v < $lo) return '[DEFICIENT]';
+    if ($hi !== null && $v > $hi) return '[EXCESS]';
+    if ($lo !== null || $hi !== null) return '[IDEAL]';
+    return '';
+}
+
+// ── Helper: resolve spec value from spec row then record row ─────────────────
+function sv(array $spec, array $row, string $col): mixed
+{
+    if (isset($spec[$col]) && $spec[$col] !== '' && $spec[$col] !== null) return $spec[$col];
+    if (isset($row[$col])  && $row[$col]  !== '' && $row[$col]  !== null) return $row[$col];
+    return null;
 }
 
 $r = $row;
 $s = $spec;
 
-$soilSummary = <<<TEXT
-SOIL TEST RESULTS
-=================
+// ── Build comprehensive soil data block ───────────────────────────────────────
+// Includes ALL measured elements with status against spec targets
+$soilData = <<<TEXT
+=====================================
+SOIL TEST DATA — COMPLETE ANALYSIS
+=====================================
 Client:       {$r['client_name']}
 Location:     {$r['site_address']}, {$r['state_postcode']}
 Crop:         {$r['sample_id']}
+Crop Type:    {$r['crop_type']}
 Soil Type:    {$r['soil_type']}
-Lab Number:   {$r['lab_no']}
-Date Sampled: {$r['date_sampled']}
+Lab No:       {$r['lab_no']}
 Batch:        {$r['batch_no']}
+Date Sampled: {$r['date_sampled']}
 
-PHYSICAL / GENERAL
-------------------
-pH (H2O):           {fv($r['ph_h2o'],   1)}  [target: 6.0–7.0]   {rangeStatus($r['ph_h2o'], 6.0, 7.0)}
-pH (CaCl2):         {fv($r['ph_cacl2'], 1)}
-EC (mS/cm):         {fv($r['ec'],       2)}
-Organic Carbon (%): {fv($r['ocarbon'],  1)}
-Organic Matter (%): {fv($r['omatter'],  1)}
-CEC:                {fv($r['cec'],      2)}
-TEC:                {fv($r['tec'],      2)}
-
-MAJOR ELEMENTS (ppm)
----------------------
-Calcium (Ca):       {fv($r['BS_ca_ppm'], 0)}  [min: {fv($s['ca_ppm_min'] ?? $r['ca_ppm_min'], 0)}, max: {fv($s['ca_ppm_max'] ?? $r['ca_ppm_max'], 0)}]  {rangeStatus($r['BS_ca_ppm'], $s['ca_ppm_min'] ?? $r['ca_ppm_min'] ?? null, $s['ca_ppm_max'] ?? $r['ca_ppm_max'] ?? null)}
-Magnesium (Mg):     {fv($r['BS_mg_ppm'], 0)}  [min: {fv($s['mg_ppm_min'] ?? $r['mg_ppm_min'], 0)}, max: {fv($s['mg_ppm_max'] ?? $r['mg_ppm_max'], 0)}]  {rangeStatus($r['BS_mg_ppm'], $s['mg_ppm_min'] ?? $r['mg_ppm_min'] ?? null, $s['mg_ppm_max'] ?? $r['mg_ppm_max'] ?? null)}
-Potassium (K):      {fv($r['BS_k_ppm'],  0)}  [min: {fv($s['k_ppm_min']  ?? $r['k_ppm_min'],  0)}, max: {fv($s['k_ppm_max']  ?? $r['k_ppm_max'],  0)}]  {rangeStatus($r['BS_k_ppm'],  $s['k_ppm_min']  ?? $r['k_ppm_min']  ?? null, $s['k_ppm_max']  ?? $r['k_ppm_max']  ?? null)}
-Sodium (Na):        {fv($r['BS_na_ppm'], 0)}  [min: {fv($s['na_ppm_min'] ?? $r['na_ppm_min'], 0)}, max: {fv($s['na_ppm_max'] ?? $r['na_ppm_max'], 0)}]  {rangeStatus($r['BS_na_ppm'], $s['na_ppm_min'] ?? $r['na_ppm_min'] ?? null, $s['na_ppm_max'] ?? $r['na_ppm_max'] ?? null)}
-Nitrate-N:          {fv($r['NO3_N'],    0)} ppm
-Ammonium-N:         {fv($r['NH3_N'],    0)} ppm
-Phosphate (P):      {fv($r['p_colwell'],0)} ppm  (Colwell)
-
-TRACE ELEMENTS (ppm)
---------------------
-Sulfur (S):         {fv($r['s_morgan'],  2)}
-Boron (B):          {fv($r['b_cacl2'],   2)}
-Manganese (Mn):     {fv($r['mn_dtpa'],   2)}
-Copper (Cu):        {fv($r['cu_dtpa'],   2)}
-Zinc (Zn):          {fv($r['zn_dtpa'],   2)}
-Iron (Fe):          {fv($r['fe_dtpa'],   2)}
-Aluminium (Al):     {fv($r['al'],        2)}
-Silicon (Si):       {fv($r['sl_cacl2'],  2)}
-
-BASE SATURATIONS (%)
---------------------
-Calcium (Ca):       {fv($r['BS_ca2'],  2)}%  [min: {fv($s['cabs_min'] ?? null)}, max: {fv($s['cabs_max'] ?? null)}]  {rangeStatus($r['BS_ca2'], $s['cabs_min'] ?? null, $s['cabs_max'] ?? null)}
-Magnesium (Mg):     {fv($r['BS_mg2'],  2)}%  [min: {fv($s['mgbs_min'] ?? null)}, max: {fv($s['mgbs_max'] ?? null)}]  {rangeStatus($r['BS_mg2'], $s['mgbs_min'] ?? null, $s['mgbs_max'] ?? null)}
-Potassium (K):      {fv($r['BS_k'],    2)}%  [min: {fv($s['kbs_min']  ?? null)}, max: {fv($s['kbs_max']  ?? null)}]  {rangeStatus($r['BS_k'],   $s['kbs_min']  ?? null, $s['kbs_max']  ?? null)}
-Sodium (Na):        {fv($r['BS_na'],   2)}%
-Other Bases:        {fv($r['BS_ob'],   2)}%
-Hydrogen:           {fv($r['BS_h'],    2)}%
-
-RATIOS
-------
-Ca:Mg ratio:        {fv(is_numeric($r['ca_mehlick3']) && is_numeric($r['mg_mehlick3']) && (float)$r['mg_mehlick3'] != 0 ? (float)$r['ca_mehlick3'] / (float)$r['mg_mehlick3'] : null, 1)}  [recommended: {fv($s['ca_mg_ratio'] ?? null, 1)}]
-C:N ratio:          {fv($r['c_n_ratio'], 1)}
+--- SOIL PHYSICAL / REACTION ---
+pH (H2O):                {fv($r['ph_h2o'],   1)}   [target: 6.2–6.8]  {rangeStatus($r['ph_h2o'], 6.2, 6.8)}
+pH (CaCl2):              {fv($r['ph_cacl2'], 1)}
+EC (mS/cm):              {fv($r['ec'],       2)}
+Colour:                  {$r['colour']}
+Texture:                 {$r['texture']}
+Gravel (%):              {fv($r['gravel'], 1)}
+
+--- ORGANIC MATTER ---
+Organic Carbon (%):      {fv($r['ocarbon'], 1)}
+Organic Matter (%):      {fv($r['omatter'], 1)}
+
+--- CATION EXCHANGE ---
+CEC (meq/100g):          {fv($r['cec'], 2)}
+TEC (meq/100g):          {fv($r['tec'], 2)}
+Paramagnetic:            {fv($r['paramag'], 0)}
+
+--- NITROGEN ---
+Nitrate-N (NO3-N ppm):   {fv($r['NO3_N'],   0)}   [target: 10–20 ppm]  {rangeStatus($r['NO3_N'], 10, 20)}
+Ammonium-N (NH3-N ppm):  {fv($r['NH3_N'],   0)}
+Total N (est. from C:N): C:N ratio {fv($r['c_n_ratio'], 1)}
+
+--- PHOSPHORUS ---
+P Colwell (ppm):         {fv($r['p_colwell'], 0)}
+P Morgan (ppm):          {fv($r['p_morgan'],  0)}
+P Mehlick (ppm):         {fv($r['p_mehlick'], 0)}
+P Bray2 (ppm):           {fv($r['p_bray2'],   0)}
+
+--- MAJOR CATIONS (ppm) ---
+Calcium Ca  (ppm):       {fv($r['BS_ca_ppm'], 0)}   [min: {fv(sv($s,$r,'ca_ppm_min'),0)}, max: {fv(sv($s,$r,'ca_ppm_max'),0)}]  {rangeStatus($r['BS_ca_ppm'], sv($s,$r,'ca_ppm_min'), sv($s,$r,'ca_ppm_max'))}
+Magnesium Mg (ppm):      {fv($r['BS_mg_ppm'], 0)}   [min: {fv(sv($s,$r,'mg_ppm_min'),0)}, max: {fv(sv($s,$r,'mg_ppm_max'),0)}]  {rangeStatus($r['BS_mg_ppm'], sv($s,$r,'mg_ppm_min'), sv($s,$r,'mg_ppm_max'))}
+Potassium K  (ppm):      {fv($r['BS_k_ppm'],  0)}   [min: {fv(sv($s,$r,'k_ppm_min'), 0)}, max: {fv(sv($s,$r,'k_ppm_max'), 0)}]  {rangeStatus($r['BS_k_ppm'],  sv($s,$r,'k_ppm_min'),  sv($s,$r,'k_ppm_max'))}
+Sodium Na    (ppm):      {fv($r['BS_na_ppm'], 0)}   [min: {fv(sv($s,$r,'na_ppm_min'),0)}, max: {fv(sv($s,$r,'na_ppm_max'),0)}]  {rangeStatus($r['BS_na_ppm'], sv($s,$r,'na_ppm_min'), sv($s,$r,'na_ppm_max'))}
+
+--- BASE SATURATIONS (%) ---
+Calcium  Ca (%):         {fv($r['BS_ca2'], 2)}%   [min: {fv(sv($s,$r,'cabs_min'),1)}, max: {fv(sv($s,$r,'cabs_max'),1)}]  {rangeStatus($r['BS_ca2'], sv($s,$r,'cabs_min'), sv($s,$r,'cabs_max'))}
+Magnesium Mg (%):        {fv($r['BS_mg2'], 2)}%   [min: {fv(sv($s,$r,'mgbs_min'),1)}, max: {fv(sv($s,$r,'mgbs_max'),1)}]  {rangeStatus($r['BS_mg2'], sv($s,$r,'mgbs_min'), sv($s,$r,'mgbs_max'))}
+Potassium  K  (%):       {fv($r['BS_k'],   2)}%   [min: {fv(sv($s,$r,'kbs_min'), 1)}, max: {fv(sv($s,$r,'kbs_max'), 1)}]  {rangeStatus($r['BS_k'],   sv($s,$r,'kbs_min'),  sv($s,$r,'kbs_max'))}
+Sodium     Na (%):       {fv($r['BS_na'],  2)}%   [min: {fv(sv($s,$r,'nabs_min'),1)}, max: {fv(sv($s,$r,'nabs_max'),1)}]  {rangeStatus($r['BS_na'],  sv($s,$r,'nabs_min'), sv($s,$r,'nabs_max'))}
+Other Bases (%):         {fv($r['BS_ob'],  2)}%   [recommended: {fv(sv($s,$r,'ob_rec'),1)}]
+Hydrogen    (%):         {fv($r['BS_h'],   2)}%   [recommended: {fv(sv($s,$r,'h_rec'), 1)}]
+Aluminium  Al3 (%):      {fv($r['BS_al3'], 2)}%
+
+--- MORGANS EXTRACT (ppm) ---
+Ca Morgan:               {fv($r['ca_morgan'], 2)}
+Mg Morgan:               {fv($r['mg_morgan'], 2)}
+K  Morgan:               {fv($r['k_morgan'],  2)}
+Na Morgan:               {fv($r['na_morgan'], 2)}
+
+--- MEHLICK-3 EXTRACT (ppm) ---
+Ca Mehlick3:             {fv($r['ca_mehlick3'], 2)}
+Mg Mehlick3:             {fv($r['mg_mehlick3'], 2)}
+K  Mehlick3:             {fv($r['k_mehlick3'],  2)}
+Na Mehlick3:             {fv($r['na_mehlick3'], 2)}
+Al Mehlick3:             {fv($r['al_mehlick3'], 2)}
+
+--- TRACE ELEMENTS (ppm) ---
+Sulfur    S  (ppm):      {fv($r['s_morgan'], 2)}
+Boron     B  (ppm):      {fv($r['b_cacl2'],  2)}
+Manganese Mn (ppm):      {fv($r['mn_dtpa'],  2)}
+Copper    Cu (ppm):      {fv($r['cu_dtpa'],  2)}
+Zinc      Zn (ppm):      {fv($r['zn_dtpa'],  2)}
+Iron      Fe (ppm):      {fv($r['fe_dtpa'],  2)}
+Iron      Fe (total):    {fv($r['fe'],       2)}
+Aluminium Al (ppm):      {fv($r['al'],       2)}
+Silicon   Si (ppm):      {fv($r['sl_cacl2'], 2)}
+Cobalt    Co (ppm):      {fv($r['co_dtpa'],  2)}
+Molybdenum M (ppm):      {fv($r['m_dtpa'],   2)}
+Selenium  Se (ppm):      {fv($r['se'],       2)}
+
+--- RATIOS ---
+Ca:Mg ratio:             {fv(is_numeric($r['ca_mehlick3']) && is_numeric($r['mg_mehlick3']) && (float)$r['mg_mehlick3'] != 0 ? round((float)$r['ca_mehlick3']/(float)$r['mg_mehlick3'],1) : null, 1)}   [recommended: {fv(sv($s,$r,'ca_mg_ratio'),1)}]
+C:N  ratio:              {fv($r['c_n_ratio'], 1)}
+
+--- DEFICIENT ELEMENTS SUMMARY ---
 TEXT;
 
-// ── Section-specific prompts ────────────────────────────────────────────────
+// Append a quick plain-English deficiency list to help the LLM focus
+$deficiencies = [];
+$excesses     = [];
+
+$checkElements = [
+    ['pH (H2O)',          $r['ph_h2o'],    6.2,  6.8],
+    ['Nitrate-N',         $r['NO3_N'],     10,   20],
+    ['Calcium (ppm)',     $r['BS_ca_ppm'], sv($s,$r,'ca_ppm_min'), sv($s,$r,'ca_ppm_max')],
+    ['Magnesium (ppm)',   $r['BS_mg_ppm'], sv($s,$r,'mg_ppm_min'), sv($s,$r,'mg_ppm_max')],
+    ['Potassium (ppm)',   $r['BS_k_ppm'],  sv($s,$r,'k_ppm_min'),  sv($s,$r,'k_ppm_max')],
+    ['Sodium (ppm)',      $r['BS_na_ppm'], sv($s,$r,'na_ppm_min'), sv($s,$r,'na_ppm_max')],
+    ['Ca sat (%)',        $r['BS_ca2'],    sv($s,$r,'cabs_min'),   sv($s,$r,'cabs_max')],
+    ['Mg sat (%)',        $r['BS_mg2'],    sv($s,$r,'mgbs_min'),   sv($s,$r,'mgbs_max')],
+    ['K sat (%)',         $r['BS_k'],      sv($s,$r,'kbs_min'),    sv($s,$r,'kbs_max')],
+    ['Na sat (%)',        $r['BS_na'],     sv($s,$r,'nabs_min'),   sv($s,$r,'nabs_max')],
+];
+
+foreach ($checkElements as [$label, $val, $lo, $hi]) {
+    if (!is_numeric($val)) continue;
+    $v = (float)$val;
+    if (is_numeric($lo) && $v < (float)$lo) $deficiencies[] = $label;
+    if (is_numeric($hi) && $v > (float)$hi) $excesses[]     = $label;
+}
+
+$soilData .= "\nDeficient: " . (empty($deficiencies) ? 'None detected' : implode(', ', $deficiencies));
+$soilData .= "\nIn Excess: " . (empty($excesses)     ? 'None detected' : implode(', ', $excesses));
+$soilData .= "\n=====================================\n";
+
+// ── RAG: embed the soil data query, retrieve relevant book passages ───────────
+$knowledgeContext = '';
+$ragChunks        = retrieveRelevantChunks($pdo, $soilData, $section, RAG_TOP_K);
+
+if (!empty($ragChunks)) {
+    $knowledgeContext = "\n\n===================================================\n"
+                      . "RELEVANT PASSAGES FROM SOIL SCIENCE LITERATURE\n"
+                      . "(William A. Albrecht and other authorities)\n"
+                      . "===================================================\n";
+    foreach ($ragChunks as $i => $chunk) {
+        $knowledgeContext .= sprintf(
+            "\n[%d] \"%s\" — %s (p.%d)\n%s\n",
+            $i + 1,
+            $chunk['source'],
+            $chunk['author'],
+            $chunk['page'],
+            $chunk['chunk_text']
+        );
+    }
+}
+
+// ── Section-specific system prompts ──────────────────────────────────────────
+$systemInstruction = "You are a certified agronomist specialising in soil fertility, "
+    . "trained in the Albrecht method of soil balancing. "
+    . "You have deep knowledge of soil chemistry, plant nutrition, and the relationship "
+    . "between soil mineral balance and crop/livestock health. "
+    . "Always ground your recommendations in the measured data. "
+    . "For Australian conditions, reference typical soil types and climate where relevant. "
+    . "Write in a professional but accessible tone suitable for a farmer-facing report. "
+    . "When the knowledge passages conflict with your training, prefer the passages — they "
+    . "are from authoritative soil science texts.";
+
+$baseContext = $soilData . $knowledgeContext;
+
 $prompts = [
 
     'overview' =>
-        "You are an experienced Australian agronomist writing a professional soil analysis report.\n\n"
-        . $soilSummary
-        . "\n\nWrite a concise executive overview (3–4 paragraphs) suitable for a farmer. "
-        . "Summarise the overall soil health, the most important deficiencies or imbalances, "
-        . "and their likely impact on crop performance. Use plain language — avoid jargon. "
-        . "Do not include specific product recommendations in this section.",
+        $systemInstruction . "\n\n" . $baseContext
+        . "\n\nTASK: Write an executive overview of these soil test results (3–4 paragraphs). "
+        . "Cover: (1) overall soil health and fertility level, "
+        . "(2) the most significant deficiencies or imbalances and their likely effect on crop performance, "
+        . "(3) any positive attributes of this soil. "
+        . "Use the Albrecht philosophy as a framework where applicable. "
+        . "Do not list specific product names in this section.",
 
     'ai_interpretation' =>
-        "You are an experienced Australian agronomist.\n\n"
-        . $soilSummary
-        . "\n\nProvide a detailed technical interpretation of these soil test results. "
-        . "For each element that is DEFICIENT or in EXCESS, explain: "
-        . "(1) the agronomic significance, "
-        . "(2) the likely cause in Australian soils, "
-        . "(3) the interactions with other nutrients shown in this test. "
-        . "Also comment on the base saturation balance, pH implications, and organic matter. "
-        . "Write in a professional tone suitable for an agronomist's report.",
+        $systemInstruction . "\n\n" . $baseContext
+        . "\n\nTASK: Write a detailed technical interpretation of ALL elements in this soil test. "
+        . "Structure your response with these sections:\n"
+        . "1. SOIL REACTION (pH, EC, Paramagnetic)\n"
+        . "2. ORGANIC MATTER & BIOLOGY (C, N, C:N ratio)\n"
+        . "3. CATION EXCHANGE CAPACITY & BASE SATURATIONS\n"
+        . "4. MAJOR ELEMENTS (Ca, Mg, K, Na, P — ppm and saturation %)\n"
+        . "5. TRACE ELEMENTS (S, B, Mn, Cu, Zn, Fe, Al, Si, Co, Mo, Se)\n"
+        . "6. ELEMENTAL RATIOS & INTERACTIONS (Ca:Mg, C:N, K:Mg antagonisms)\n"
+        . "7. OVERALL SOIL BALANCE ASSESSMENT\n"
+        . "For each element marked [DEFICIENT] or [EXCESS], explain the agronomic significance "
+        . "and interactions with other elements. Reference the Albrecht literature where relevant.",
 
     'foliar' =>
-        "You are an experienced Australian agronomist.\n\n"
-        . $soilSummary
-        . "\n\nDesign a foliar spray program to correct the nutrient deficiencies shown in these soil test results. "
-        . "List recommended applications by growth stage, including product types, rates (L/ha or kg/ha), "
-        . "and timing. Focus on elements that are DEFICIENT. "
-        . "Note any antagonisms to avoid (e.g. Ca and Mg competing). "
-        . "Write as a practical program a farmer can follow.",
+        $systemInstruction . "\n\n" . $baseContext
+        . "\n\nTASK: Design a foliar nutrition program to address the deficiencies shown. "
+        . "Format the program as a table or numbered list with: "
+        . "Growth Stage | Product Type | Active Element | Rate (L or kg/ha) | Timing/Frequency. "
+        . "Prioritise elements marked [DEFICIENT]. "
+        . "Note any antagonisms (e.g. Ca/Mg competition, Zn/P interaction, K/Mg lockout). "
+        . "Keep product recommendations generic (e.g. 'chelated zinc', 'calcium nitrate') "
+        . "rather than brand names. "
+        . "Add a note on carrier water pH and adjuvant recommendations.",
 
     'microbial' =>
-        "You are an experienced Australian agronomist with expertise in soil biology.\n\n"
-        . $soilSummary
-        . "\n\nDesign a microbial and biological soil program suited to these test results. "
-        . "Consider: the organic matter level, pH, and nutrient imbalances shown. "
-        . "Recommend specific microbial inoculants, compost teas, or biological amendments "
-        . "that would improve soil biology and nutrient cycling for the crop type listed. "
-        . "Include timing and application rates where possible.",
+        $systemInstruction . "\n\n" . $baseContext
+        . "\n\nTASK: Design a biological/microbial soil improvement program. "
+        . "Consider the organic matter level, C:N ratio, pH, and base saturation balance shown. "
+        . "Structure your response:\n"
+        . "1. CURRENT BIOLOGY ASSESSMENT (based on OM, C:N, pH)\n"
+        . "2. RECOMMENDED INOCULANTS (e.g. mycorrhizae, rhizobia, EM, compost tea)\n"
+        . "3. CARBON FEEDING STRATEGY (humates, fish hydrolysate, molasses, cover crops)\n"
+        . "4. TIMING & INTEGRATION with the soil balancing program\n"
+        . "Reference Albrecht's work on the relationship between mineral balance and soil biology.",
 ];
 
-// ── Call Ollama ─────────────────────────────────────────────────────────────
-define('OLLAMA_URL',   'http://localhost:11434/api/generate');
-define('OLLAMA_MODEL', 'llama3.2');   // change to match your installed model
-define('OLLAMA_TIMEOUT', 120);        // seconds — LLM can be slow
-
-$prompt  = $prompts[$section];
+// ── Call Ollama ───────────────────────────────────────────────────────────────
 $payload = json_encode([
     'model'  => OLLAMA_MODEL,
-    'prompt' => $prompt,
+    'prompt' => $prompts[$section],
     'stream' => false,
+    'options' => [
+        'temperature' => 0.3,   // lower = more factual / less creative
+        'num_predict' => 2048,
+    ],
 ]);
 
-$ch = curl_init(OLLAMA_URL);
+$ch = curl_init(OLLAMA_HOST . '/api/generate');
 curl_setopt_array($ch, [
     CURLOPT_POST           => true,
     CURLOPT_POSTFIELDS     => $payload,
@@ -236,19 +356,13 @@ curl_close($ch);
 
 if ($curlErr || $response === false) {
     http_response_code(502);
-    echo json_encode([
-        'success' => false,
-        'error'   => 'Could not connect to Ollama: ' . ($curlErr ?: 'no response'),
-    ]);
+    echo json_encode(['success' => false, 'error' => 'Could not connect to Ollama: ' . ($curlErr ?: 'no response')]);
     exit;
 }
 
 if ($httpCode !== 200) {
     http_response_code(502);
-    echo json_encode([
-        'success' => false,
-        'error'   => 'Ollama returned HTTP ' . $httpCode,
-    ]);
+    echo json_encode(['success' => false, 'error' => 'Ollama returned HTTP ' . $httpCode]);
     exit;
 }
 
@@ -261,5 +375,158 @@ if ($text === '') {
     exit;
 }
 
-echo json_encode(['success' => true, 'text' => $text]);
+echo json_encode([
+    'success' => true,
+    'text'    => $text,
+    'rag_chunks_used' => count($ragChunks),
+]);
 exit;
+
+// ── RAG retrieval ────────────────────────────────────────────────────────────
+
+/**
+ * Embed a query string, then retrieve the top-K most similar knowledge chunks.
+ * Falls back to MySQL FULLTEXT search if no embeddings are in the table or
+ * if the embedding API is unavailable.
+ *
+ * @param PDO    $pdo
+ * @param string $queryText  The soil data summary used as the retrieval query
+ * @param string $section    Current section (used to build keyword fallback)
+ * @param int    $topK
+ * @return array  Array of row arrays (source, author, page, chunk_text)
+ */
+function retrieveRelevantChunks(PDO $pdo, string $queryText, string $section, int $topK): array
+{
+    // Check if we have any chunks at all
+    $count = (int)$pdo->query('SELECT COUNT(*) FROM knowledge_chunks')->fetchColumn();
+    if ($count === 0) {
+        return [];  // Knowledge base not yet populated
+    }
+
+    // ── Try vector similarity search first ──────────────────────────────────
+    $queryEmbedding = getQueryEmbedding($queryText);
+
+    if ($queryEmbedding !== null) {
+        return vectorSearch($pdo, $queryEmbedding, $topK);
+    }
+
+    // ── Fallback: MySQL FULLTEXT search ─────────────────────────────────────
+    return fulltextSearch($pdo, $section, $topK);
+}
+
+/**
+ * Embed text via Ollama /api/embeddings. Returns float[] or null.
+ */
+function getQueryEmbedding(string $text): ?array
+{
+    // Use a shorter representative string for the query (first 2000 chars)
+    $queryText = substr($text, 0, 2000);
+
+    $payload = json_encode([
+        'model'  => EMBED_MODEL,
+        'prompt' => $queryText,
+    ]);
+
+    $ch = curl_init(OLLAMA_HOST . '/api/embeddings');
+    curl_setopt_array($ch, [
+        CURLOPT_POST           => true,
+        CURLOPT_POSTFIELDS     => $payload,
+        CURLOPT_HTTPHEADER     => ['Content-Type: application/json'],
+        CURLOPT_RETURNTRANSFER => true,
+        CURLOPT_TIMEOUT        => 15,
+        CURLOPT_CONNECTTIMEOUT => 3,
+    ]);
+
+    $response = curl_exec($ch);
+    $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+    curl_close($ch);
+
+    if (!$response || $httpCode !== 200) return null;
+
+    $data = json_decode($response, true);
+    $emb  = $data['embedding'] ?? null;
+    return (is_array($emb) && count($emb) > 0) ? $emb : null;
+}
+
+/**
+ * Load all chunk embeddings from DB, compute cosine similarity, return top-K.
+ * For corpora up to ~10k chunks this is fast enough in PHP.
+ */
+function vectorSearch(PDO $pdo, array $queryVec, int $topK): array
+{
+    $stmt = $pdo->query(
+        'SELECT id, source, author, page, chunk_text, embedding FROM knowledge_chunks'
+    );
+
+    $scores = [];
+
+    while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) {
+        $chunkVec = json_decode($row['embedding'], true);
+        if (!is_array($chunkVec)) continue;
+
+        $sim = cosineSimilarity($queryVec, $chunkVec);
+        $scores[] = [
+            'score'      => $sim,
+            'source'     => $row['source'],
+            'author'     => $row['author'],
+            'page'       => $row['page'],
+            'chunk_text' => $row['chunk_text'],
+        ];
+    }
+
+    // Sort descending by score, return top-K
+    usort($scores, fn($a, $b) => $b['score'] <=> $a['score']);
+    return array_slice($scores, 0, $topK);
+}
+
+/**
+ * MySQL FULLTEXT fallback when embeddings aren't available.
+ */
+function fulltextSearch(PDO $pdo, string $section, int $topK): array
+{
+    // Section-specific keyword hints for the search
+    $keywords = [
+        'overview'          => 'soil fertility mineral balance calcium magnesium',
+        'ai_interpretation' => 'base saturation calcium magnesium potassium pH organic matter',
+        'foliar'            => 'foliar nutrition trace elements deficiency correction spray',
+        'microbial'         => 'soil biology microbial organic matter carbon nitrogen humus',
+    ];
+
+    $query = $keywords[$section] ?? 'soil fertility mineral nutrition';
+
+    try {
+        $stmt = $pdo->prepare(
+            'SELECT source, author, page, chunk_text,
+                    MATCH(chunk_text) AGAINST(? IN NATURAL LANGUAGE MODE) AS score
+             FROM knowledge_chunks
+             WHERE MATCH(chunk_text) AGAINST(? IN NATURAL LANGUAGE MODE)
+             ORDER BY score DESC
+             LIMIT ?'
+        );
+        $stmt->execute([$query, $query, $topK]);
+        return $stmt->fetchAll(PDO::FETCH_ASSOC);
+    } catch (PDOException $e) {
+        error_log('RAG fulltext search failed: ' . $e->getMessage());
+        return [];
+    }
+}
+
+/**
+ * Cosine similarity between two equal-length float vectors.
+ */
+function cosineSimilarity(array $a, array $b): float
+{
+    $dot = 0.0;
+    $normA = 0.0;
+    $normB = 0.0;
+    $len = min(count($a), count($b));
+
+    for ($i = 0; $i < $len; $i++) {
+        $dot   += $a[$i] * $b[$i];
+        $normA += $a[$i] * $a[$i];
+        $normB += $b[$i] * $b[$i];
+    }
+
+    $denom = sqrt($normA) * sqrt($normB);
+    return $denom > 0 ? $dot / $denom : 0.0;
+}

dashboard/crop-analysis/soil-test-data/soil-recommendations.php

@@ -101,7 +101,7 @@ include __DIR__ . '/../../../layouts/header.php';
                             <td class="text-center spec-cell px-2"
                                 contenteditable="true"
                                 data-id="<?= (int)$spec['id'] ?>"
-                                data-col="<?= htmlspecialchars($col, ENT_QUOTES, 'UTF-8') ?>">
+                                data-col="<?= htmlspecialchars($col, ENT_QUOTES, 'UTF-8') ?>"><br>
                                 <?= htmlspecialchars($display, ENT_QUOTES, 'UTF-8') ?>
                             </td>
                             <?php endforeach; ?>

database/migrations/002_create_knowledge_base.sql

@@ -0,0 +1,24 @@
+-- ============================================================
+-- 002_create_knowledge_base.sql
+--
+-- Knowledge base for RAG (Retrieval-Augmented Generation).
+-- Stores chunked text from soil science books (Albrecht et al.)
+-- plus their vector embeddings from Ollama nomic-embed-text.
+--
+-- Run once:
+--   mysql -u <user> -p cropmonitor < database/migrations/002_create_knowledge_base.sql
+-- ============================================================
+
+CREATE TABLE IF NOT EXISTS `knowledge_chunks` (
+  `id`          INT UNSIGNED   NOT NULL AUTO_INCREMENT,
+  `source`      VARCHAR(255)   NOT NULL COMMENT 'Book title or filename',
+  `author`      VARCHAR(255)   NOT NULL DEFAULT '' COMMENT 'Author name',
+  `page`        SMALLINT       NOT NULL DEFAULT 0 COMMENT 'Source PDF page number',
+  `chunk_index` SMALLINT       NOT NULL DEFAULT 0 COMMENT 'Chunk position within the source',
+  `chunk_text`  TEXT           NOT NULL COMMENT 'Raw text of this chunk',
+  `embedding`   JSON           NOT NULL COMMENT 'Float array from nomic-embed-text (768 dims)',
+  `created_at`  DATETIME       NOT NULL DEFAULT CURRENT_TIMESTAMP,
+  PRIMARY KEY (`id`),
+  FULLTEXT KEY `ft_chunk_text` (`chunk_text`),
+  KEY `idx_source` (`source`(100))
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;

lib/soil_calculations.php

@@ -176,7 +176,7 @@ function soilAnalysisReportCalcs($symbol, $element, $min, $max, $nutrient, $type
         $result = $value_converted . " " . $measurement;
 
         // Return HTML div
-        return "<div class='{$class}'>{$nutrient}: {$result}</div>";
+        return "<div class='{$class}'>{$nutrient}:<br>{$result}</div>";
 
     } catch (PDOException $e) {
         error_log("Database error in soilAnalysisReportCalcs: " . $e->getMessage());

tools/ingest_knowledge.php

@@ -0,0 +1,297 @@
+<?php
+/**
+ * tools/ingest_knowledge.php
+ *
+ * CLI script: ingests soil science PDF books into the knowledge_chunks table.
+ * Each page is split into overlapping chunks, embedded via Ollama, and stored.
+ *
+ * Usage:
+ *   php tools/ingest_knowledge.php --file="path/to/book.pdf" --author="William A. Albrecht"
+ *   php tools/ingest_knowledge.php --dir="path/to/books/" --author="Various"
+ *   php tools/ingest_knowledge.php --list          (show all indexed sources)
+ *   php tools/ingest_knowledge.php --clear="Book Title"   (remove a source)
+ *
+ * Requirements:
+ *   composer require smalot/pdfparser
+ *   Ollama running with nomic-embed-text pulled:
+ *     ollama pull nomic-embed-text
+ *
+ * The embedding model (nomic-embed-text) produces 768-dimensional vectors.
+ * Each chunk is ~500 words with a 100-word overlap to preserve context across boundaries.
+ */
+
+// ── Must run from CLI ────────────────────────────────────────────────────────
+if (PHP_SAPI !== 'cli') {
+    die("This script must be run from the command line.\n");
+}
+
+define('ROOT', dirname(__DIR__));
+
+require ROOT . '/vendor/autoload.php';
+require ROOT . '/config/database.php';
+
+use Smalot\PdfParser\Parser;
+
+// ── Config ───────────────────────────────────────────────────────────────────
+define('OLLAMA_EMBED_URL', 'http://192.168.8.73:11434/api/embeddings');
+define('EMBED_MODEL',      'nomic-embed-text');
+define('CHUNK_WORDS',      500);   // target words per chunk
+define('OVERLAP_WORDS',    80);    // overlap between consecutive chunks
+
+// ── Parse args ───────────────────────────────────────────────────────────────
+$opts = getopt('', ['file:', 'dir:', 'author:', 'list', 'clear:', 'help']);
+
+if (isset($opts['help']) || (empty($opts['file']) && empty($opts['dir']) && !isset($opts['list']) && empty($opts['clear']))) {
+    echo <<<HELP
+Usage:
+  php tools/ingest_knowledge.php --file="book.pdf" --author="William A. Albrecht"
+  php tools/ingest_knowledge.php --dir="books/"    --author="Various"
+  php tools/ingest_knowledge.php --list
+  php tools/ingest_knowledge.php --clear="Soil Fertility and Animal Health"
+
+Options:
+  --file    Path to a single PDF file
+  --dir     Path to a directory of PDF files (processed recursively)
+  --author  Author name to tag all chunks from this run
+  --list    List all indexed sources with chunk counts
+  --clear   Remove all chunks from a named source
+
+HELP;
+    exit(0);
+}
+
+$pdo = getDBConnection();
+
+// ── List mode ────────────────────────────────────────────────────────────────
+if (isset($opts['list'])) {
+    $stmt = $pdo->query(
+        "SELECT source, author, COUNT(*) AS chunks, MAX(created_at) AS indexed_at
+         FROM knowledge_chunks GROUP BY source, author ORDER BY source"
+    );
+    $rows = $stmt->fetchAll(PDO::FETCH_ASSOC);
+    if (!$rows) {
+        echo "No sources indexed yet.\n";
+    } else {
+        printf("%-55s %-25s %6s  %s\n", 'Source', 'Author', 'Chunks', 'Indexed');
+        echo str_repeat('-', 100) . "\n";
+        foreach ($rows as $r) {
+            printf("%-55s %-25s %6d  %s\n",
+                substr($r['source'], 0, 54),
+                substr($r['author'], 0, 24),
+                $r['chunks'],
+                $r['indexed_at']
+            );
+        }
+    }
+    exit(0);
+}
+
+// ── Clear mode ───────────────────────────────────────────────────────────────
+if (!empty($opts['clear'])) {
+    $title = $opts['clear'];
+    $stmt  = $pdo->prepare('SELECT COUNT(*) FROM knowledge_chunks WHERE source = ?');
+    $stmt->execute([$title]);
+    $count = (int)$stmt->fetchColumn();
+    if ($count === 0) {
+        echo "No chunks found for source: $title\n";
+        exit(0);
+    }
+    $del = $pdo->prepare('DELETE FROM knowledge_chunks WHERE source = ?');
+    $del->execute([$title]);
+    echo "Deleted $count chunks for: $title\n";
+    exit(0);
+}
+
+// ── Collect PDF files ────────────────────────────────────────────────────────
+$files  = [];
+$author = trim($opts['author'] ?? 'Unknown');
+
+if (!empty($opts['file'])) {
+    $path = $opts['file'];
+    if (!is_file($path)) {
+        die("File not found: $path\n");
+    }
+    $files[] = $path;
+}
+
+if (!empty($opts['dir'])) {
+    $dir = rtrim($opts['dir'], '/\\');
+    if (!is_dir($dir)) {
+        die("Directory not found: $dir\n");
+    }
+    $it = new RecursiveIteratorIterator(new RecursiveDirectoryIterator($dir));
+    foreach ($it as $f) {
+        if ($f->isFile() && strtolower($f->getExtension()) === 'pdf') {
+            $files[] = $f->getPathname();
+        }
+    }
+    if (!$files) {
+        die("No PDF files found in: $dir\n");
+    }
+}
+
+echo "Found " . count($files) . " PDF file(s) to ingest.\n\n";
+
+// ── Process each file ────────────────────────────────────────────────────────
+$parser = new Parser();
+
+foreach ($files as $filePath) {
+    $source = pathinfo($filePath, PATHINFO_FILENAME);
+    echo "Processing: $source\n";
+
+    // Check if already indexed
+    $chk = $pdo->prepare('SELECT COUNT(*) FROM knowledge_chunks WHERE source = ?');
+    $chk->execute([$source]);
+    if ((int)$chk->fetchColumn() > 0) {
+        echo "  Already indexed — skipping. Use --clear=\"$source\" to re-index.\n\n";
+        continue;
+    }
+
+    try {
+        $pdf   = $parser->parseFile($filePath);
+        $pages = $pdf->getPages();
+    } catch (Exception $e) {
+        echo "  ERROR parsing PDF: " . $e->getMessage() . "\n\n";
+        continue;
+    }
+
+    echo "  Pages: " . count($pages) . "\n";
+
+    $totalChunks  = 0;
+    $totalTokens  = 0;
+    $pageBuffer   = [];  // accumulate pages into a rolling word buffer
+
+    $insertStmt = $pdo->prepare(
+        'INSERT INTO knowledge_chunks (source, author, page, chunk_index, chunk_text, embedding)
+         VALUES (?, ?, ?, ?, ?, ?)'
+    );
+
+    $chunkIndex  = 0;
+    $wordBuffer  = [];
+    $bufferPages = [];  // page numbers corresponding to words in buffer
+
+    foreach ($pages as $pageNum => $page) {
+        $pageText = cleanText($page->getText());
+        if (strlen($pageText) < 50) continue;  // skip blank/image-only pages
+
+        $words = explode(' ', $pageText);
+        foreach ($words as $word) {
+            $wordBuffer[]  = $word;
+            $bufferPages[] = $pageNum + 1;
+        }
+
+        // Flush when buffer reaches chunk size
+        while (count($wordBuffer) >= CHUNK_WORDS) {
+            $chunkWords  = array_slice($wordBuffer, 0, CHUNK_WORDS);
+            $chunkText   = implode(' ', $chunkWords);
+            $chunkPage   = $bufferPages[0];
+
+            if (strlen(trim($chunkText)) > 50) {
+                $embedding = getEmbedding($chunkText);
+                if ($embedding === null) {
+                    echo "  WARNING: embedding failed for chunk $chunkIndex — skipping.\n";
+                } else {
+                    $insertStmt->execute([
+                        $source,
+                        $author,
+                        $chunkPage,
+                        $chunkIndex,
+                        $chunkText,
+                        json_encode($embedding),
+                    ]);
+                    $chunkIndex++;
+                    $totalChunks++;
+                }
+            }
+
+            // Slide window with overlap
+            $step        = CHUNK_WORDS - OVERLAP_WORDS;
+            $wordBuffer  = array_slice($wordBuffer, $step);
+            $bufferPages = array_slice($bufferPages, $step);
+
+            if ($chunkIndex % 20 === 0 && $chunkIndex > 0) {
+                echo "  ...{$chunkIndex} chunks embedded\n";
+            }
+        }
+    }
+
+    // Flush remaining words as final chunk
+    if (count($wordBuffer) > 30) {
+        $chunkText = implode(' ', $wordBuffer);
+        $embedding = getEmbedding($chunkText);
+        if ($embedding !== null) {
+            $insertStmt->execute([
+                $source, $author, $bufferPages[0] ?? 0, $chunkIndex,
+                $chunkText, json_encode($embedding),
+            ]);
+            $chunkIndex++;
+            $totalChunks++;
+        }
+    }
+
+    echo "  Done: $totalChunks chunks stored.\n\n";
+}
+
+echo "Ingestion complete.\n";
+exit(0);
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+/**
+ * Normalise extracted PDF text: collapse whitespace, fix ligatures, etc.
+ */
+function cleanText(string $text): string
+{
+    // Common PDF ligature replacements
+    $ligatures = [
+        'fi' => 'fi', 'fl' => 'fl', 'ff' => 'ff',
+        'ffi' => 'ffi', 'ffl' => 'ffl', 'ſt' => 'st',
+    ];
+    $text = strtr($text, $ligatures);
+
+    // Collapse multiple spaces / newlines into single space
+    $text = preg_replace('/\s+/', ' ', $text);
+
+    // Remove non-printable characters except newlines
+    $text = preg_replace('/[^\x09\x0A\x0D\x20-\x7E\xA0-\xFF]/u', '', $text);
+
+    return trim($text);
+}
+
+/**
+ * Call Ollama's /api/embeddings and return float[] or null on failure.
+ */
+function getEmbedding(string $text): ?array
+{
+    $payload = json_encode([
+        'model'  => EMBED_MODEL,
+        'prompt' => $text,
+    ]);
+
+    $ch = curl_init(OLLAMA_EMBED_URL);
+    curl_setopt_array($ch, [
+        CURLOPT_POST           => true,
+        CURLOPT_POSTFIELDS     => $payload,
+        CURLOPT_HTTPHEADER     => ['Content-Type: application/json'],
+        CURLOPT_RETURNTRANSFER => true,
+        CURLOPT_TIMEOUT        => 30,
+        CURLOPT_CONNECTTIMEOUT => 5,
+    ]);
+
+    $response = curl_exec($ch);
+    $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
+    curl_close($ch);
+
+    if (!$response || $httpCode !== 200) {
+        return null;
+    }
+
+    $data = json_decode($response, true);
+    $embedding = $data['embedding'] ?? null;
+
+    if (!is_array($embedding) || count($embedding) === 0) {
+        return null;
+    }
+
+    return $embedding;
+}