Async Scraping

services/gateway/server.js

@@ -17,6 +17,10 @@ const UPLOAD_DIR = process.env.UPLOAD_DIR || '/uploads';
 const ALLOWED_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.mp4', '.mov', '.avi']);
 const MAX_FILE_SIZE = 100 * 1024 * 1024; // 100 MB
 
+// In-memory job state for async competitor scrapes.
+// Max 2 competitors, scrapes complete in seconds — no persistence needed.
+const activeScrapeJobs = new Map();
+
 fs.mkdirSync(UPLOAD_DIR, { recursive: true });
 
 app.register(multipart, { limits: { fileSize: MAX_FILE_SIZE } });
@@ -2302,14 +2306,32 @@ app.delete('/competitors/:id', async (request, reply) => {
   return { success: true };
 });
 
-// Scrape one competitor
+// Scrape one competitor — returns jobId immediately, runs in background
 app.post('/competitors/:id/scrape', async (request, reply) => {
-  try {
-    const result = await runCompetitorScrape(request.params.id);
-    return { success: result.ok, sources: result.sources, message: result.message };
-  } catch (err) {
-    return reply.code(500).send({ error: 'Scrape failed', detail: err.message });
-  }
+  const jobId = new ObjectId().toString();
+  activeScrapeJobs.set(jobId, { status: 'running', sources: 0, message: '' });
+
+  (async () => {
+    try {
+      const result = await runCompetitorScrape(request.params.id);
+      activeScrapeJobs.set(jobId, {
+        status: result.ok ? 'done' : 'failed',
+        sources: result.sources,
+        message: result.message,
+      });
+    } catch (err) {
+      activeScrapeJobs.set(jobId, { status: 'failed', sources: 0, message: err.message });
+    }
+  })();
+
+  return reply.code(202).send({ jobId });
+});
+
+// Poll scrape job status
+app.get('/competitors/:id/scrape-status/:jobId', async (request, reply) => {
+  const job = activeScrapeJobs.get(request.params.jobId);
+  if (!job) return reply.code(404).send({ error: 'Job not found or expired' });
+  return job;
 });
 
 // Scrape all competitors (called by scheduler)

ui/src/stores/competitors.ts

@@ -100,14 +100,32 @@ export const useCompetitorStore = defineStore('competitors', () => {
     }
   }
 
+  async function pollScrapeJob(competitorId: string, jobId: string): Promise<{ ok: boolean; sources: number; message: string }> {
+    return new Promise((resolve) => {
+      const check = async () => {
+        try {
+          const res = await axios.get(`/api/competitors/${competitorId}/scrape-status/${jobId}`)
+          const { status, sources, message } = res.data
+          if (status === 'done' || status === 'failed') {
+            resolve({ ok: status === 'done', sources: sources ?? 0, message: message || '' })
+          } else {
+            setTimeout(check, 2000)
+          }
+        } catch {
+          resolve({ ok: false, sources: 0, message: 'Status check failed' })
+        }
+      }
+      check()
+    })
+  }
+
   async function scrapeCompetitor(id: string): Promise<void> {
     scraping.value = { ...scraping.value, [id]: true }
     try {
       const res = await axios.post(`/api/competitors/${id}/scrape`)
-      scrapeResults.value = {
-        ...scrapeResults.value,
-        [id]: { sources: res.data.sources ?? 0, ok: res.data.success, message: res.data.message || '' },
-      }
+      const { jobId } = res.data
+      const result = await pollScrapeJob(id, jobId)
+      scrapeResults.value = { ...scrapeResults.value, [id]: result }
       await fetchCompetitors()
     } catch (err: any) {
       const msg = err.response?.data?.detail || err.response?.data?.error || err.message