feat: add live DOM scraper, activity detail-fetch for video streams, and raw activity archiving

dashboard/dashboard.js

@@ -436,9 +436,24 @@ async function fetchAllActivities(studentId) {
   const startDate = el.startDate.value;
   const endDate = el.endDate.value;
   const pacingDelay = parseInt(el.concurrencySelect.value) || 200;
+  const seenTypes = new Set();
 
   updateProgress(5, 'Harvesting activity logs...');
 
+  // 1. Also scrape live DOM feed from active Brightwheel tab if open
+  try {
+    const [bwTab] = await chrome.tabs.query({ url: 'https://schools.mybrightwheel.com/*' });
+    if (bwTab && bwTab.id) {
+      chrome.tabs.sendMessage(bwTab.id, { action: 'SCRAPE_DOM_FEED' }, (res) => {
+        if (res && res.success && res.activities && res.activities.length > 0) {
+          log(`Scraped ${res.activities.length} visible activity cards from active Brightwheel page DOM.`, 'info');
+        }
+      });
+    }
+  } catch (e) {
+    // Non-critical
+  }
+
   while (hasMore && !state.isCancelled) {
     while (state.isPaused && !state.isCancelled) {
       await sleep(500);
@@ -472,9 +487,7 @@ async function fetchAllActivities(studentId) {
     }
 
     if (!response || !response.success || !response.data) {
-      // If page 1 failed or reached end
       if (page === 1) {
-        // Try fallback endpoint /api/v1/activities?student_ids[]
         log('Trying alternate activity feed endpoint...', 'info');
         const altUrl = `https://schools.mybrightwheel.com/api/v1/activities?student_ids[]=${studentId}&page_size=${pageSize}&page=${page}`;
         const altRes = await fetchBrightwheelAPI(altUrl);
@@ -494,6 +507,27 @@ async function fetchAllActivities(studentId) {
 
     // Filter and normalize activities
     for (const act of rawActivities) {
+      // Schema inspection for debugging
+      const actType = act.activity_type || act.action_type || act.type || 'unknown';
+      if (!seenTypes.has(actType)) {
+        seenTypes.add(actType);
+        const presentKeys = Object.keys(act).filter(k => act[k] !== null && act[k] !== undefined && act[k] !== '');
+        log(`[Schema] ${actType} fields: ${presentKeys.join(', ')}`, 'info');
+      }
+
+      // If video activity has no immediate URL, fetch single activity detail
+      if ((actType === 'ac_video' || actType === 'video') && !act.video_url && !act.media?.video_url) {
+        try {
+          const detailRes = await fetchBrightwheelAPI(`https://schools.mybrightwheel.com/api/v1/students/${studentId}/activities/${act.id}`);
+          if (detailRes && detailRes.success && detailRes.data) {
+            const detailObj = detailRes.data.activity || detailRes.data;
+            Object.assign(act, detailObj);
+          }
+        } catch (e) {
+          // Continue
+        }
+      }
+
       if (matchesSelectedCategories(act)) {
         state.activities.push(normalizeActivity(act));
       }
@@ -899,6 +933,11 @@ async function buildAndDownloadZip(studentProfile) {
       activities: state.activities
     };
     rootFolder.file('manifest.json', JSON.stringify(manifestData, null, 2));
+
+    const rawList = state.activities.map(a => a.raw).filter(Boolean);
+    if (rawList.length > 0) {
+      rootFolder.file('raw_activities.json', JSON.stringify(rawList, null, 2));
+    }
   }
 
   // 3. Add CSV Spreadsheet

scripts/content_script.js

@@ -40,8 +40,17 @@
       return true;
     }
 
+    if (message.action === 'SCRAPE_DOM_FEED') {
+      try {
+        const domActivities = scrapeDomFeedCards();
+        sendResponse({ success: true, activities: domActivities, htmlSample: document.body.innerHTML.substring(0, 5000) });
+      } catch (err) {
+        sendResponse({ success: false, error: err.message });
+      }
+      return true;
+    }
+
     if (message.action === 'FETCH_WITH_CREDENTIALS') {
-      // Execute fetch inside page context with credentials
       fetch(message.url, {
         method: message.method || 'GET',
         headers: {
@@ -65,6 +74,53 @@
     }
   });
 
+  // DOM Scraper for Brightwheel Web Feed
+  function scrapeDomFeedCards() {
+    const results = [];
+    // Broad selector covering common Brightwheel feed cards
+    const cardSelectors = [
+      '[data-testid*="activity"]',
+      '[data-testid*="feed"]',
+      '.activity-item',
+      '.feed-item',
+      '.timeline-item',
+      'article',
+      '[class*="ActivityCard"]',
+      '[class*="feed-card"]',
+      '[class*="FeedItem"]',
+      '[class*="TimelineItem"]'
+    ];
+
+    let cards = document.querySelectorAll(cardSelectors.join(','));
+    if (!cards || cards.length === 0) {
+      // Fallback: look for cards by common structure
+      cards = document.querySelectorAll('div[class*="card"], div[class*="Card"]');
+    }
+
+    cards.forEach((card, index) => {
+      const text = card.innerText || '';
+      if (!text.trim()) return;
+
+      const cardId = card.getAttribute('data-id') || card.getAttribute('data-activity-id') || card.id || `dom_${index}`;
+      const img = card.querySelector('img[src*="cdn"], img[src*="brightwheel"], img[src*="cloudfront"], img[src*="s3"]');
+      const video = card.querySelector('video, source, a[href*=".mp4"], a[href*="video"]');
+
+      const videoUrl = video?.src || video?.getAttribute('src') || video?.getAttribute('href') || null;
+      const imageUrl = img?.src || null;
+
+      results.push({
+        dom_id: cardId,
+        raw_text: text,
+        image_url: imageUrl,
+        video_url: videoUrl,
+        class_name: card.className,
+        html_snippet: card.outerHTML.substring(0, 800)
+      });
+    });
+
+    return results;
+  }
+
   // Inject a discreet floating button on Brightwheel page
   function injectExporterButton() {
     if (document.getElementById('bw-exporter-floating-btn')) return;