feat: add live DOM scraper, activity detail-fetch for video streams, and raw activity archiving

dashboard/dashboard.js

@@ -436,9 +436,24 @@ async function fetchAllActivities(studentId) {
   const startDate = el.startDate.value;
   const endDate = el.endDate.value;
   const pacingDelay = parseInt(el.concurrencySelect.value) || 200;
+  const seenTypes = new Set();
 
   updateProgress(5, 'Harvesting activity logs...');
 
+  // 1. Also scrape live DOM feed from active Brightwheel tab if open
+  try {
+    const [bwTab] = await chrome.tabs.query({ url: 'https://schools.mybrightwheel.com/*' });
+    if (bwTab && bwTab.id) {
+      chrome.tabs.sendMessage(bwTab.id, { action: 'SCRAPE_DOM_FEED' }, (res) => {
+        if (res && res.success && res.activities && res.activities.length > 0) {
+          log(`Scraped ${res.activities.length} visible activity cards from active Brightwheel page DOM.`, 'info');
+        }
+      });
+    }
+  } catch (e) {
+    // Non-critical
+  }
+
   while (hasMore && !state.isCancelled) {
     while (state.isPaused && !state.isCancelled) {
       await sleep(500);
@@ -472,9 +487,7 @@ async function fetchAllActivities(studentId) {
     }
 
     if (!response || !response.success || !response.data) {
-      // If page 1 failed or reached end
       if (page === 1) {
-        // Try fallback endpoint /api/v1/activities?student_ids[]
         log('Trying alternate activity feed endpoint...', 'info');
         const altUrl = `https://schools.mybrightwheel.com/api/v1/activities?student_ids[]=${studentId}&page_size=${pageSize}&page=${page}`;
         const altRes = await fetchBrightwheelAPI(altUrl);
@@ -494,6 +507,27 @@ async function fetchAllActivities(studentId) {
 
     // Filter and normalize activities
     for (const act of rawActivities) {
+      // Schema inspection for debugging
+      const actType = act.activity_type || act.action_type || act.type || 'unknown';
+      if (!seenTypes.has(actType)) {
+        seenTypes.add(actType);
+        const presentKeys = Object.keys(act).filter(k => act[k] !== null && act[k] !== undefined && act[k] !== '');
+        log(`[Schema] ${actType} fields: ${presentKeys.join(', ')}`, 'info');
+      }
+
+      // If video activity has no immediate URL, fetch single activity detail
+      if ((actType === 'ac_video' || actType === 'video') && !act.video_url && !act.media?.video_url) {
+        try {
+          const detailRes = await fetchBrightwheelAPI(`https://schools.mybrightwheel.com/api/v1/students/${studentId}/activities/${act.id}`);
+          if (detailRes && detailRes.success && detailRes.data) {
+            const detailObj = detailRes.data.activity || detailRes.data;
+            Object.assign(act, detailObj);
+          }
+        } catch (e) {
+          // Continue
+        }
+      }
+
       if (matchesSelectedCategories(act)) {
         state.activities.push(normalizeActivity(act));
       }
@@ -899,6 +933,11 @@ async function buildAndDownloadZip(studentProfile) {
       activities: state.activities
     };
     rootFolder.file('manifest.json', JSON.stringify(manifestData, null, 2));
+
+    const rawList = state.activities.map(a => a.raw).filter(Boolean);
+    if (rawList.length > 0) {
+      rootFolder.file('raw_activities.json', JSON.stringify(rawList, null, 2));
+    }
   }
 
   // 3. Add CSV Spreadsheet