feat: implement Meta crawler using chromedp and register it in worker pool

2026-08-22 01:06:47 -07:00
parent f8bfc4b5e3
commit af1ee1c9b3
6 changed files with 251 additions and 2 deletions
@@ -0,0 +1,142 @@
+package meta
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/chromedp/cdproto/cdp"
+	"github.com/chromedp/chromedp"
+	"github.com/hireme/hireme/internal/crawler"
+	"github.com/hireme/hireme/internal/logger"
+)
+
+type MetaCrawler struct{}
+
+func NewMetaCrawler() *MetaCrawler {
+	return &MetaCrawler{}
+}
+
+func (c *MetaCrawler) CanHandle(platform string) bool {
+	return strings.ToLower(platform) == "meta" || strings.ToLower(platform) == "facebook"
+}
+
+func (c *MetaCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
+	opts := append(chromedp.DefaultExecAllocatorOptions[:],
+		chromedp.Flag("headless", true),
+		chromedp.Flag("no-sandbox", true),
+		chromedp.Flag("disable-gpu", true),
+		chromedp.Flag("disable-dev-shm-usage", true),
+	)
+
+	allocCtx, cancelAlloc := chromedp.NewExecAllocator(ctx, opts...)
+	defer cancelAlloc()
+
+	chromeCtx, cancelCtx := chromedp.NewContext(allocCtx)
+	defer cancelCtx()
+
+	chromeCtx, cancelTimeout := context.WithTimeout(chromeCtx, 30*time.Second)
+	defer cancelTimeout()
+
+	var nodes []*cdp.Node
+	logger.Log.Infow("Navigating to Meta careers search", "url", targetURL)
+
+	err := chromedp.Run(chromeCtx,
+		chromedp.Navigate(targetURL),
+		chromedp.Sleep(5*time.Second), // Wait for React and GraphQL to load
+		chromedp.Nodes(`a`, &nodes, chromedp.ByQueryAll),
+	)
+
+	if err != nil {
+		return nil, fmt.Errorf("chromedp navigation failed: %w", err)
+	}
+
+	var results []crawler.DiscoveredJob
+	seen := make(map[string]bool)
+
+	for _, n := range nodes {
+		href := n.AttributeValue("href")
+		// Meta uses /v2/jobs/..., /jobs/..., or /profile/job_details/...
+		if (strings.Contains(href, "/v2/jobs/") || strings.Contains(href, "/jobs/") || strings.Contains(href, "/profile/job_details/")) && !strings.Contains(href, "jobsearch") {
+			fullURL := href
+			if strings.HasPrefix(href, "/") {
+				fullURL = "https://www.metacareers.com" + href
+			}
+			
+			// Extract Canonical ID from URL
+			parts := strings.Split(strings.Trim(href, "/"), "/")
+			canonicalID := "unknown"
+			for i, p := range parts {
+				if (p == "jobs" || p == "job_details") && i+1 < len(parts) {
+					canonicalID = parts[i+1]
+					break
+				}
+			}
+
+			if !seen[fullURL] {
+				seen[fullURL] = true
+				results = append(results, crawler.DiscoveredJob{
+					ApplyURL:    fullURL,
+					CanonicalID: "meta_" + canonicalID,
+				})
+			}
+		}
+	}
+
+	logger.Log.Infow("Meta discovery finished", "found_count", len(results))
+	return results, nil
+}
+
+func (c *MetaCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
+	opts := append(chromedp.DefaultExecAllocatorOptions[:],
+		chromedp.Flag("headless", true),
+		chromedp.Flag("no-sandbox", true),
+		chromedp.Flag("disable-gpu", true),
+		chromedp.Flag("disable-dev-shm-usage", true),
+	)
+
+	allocCtx, cancelAlloc := chromedp.NewExecAllocator(ctx, opts...)
+	defer cancelAlloc()
+
+	chromeCtx, cancelCtx := chromedp.NewContext(allocCtx)
+	defer cancelCtx()
+
+	chromeCtx, cancelTimeout := context.WithTimeout(chromeCtx, 30*time.Second)
+	defer cancelTimeout()
+
+	logger.Log.Infow("Fetching Meta job detail", "url", applyURL)
+
+	var titleText string
+	var bodyText string
+
+	err := chromedp.Run(chromeCtx,
+		chromedp.Navigate(applyURL),
+		chromedp.Sleep(5*time.Second),
+		chromedp.Evaluate(`document.querySelector("h1") ? document.querySelector("h1").innerText : "Unknown Meta Title"`, &titleText),
+		chromedp.Evaluate(`document.body.innerText`, &bodyText),
+	)
+
+	if err != nil {
+		return nil, fmt.Errorf("chromedp navigation failed: %w", err)
+	}
+
+	// Extract ID
+	parts := strings.Split(strings.Trim(applyURL, "/"), "/")
+	canonicalID := "unknown"
+	for i, p := range parts {
+		if (p == "jobs" || p == "job_details") && i+1 < len(parts) {
+			canonicalID = parts[i+1]
+			break
+		}
+	}
+
+	return &crawler.ExtractedJob{
+		CanonicalID: "meta_" + canonicalID,
+		Title:       titleText,
+		Company:     "Meta",
+		Location:    "Check description", // Location extraction is complex via text, leave it or parse later
+		ApplyURL:    applyURL,
+		RawContent:  bodyText,
+	}, nil
+}