feat: implement Meta crawler using chromedp and register it in worker pool

pkg/crawler/meta/meta.go

@@ -0,0 +1,142 @@
+package meta
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/chromedp/cdproto/cdp"
+	"github.com/chromedp/chromedp"
+	"github.com/hireme/hireme/internal/crawler"
+	"github.com/hireme/hireme/internal/logger"
+)
+
+type MetaCrawler struct{}
+
+func NewMetaCrawler() *MetaCrawler {
+	return &MetaCrawler{}
+}
+
+func (c *MetaCrawler) CanHandle(platform string) bool {
+	return strings.ToLower(platform) == "meta" || strings.ToLower(platform) == "facebook"
+}
+
+func (c *MetaCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
+	opts := append(chromedp.DefaultExecAllocatorOptions[:],
+		chromedp.Flag("headless", true),
+		chromedp.Flag("no-sandbox", true),
+		chromedp.Flag("disable-gpu", true),
+		chromedp.Flag("disable-dev-shm-usage", true),
+	)
+
+	allocCtx, cancelAlloc := chromedp.NewExecAllocator(ctx, opts...)
+	defer cancelAlloc()
+
+	chromeCtx, cancelCtx := chromedp.NewContext(allocCtx)
+	defer cancelCtx()
+
+	chromeCtx, cancelTimeout := context.WithTimeout(chromeCtx, 30*time.Second)
+	defer cancelTimeout()
+
+	var nodes []*cdp.Node
+	logger.Log.Infow("Navigating to Meta careers search", "url", targetURL)
+
+	err := chromedp.Run(chromeCtx,
+		chromedp.Navigate(targetURL),
+		chromedp.Sleep(5*time.Second), // Wait for React and GraphQL to load
+		chromedp.Nodes(`a`, &nodes, chromedp.ByQueryAll),
+	)
+
+	if err != nil {
+		return nil, fmt.Errorf("chromedp navigation failed: %w", err)
+	}
+
+	var results []crawler.DiscoveredJob
+	seen := make(map[string]bool)
+
+	for _, n := range nodes {
+		href := n.AttributeValue("href")
+		// Meta uses /v2/jobs/..., /jobs/..., or /profile/job_details/...
+		if (strings.Contains(href, "/v2/jobs/") || strings.Contains(href, "/jobs/") || strings.Contains(href, "/profile/job_details/")) && !strings.Contains(href, "jobsearch") {
+			fullURL := href
+			if strings.HasPrefix(href, "/") {
+				fullURL = "https://www.metacareers.com" + href
+			}
+			
+			// Extract Canonical ID from URL
+			parts := strings.Split(strings.Trim(href, "/"), "/")
+			canonicalID := "unknown"
+			for i, p := range parts {
+				if (p == "jobs" || p == "job_details") && i+1 < len(parts) {
+					canonicalID = parts[i+1]
+					break
+				}
+			}
+
+			if !seen[fullURL] {
+				seen[fullURL] = true
+				results = append(results, crawler.DiscoveredJob{
+					ApplyURL:    fullURL,
+					CanonicalID: "meta_" + canonicalID,
+				})
+			}
+		}
+	}
+
+	logger.Log.Infow("Meta discovery finished", "found_count", len(results))
+	return results, nil
+}
+
+func (c *MetaCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
+	opts := append(chromedp.DefaultExecAllocatorOptions[:],
+		chromedp.Flag("headless", true),
+		chromedp.Flag("no-sandbox", true),
+		chromedp.Flag("disable-gpu", true),
+		chromedp.Flag("disable-dev-shm-usage", true),
+	)
+
+	allocCtx, cancelAlloc := chromedp.NewExecAllocator(ctx, opts...)
+	defer cancelAlloc()
+
+	chromeCtx, cancelCtx := chromedp.NewContext(allocCtx)
+	defer cancelCtx()
+
+	chromeCtx, cancelTimeout := context.WithTimeout(chromeCtx, 30*time.Second)
+	defer cancelTimeout()
+
+	logger.Log.Infow("Fetching Meta job detail", "url", applyURL)
+
+	var titleText string
+	var bodyText string
+
+	err := chromedp.Run(chromeCtx,
+		chromedp.Navigate(applyURL),
+		chromedp.Sleep(5*time.Second),
+		chromedp.Evaluate(`document.querySelector("h1") ? document.querySelector("h1").innerText : "Unknown Meta Title"`, &titleText),
+		chromedp.Evaluate(`document.body.innerText`, &bodyText),
+	)
+
+	if err != nil {
+		return nil, fmt.Errorf("chromedp navigation failed: %w", err)
+	}
+
+	// Extract ID
+	parts := strings.Split(strings.Trim(applyURL, "/"), "/")
+	canonicalID := "unknown"
+	for i, p := range parts {
+		if (p == "jobs" || p == "job_details") && i+1 < len(parts) {
+			canonicalID = parts[i+1]
+			break
+		}
+	}
+
+	return &crawler.ExtractedJob{
+		CanonicalID: "meta_" + canonicalID,
+		Title:       titleText,
+		Company:     "Meta",
+		Location:    "Check description", // Location extraction is complex via text, leave it or parse later
+		ApplyURL:    applyURL,
+		RawContent:  bodyText,
+	}, nil
+}

pkg/crawler/meta/meta_test.go

@@ -0,0 +1,56 @@
+package meta
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestMetaCrawler_EndToEnd(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping long-running Meta crawler network test in short mode")
+	}
+
+	c := NewMetaCrawler()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+
+	testURL := "https://www.metacareers.com/jobsearch/?offices[0]=Seattle%2C%20WA&roles[0]=Full%20time%20employment"
+
+	t.Logf("Discovering jobs from: %s", testURL)
+	discovered, err := c.DiscoverJobs(ctx, testURL)
+	if err != nil {
+		t.Fatalf("Failed to discover jobs: %v", err)
+	}
+
+	if len(discovered) == 0 {
+		t.Fatalf("Expected to find at least 1 job, found 0")
+	}
+
+	t.Logf("Successfully discovered %d jobs", len(discovered))
+
+	// Fetch details for the first discovered job
+	firstJob := discovered[0]
+	t.Logf("Fetching details for job: %s", firstJob.ApplyURL)
+
+	detail, err := c.FetchJobDetail(ctx, firstJob.ApplyURL)
+	if err != nil {
+		t.Fatalf("Failed to fetch job details: %v", err)
+	}
+
+	if detail.Title == "" || detail.Title == "Unknown Meta Title" {
+		t.Errorf("Expected a valid job title, got %s", detail.Title)
+	}
+
+	if len(detail.RawContent) < 100 {
+		t.Errorf("Expected RawContent to contain the job description, but it was too short (%d bytes)", len(detail.RawContent))
+	}
+
+	if detail.CanonicalID == "unknown" || detail.CanonicalID == "" {
+		t.Errorf("Failed to extract canonical ID from %s", firstJob.ApplyURL)
+	}
+
+	t.Logf("Extracted Title: %s", detail.Title)
+	t.Logf("Extracted CanonicalID: %s", detail.CanonicalID)
+	t.Logf("Raw Content Length: %d", len(detail.RawContent))
+}