feat: implement Meta crawler using chromedp and register it in worker pool

pkg/crawler/meta/meta_test.go

@@ -0,0 +1,56 @@
+package meta
+
+import (
+	"context"
+	"testing"
+	"time"
+)
+
+func TestMetaCrawler_EndToEnd(t *testing.T) {
+	if testing.Short() {
+		t.Skip("Skipping long-running Meta crawler network test in short mode")
+	}
+
+	c := NewMetaCrawler()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
+	defer cancel()
+
+	testURL := "https://www.metacareers.com/jobsearch/?offices[0]=Seattle%2C%20WA&roles[0]=Full%20time%20employment"
+
+	t.Logf("Discovering jobs from: %s", testURL)
+	discovered, err := c.DiscoverJobs(ctx, testURL)
+	if err != nil {
+		t.Fatalf("Failed to discover jobs: %v", err)
+	}
+
+	if len(discovered) == 0 {
+		t.Fatalf("Expected to find at least 1 job, found 0")
+	}
+
+	t.Logf("Successfully discovered %d jobs", len(discovered))
+
+	// Fetch details for the first discovered job
+	firstJob := discovered[0]
+	t.Logf("Fetching details for job: %s", firstJob.ApplyURL)
+
+	detail, err := c.FetchJobDetail(ctx, firstJob.ApplyURL)
+	if err != nil {
+		t.Fatalf("Failed to fetch job details: %v", err)
+	}
+
+	if detail.Title == "" || detail.Title == "Unknown Meta Title" {
+		t.Errorf("Expected a valid job title, got %s", detail.Title)
+	}
+
+	if len(detail.RawContent) < 100 {
+		t.Errorf("Expected RawContent to contain the job description, but it was too short (%d bytes)", len(detail.RawContent))
+	}
+
+	if detail.CanonicalID == "unknown" || detail.CanonicalID == "" {
+		t.Errorf("Failed to extract canonical ID from %s", firstJob.ApplyURL)
+	}
+
+	t.Logf("Extracted Title: %s", detail.Title)
+	t.Logf("Extracted CanonicalID: %s", detail.CanonicalID)
+	t.Logf("Raw Content Length: %d", len(detail.RawContent))
+}