feat: add Apple jobs crawler and implement duplicate-prevention logic for target loading

pkg/crawler/apple/apple_test.go

@@ -0,0 +1,66 @@
+package apple
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestAppleCrawler_EndToEnd(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping end-to-end network test in short mode")
+	}
+
+	crawler := NewAppleCrawler()
+
+	// Use the default test URL
+	targetURL := "https://jobs.apple.com/en-us/search?location=seattle-SEA&key=Software%2520Engineer"
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// 1. Test DiscoverJobs
+	t.Logf("Discovering jobs from: %s", targetURL)
+	jobs, err := crawler.DiscoverJobs(ctx, targetURL)
+	if err != nil {
+		t.Fatalf("DiscoverJobs failed: %v", err)
+	}
+
+	if len(jobs) == 0 {
+		t.Fatalf("DiscoverJobs returned 0 jobs for URL: %s", targetURL)
+	}
+	t.Logf("Successfully discovered %d jobs", len(jobs))
+
+	// 2. Pick the first job and test FetchJobDetail
+	testJob := jobs[0]
+	t.Logf("Fetching details for job: %s", testJob.ApplyURL)
+
+	details, err := crawler.FetchJobDetail(ctx, testJob.ApplyURL)
+	if err != nil {
+		t.Fatalf("FetchJobDetail failed: %v", err)
+	}
+
+	if details == nil {
+		t.Fatal("FetchJobDetail returned nil ExtractedJob")
+	}
+
+	// 3. Verify extracted details
+	t.Logf("Extracted Title: %s", details.Title)
+	t.Logf("Extracted Company: %s", details.Company)
+	t.Logf("Extracted Location: %s", details.Location)
+	t.Logf("Raw Content Length: %d", len(details.RawContent))
+
+	if details.Title == "" || details.Title == "Unknown Apple Position" {
+		t.Errorf("Expected valid job title, got %q", details.Title)
+	}
+	if details.Company != "Apple" {
+		t.Errorf("Expected company 'Apple', got %q", details.Company)
+	}
+	if !strings.HasPrefix(details.CanonicalID, "apple:") {
+		t.Errorf("Expected CanonicalID to start with 'apple:', got %q", details.CanonicalID)
+	}
+	if len(details.RawContent) < 100 {
+		t.Errorf("Expected RawContent to be substantive, got length %d", len(details.RawContent))
+	}
+}