feat: implement multiple new job crawlers and add corresponding test suites

2026-08-25 00:12:52 -07:00
parent 0de312edbd
commit 55d4f4a0d9
26 changed files with 4045 additions and 15 deletions
@@ -0,0 +1,326 @@
+// Package crawlertest provides a standardized integration test harness for
+// validating that each SiteCrawler still functions against its real target URL.
+//
+// Each crawler package should have a TestXxx_LiveIntegration test that calls
+// crawlertest.RunLiveIntegration(). This tests three real-world failure modes:
+//
+//  1. DISCOVER STAGE - Can we still hit the search URL and get job listings?
+//     Detects: search API endpoint moved, query param format changed, auth required.
+//
+//  2. FETCH STAGE - Can we still fetch a single job detail page?
+//     Detects: detail URL pattern changed, page returns 403/404, rate limiting.
+//
+//  3. PARSE STAGE - Can we still extract structured data from the response?
+//     Detects: HTML/JSON schema changed, title/company/content extraction broken.
+package crawlertest
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/hireme/hireme/internal/crawler"
+)
+
+// LiveTestConfig defines what a live integration test expects from a crawler.
+type LiveTestConfig struct {
+	// PlatformName is the human-readable platform name (e.g., "Amazon", "Google").
+	PlatformName string
+
+	// Crawler is the SiteCrawler instance to test.
+	Crawler crawler.SiteCrawler
+
+	// TargetURL is the real search/listing URL to test discovery against.
+	TargetURL string
+
+	// ExpectedCanonicalPrefix is the prefix each CanonicalID should start with
+	// (e.g., "amazon:", "google:"). Empty string skips this check.
+	ExpectedCanonicalPrefix string
+
+	// ExpectedCompany is the expected Company field in FetchJobDetail results.
+	// Empty string skips this check.
+	ExpectedCompany string
+
+	// MinDiscoverCount is the minimum number of jobs DiscoverJobs should return.
+	// Defaults to 1 if not set.
+	MinDiscoverCount int
+
+	// MinContentLength is the minimum byte length of RawContent in FetchJobDetail.
+	// Defaults to 100 if not set.
+	MinContentLength int
+
+	// Timeout is the context timeout for the entire test. Defaults to 30s.
+	Timeout time.Duration
+}
+
+// RunLiveIntegration runs a standardized 3-stage integration test against real
+// targets. It gracefully skips on network errors (offline/sandbox) but FAILS
+// loudly when the crawler reaches the site but can't parse data correctly.
+//
+// This is the test that tells you "the site changed and our crawler is broken".
+//
+// Usage in each crawler package's _test.go:
+//
+//	func TestAmazonCrawler_LiveIntegration(t *testing.T) {
+//	    crawlertest.RunLiveIntegration(t, crawlertest.LiveTestConfig{
+//	        PlatformName:            "Amazon",
+//	        Crawler:                 NewAmazonCrawler(),
+//	        TargetURL:               "https://www.amazon.jobs/en/search.json?...",
+//	        ExpectedCanonicalPrefix: "amazon:",
+//	        ExpectedCompany:         "Amazon",
+//	    })
+//	}
+func RunLiveIntegration(t *testing.T, cfg LiveTestConfig) {
+	t.Helper()
+
+	// Apply defaults
+	if cfg.MinDiscoverCount <= 0 {
+		cfg.MinDiscoverCount = 1
+	}
+	if cfg.MinContentLength <= 0 {
+		cfg.MinContentLength = 100
+	}
+	if cfg.Timeout <= 0 {
+		cfg.Timeout = 30 * time.Second
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout)
+	defer cancel()
+
+	// =========================================================================
+	// STAGE 1: DISCOVER — Can we reach the search URL and get job listings?
+	// =========================================================================
+	t.Run("Stage1_Discover", func(t *testing.T) {
+		t.Logf("[%s] Discovering jobs from: %s", cfg.PlatformName, cfg.TargetURL)
+
+		discovered, err := cfg.Crawler.DiscoverJobs(ctx, cfg.TargetURL)
+		if err != nil {
+			if isNetworkError(err) {
+				t.Skipf("SKIP (offline): %v", err)
+			}
+			t.Fatalf("❌ DISCOVER FAILED: %v\n"+
+				"  → The search URL may have changed format or the API endpoint moved.\n"+
+				"  → Target URL: %s", err, cfg.TargetURL)
+		}
+
+		if len(discovered) == 0 {
+			t.Fatalf("❌ DISCOVER RETURNED 0 JOBS\n"+
+				"  → The search results page format may have changed.\n"+
+				"  → Check if the target URL still returns results in a browser.\n"+
+				"  → Target URL: %s", cfg.TargetURL)
+		}
+
+		if len(discovered) < cfg.MinDiscoverCount {
+			t.Errorf("⚠️  DISCOVER LOW COUNT: got %d jobs, expected at least %d\n"+
+				"  → The search API may be returning fewer results than expected.",
+				len(discovered), cfg.MinDiscoverCount)
+		}
+
+		t.Logf("✅ Discovered %d jobs", len(discovered))
+
+		// Validate first result structure
+		first := discovered[0]
+		if first.ApplyURL == "" {
+			t.Errorf("❌ First discovered job has empty ApplyURL")
+		}
+		if first.CanonicalID == "" {
+			t.Errorf("❌ First discovered job has empty CanonicalID")
+		}
+		if cfg.ExpectedCanonicalPrefix != "" && !strings.HasPrefix(first.CanonicalID, cfg.ExpectedCanonicalPrefix) {
+			t.Errorf("❌ CANONICAL ID FORMAT CHANGED: got %q, expected prefix %q\n"+
+				"  → The job ID extraction pattern may need updating.",
+				first.CanonicalID, cfg.ExpectedCanonicalPrefix)
+		}
+
+		t.Logf("   First job: ID=%s URL=%s", first.CanonicalID, first.ApplyURL)
+	})
+
+	// Re-discover for stage 2 (subtests get their own scope)
+	discovered, err := cfg.Crawler.DiscoverJobs(ctx, cfg.TargetURL)
+	if err != nil || len(discovered) == 0 {
+		// Stage 1 already reported the failure
+		return
+	}
+
+	// =========================================================================
+	// STAGE 2: FETCH — Can we fetch a job detail page?
+	// =========================================================================
+	firstJob := discovered[0]
+	var detail *crawler.ExtractedJob
+
+	t.Run("Stage2_Fetch", func(t *testing.T) {
+		t.Logf("[%s] Fetching job detail: %s", cfg.PlatformName, firstJob.ApplyURL)
+
+		var fetchErr error
+		detail, fetchErr = cfg.Crawler.FetchJobDetail(ctx, firstJob.ApplyURL)
+		if fetchErr != nil {
+			if isNetworkError(fetchErr) {
+				t.Skipf("SKIP (offline): %v", fetchErr)
+			}
+			t.Fatalf("❌ FETCH FAILED: %v\n"+
+				"  → The job detail URL pattern may have changed.\n"+
+				"  → The page may now require JavaScript rendering.\n"+
+				"  → Job URL: %s", fetchErr, firstJob.ApplyURL)
+		}
+
+		if detail == nil {
+			t.Fatalf("❌ FETCH RETURNED nil ExtractedJob")
+		}
+
+		t.Logf("✅ Fetched job detail successfully")
+	})
+
+	if detail == nil {
+		return
+	}
+
+	// =========================================================================
+	// STAGE 3: PARSE — Can we extract structured data from the response?
+	// =========================================================================
+	t.Run("Stage3_Parse", func(t *testing.T) {
+		t.Logf("[%s] Validating parsed fields", cfg.PlatformName)
+		failures := 0
+
+		// Title
+		if strings.TrimSpace(detail.Title) == "" {
+			t.Errorf("❌ PARSE: Title is empty\n"+
+				"  → The title extraction selector/pattern may have changed.\n"+
+				"  → Job URL: %s", firstJob.ApplyURL)
+			failures++
+		} else {
+			t.Logf("   Title:    %s", detail.Title)
+		}
+
+		// Company
+		if strings.TrimSpace(detail.Company) == "" {
+			t.Errorf("❌ PARSE: Company is empty\n"+
+				"  → The company extraction logic may have changed.")
+			failures++
+		} else {
+			t.Logf("   Company:  %s", detail.Company)
+		}
+
+		if cfg.ExpectedCompany != "" && !strings.Contains(detail.Company, cfg.ExpectedCompany) {
+			t.Errorf("⚠️  PARSE: Company mismatch: got %q, expected to contain %q\n"+
+				"  → The company name format may have changed.",
+				detail.Company, cfg.ExpectedCompany)
+		}
+
+		// Location
+		if strings.TrimSpace(detail.Location) == "" {
+			t.Errorf("⚠️  PARSE: Location is empty\n"+
+				"  → Location extraction may have changed.")
+		} else {
+			t.Logf("   Location: %s", detail.Location)
+		}
+
+		// CanonicalID
+		if detail.CanonicalID == "" {
+			t.Errorf("❌ PARSE: CanonicalID is empty")
+			failures++
+		}
+
+		// ApplyURL
+		if detail.ApplyURL == "" {
+			t.Errorf("❌ PARSE: ApplyURL is empty")
+			failures++
+		}
+
+		// RawContent (the critical one for downstream LLM analysis)
+		contentLen := len(detail.RawContent)
+		if contentLen == 0 {
+			t.Errorf("❌ PARSE: RawContent is EMPTY\n"+
+				"  → The content extraction is completely broken.\n"+
+				"  → The page structure or API response schema has changed.\n"+
+				"  → Job URL: %s", firstJob.ApplyURL)
+			failures++
+		} else if contentLen < cfg.MinContentLength {
+			t.Errorf("⚠️  PARSE: RawContent is too short (%d bytes, min %d)\n"+
+				"  → The content extraction may be partially broken.\n"+
+				"  → Job URL: %s", contentLen, cfg.MinContentLength, firstJob.ApplyURL)
+		} else {
+			t.Logf("   Content:  %d bytes", contentLen)
+		}
+
+		if failures == 0 {
+			t.Logf("✅ All parse checks passed")
+		} else {
+			t.Logf("❌ %d parse check(s) failed — crawler needs fixing", failures)
+		}
+	})
+}
+
+// isNetworkError checks if an error is a network/DNS/connectivity issue
+// (as opposed to a parsing or HTTP status error from a reachable server).
+func isNetworkError(err error) bool {
+	if err == nil {
+		return false
+	}
+	msg := err.Error()
+	networkPatterns := []string{
+		"no such host",
+		"dial tcp",
+		"connection refused",
+		"network is unreachable",
+		"i/o timeout",
+		"TLS handshake timeout",
+		"operation not permitted",
+		"connect: operation not permitted",
+	}
+	for _, pattern := range networkPatterns {
+		if strings.Contains(msg, pattern) {
+			return true
+		}
+	}
+	return false
+}
+
+// FormatHealthReport generates a summary table from multiple test results.
+// This can be called from a top-level aggregation test.
+func FormatHealthReport(results []HealthResult) string {
+	var sb strings.Builder
+	sb.WriteString(fmt.Sprintf("%-15s | %-10s | %-10s | %-10s | %s\n",
+		"Platform", "Discover", "Fetch", "Content", "Status"))
+	sb.WriteString(strings.Repeat("-", 70) + "\n")
+	for _, r := range results {
+		discoverStr := fmt.Sprintf("%d jobs", r.DiscoverCount)
+		fetchStr := "✓"
+		contentStr := fmt.Sprintf("%d B", r.ContentLength)
+		status := "✅ HEALTHY"
+
+		if r.Skipped {
+			status = "⏭ SKIPPED"
+			discoverStr = "-"
+			fetchStr = "-"
+			contentStr = "-"
+		} else if r.Error != nil {
+			status = "❌ BROKEN"
+			if r.DiscoverCount == 0 {
+				discoverStr = "FAIL"
+				fetchStr = "-"
+				contentStr = "-"
+			} else if r.ContentLength == 0 {
+				fetchStr = "FAIL"
+				contentStr = "-"
+			}
+		} else if r.DiscoverCount == 0 {
+			status = "⚠️  DEGRADED"
+			discoverStr = "0 ⚠️"
+		}
+
+		sb.WriteString(fmt.Sprintf("%-15s | %-10s | %-10s | %-10s | %s\n",
+			r.Platform, discoverStr, fetchStr, contentStr, status))
+	}
+	return sb.String()
+}
+
+// HealthResult is a structured result from a crawler health check.
+type HealthResult struct {
+	Platform      string
+	DiscoverCount int
+	ContentLength int
+	Error         error
+	Skipped       bool
+}
@@ -15,9 +15,19 @@ import (
 	"github.com/hireme/hireme/internal/db"
 	"github.com/hireme/hireme/internal/filter"
 	"github.com/hireme/hireme/internal/logger"
+	"github.com/hireme/hireme/pkg/crawler/amazon"
 	"github.com/hireme/hireme/pkg/crawler/apple"
+	"github.com/hireme/hireme/pkg/crawler/bytedance"
+	"github.com/hireme/hireme/pkg/crawler/databricks"
 	"github.com/hireme/hireme/pkg/crawler/google"
+	"github.com/hireme/hireme/pkg/crawler/linkedin"
 	"github.com/hireme/hireme/pkg/crawler/meta"
+	"github.com/hireme/hireme/pkg/crawler/microsoft"
+	"github.com/hireme/hireme/pkg/crawler/netflix"
+	"github.com/hireme/hireme/pkg/crawler/nvidia"
+	"github.com/hireme/hireme/pkg/crawler/snowflake"
+	"github.com/hireme/hireme/pkg/crawler/stripe"
+	"github.com/hireme/hireme/pkg/crawler/uber"
 )

 type WorkerPool struct {
@@ -347,5 +357,35 @@ func (p *WorkerPool) getCrawlerForPlatform(platform string) crawler.SiteCrawler
 	if strings.Contains(plat, "meta") || strings.Contains(plat, "facebook") {
 		return meta.NewMetaCrawler()
 	}
+	if strings.Contains(plat, "amazon") || strings.Contains(plat, "aws") {
+		return amazon.NewAmazonCrawler()
+	}
+	if strings.Contains(plat, "microsoft") || strings.Contains(plat, "azure") {
+		return microsoft.NewMicrosoftCrawler()
+	}
+	if strings.Contains(plat, "bytedance") || strings.Contains(plat, "tiktok") {
+		return bytedance.NewByteDanceCrawler()
+	}
+	if strings.Contains(plat, "nvidia") {
+		return nvidia.NewNvidiaCrawler()
+	}
+	if strings.Contains(plat, "netflix") {
+		return netflix.NewNetflixCrawler()
+	}
+	if strings.Contains(plat, "uber") {
+		return uber.NewUberCrawler()
+	}
+	if strings.Contains(plat, "linkedin") {
+		return linkedin.NewLinkedInCrawler()
+	}
+	if strings.Contains(plat, "stripe") {
+		return stripe.NewStripeCrawler()
+	}
+	if strings.Contains(plat, "databricks") {
+		return databricks.NewDatabricksCrawler()
+	}
+	if strings.Contains(plat, "snowflake") {
+		return snowflake.NewSnowflakeCrawler()
+	}
 	return crawler.NewGenericCrawler()
 }