hireme/internal/crawler/crawlertest/crawlertest.go

// Package crawlertest provides a standardized integration test harness for
// validating that each SiteCrawler still functions against its real target URL.
//
// Each crawler package should have a TestXxx_LiveIntegration test that calls
// crawlertest.RunLiveIntegration(). This tests three real-world failure modes:
//
//  1. DISCOVER STAGE - Can we still hit the search URL and get job listings?
//     Detects: search API endpoint moved, query param format changed, auth required.
//
//  2. FETCH STAGE - Can we still fetch a single job detail page?
//     Detects: detail URL pattern changed, page returns 403/404, rate limiting.
//
//  3. PARSE STAGE - Can we still extract structured data from the response?
//     Detects: HTML/JSON schema changed, title/company/content extraction broken.
package crawlertest

import (
	"context"
	"fmt"
	"strings"
	"testing"
	"time"

	"git.pengzhan.dev/haopengzhan/hireme/internal/crawler"
)

// LiveTestConfig defines what a live integration test expects from a crawler.
type LiveTestConfig struct {
	// PlatformName is the human-readable platform name (e.g., "Amazon", "Google").
	PlatformName string

	// Crawler is the SiteCrawler instance to test.
	Crawler crawler.SiteCrawler

	// TargetURL is the real search/listing URL to test discovery against.
	TargetURL string

	// ExpectedCanonicalPrefix is the prefix each CanonicalID should start with
	// (e.g., "amazon:", "google:"). Empty string skips this check.
	ExpectedCanonicalPrefix string

	// ExpectedCompany is the expected Company field in FetchJobDetail results.
	// Empty string skips this check.
	ExpectedCompany string

	// MinDiscoverCount is the minimum number of jobs DiscoverJobs should return.
	// Defaults to 1 if not set.
	MinDiscoverCount int

	// MinContentLength is the minimum byte length of RawContent in FetchJobDetail.
	// Defaults to 100 if not set.
	MinContentLength int

	// Timeout is the context timeout for the entire test. Defaults to 30s.
	Timeout time.Duration
}

// RunLiveIntegration runs a standardized 3-stage integration test against real
// targets. It gracefully skips on network errors (offline/sandbox) but FAILS
// loudly when the crawler reaches the site but can't parse data correctly.
//
// This is the test that tells you "the site changed and our crawler is broken".
//
// Usage in each crawler package's _test.go:
//
//	func TestAmazonCrawler_LiveIntegration(t *testing.T) {
//	    crawlertest.RunLiveIntegration(t, crawlertest.LiveTestConfig{
//	        PlatformName:            "Amazon",
//	        Crawler:                 NewAmazonCrawler(),
//	        TargetURL:               "https://www.amazon.jobs/en/search.json?...",
//	        ExpectedCanonicalPrefix: "amazon:",
//	        ExpectedCompany:         "Amazon",
//	    })
//	}
func RunLiveIntegration(t *testing.T, cfg LiveTestConfig) {
	t.Helper()

	// Apply defaults
	if cfg.MinDiscoverCount <= 0 {
		cfg.MinDiscoverCount = 1
	}
	if cfg.MinContentLength <= 0 {
		cfg.MinContentLength = 100
	}
	if cfg.Timeout <= 0 {
		cfg.Timeout = 30 * time.Second
	}

	ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout)
	defer cancel()

	// =========================================================================
	// STAGE 1: DISCOVER — Can we reach the search URL and get job listings?
	// =========================================================================
	t.Run("Stage1_Discover", func(t *testing.T) {
		t.Logf("[%s] Discovering jobs from: %s", cfg.PlatformName, cfg.TargetURL)

		discovered, err := cfg.Crawler.DiscoverJobs(ctx, cfg.TargetURL)
		if err != nil {
			if isNetworkError(err) {
				t.Skipf("SKIP (offline): %v", err)
			}
			t.Fatalf("❌ DISCOVER FAILED: %v\n"+
				"  → The search URL may have changed format or the API endpoint moved.\n"+
				"  → Target URL: %s", err, cfg.TargetURL)
		}

		if len(discovered) == 0 {
			t.Fatalf("❌ DISCOVER RETURNED 0 JOBS\n"+
				"  → The search results page format may have changed.\n"+
				"  → Check if the target URL still returns results in a browser.\n"+
				"  → Target URL: %s", cfg.TargetURL)
		}

		if len(discovered) < cfg.MinDiscoverCount {
			t.Errorf("⚠️  DISCOVER LOW COUNT: got %d jobs, expected at least %d\n"+
				"  → The search API may be returning fewer results than expected.",
				len(discovered), cfg.MinDiscoverCount)
		}

		t.Logf("✅ Discovered %d jobs", len(discovered))

		// Validate first result structure
		first := discovered[0]
		if first.ApplyURL == "" {
			t.Errorf("❌ First discovered job has empty ApplyURL")
		}
		if first.CanonicalID == "" {
			t.Errorf("❌ First discovered job has empty CanonicalID")
		}
		if cfg.ExpectedCanonicalPrefix != "" && !strings.HasPrefix(first.CanonicalID, cfg.ExpectedCanonicalPrefix) {
			t.Errorf("❌ CANONICAL ID FORMAT CHANGED: got %q, expected prefix %q\n"+
				"  → The job ID extraction pattern may need updating.",
				first.CanonicalID, cfg.ExpectedCanonicalPrefix)
		}

		t.Logf("   First job: ID=%s URL=%s", first.CanonicalID, first.ApplyURL)
	})

	// Re-discover for stage 2 (subtests get their own scope)
	discovered, err := cfg.Crawler.DiscoverJobs(ctx, cfg.TargetURL)
	if err != nil || len(discovered) == 0 {
		// Stage 1 already reported the failure
		return
	}

	// =========================================================================
	// STAGE 2: FETCH — Can we fetch a job detail page?
	// =========================================================================
	firstJob := discovered[0]
	var detail *crawler.ExtractedJob

	t.Run("Stage2_Fetch", func(t *testing.T) {
		t.Logf("[%s] Fetching job detail: %s", cfg.PlatformName, firstJob.ApplyURL)

		var fetchErr error
		detail, fetchErr = cfg.Crawler.FetchJobDetail(ctx, firstJob.ApplyURL)
		if fetchErr != nil {
			if isNetworkError(fetchErr) {
				t.Skipf("SKIP (offline): %v", fetchErr)
			}
			t.Fatalf("❌ FETCH FAILED: %v\n"+
				"  → The job detail URL pattern may have changed.\n"+
				"  → The page may now require JavaScript rendering.\n"+
				"  → Job URL: %s", fetchErr, firstJob.ApplyURL)
		}

		if detail == nil {
			t.Fatalf("❌ FETCH RETURNED nil ExtractedJob")
		}

		t.Logf("✅ Fetched job detail successfully")
	})

	if detail == nil {
		return
	}

	// =========================================================================
	// STAGE 3: PARSE — Can we extract structured data from the response?
	// =========================================================================
	t.Run("Stage3_Parse", func(t *testing.T) {
		t.Logf("[%s] Validating parsed fields", cfg.PlatformName)
		failures := 0

		// Title
		if strings.TrimSpace(detail.Title) == "" {
			t.Errorf("❌ PARSE: Title is empty\n"+
				"  → The title extraction selector/pattern may have changed.\n"+
				"  → Job URL: %s", firstJob.ApplyURL)
			failures++
		} else {
			t.Logf("   Title:    %s", detail.Title)
		}

		// Company
		if strings.TrimSpace(detail.Company) == "" {
			t.Errorf("❌ PARSE: Company is empty\n"+
				"  → The company extraction logic may have changed.")
			failures++
		} else {
			t.Logf("   Company:  %s", detail.Company)
		}

		if cfg.ExpectedCompany != "" && !strings.Contains(detail.Company, cfg.ExpectedCompany) {
			t.Errorf("⚠️  PARSE: Company mismatch: got %q, expected to contain %q\n"+
				"  → The company name format may have changed.",
				detail.Company, cfg.ExpectedCompany)
		}

		// Location
		if strings.TrimSpace(detail.Location) == "" {
			t.Errorf("⚠️  PARSE: Location is empty\n"+
				"  → Location extraction may have changed.")
		} else {
			t.Logf("   Location: %s", detail.Location)
		}

		// CanonicalID
		if detail.CanonicalID == "" {
			t.Errorf("❌ PARSE: CanonicalID is empty")
			failures++
		}

		// ApplyURL
		if detail.ApplyURL == "" {
			t.Errorf("❌ PARSE: ApplyURL is empty")
			failures++
		}

		// RawContent (the critical one for downstream LLM analysis)
		contentLen := len(detail.RawContent)
		if contentLen == 0 {
			t.Errorf("❌ PARSE: RawContent is EMPTY\n"+
				"  → The content extraction is completely broken.\n"+
				"  → The page structure or API response schema has changed.\n"+
				"  → Job URL: %s", firstJob.ApplyURL)
			failures++
		} else if contentLen < cfg.MinContentLength {
			t.Errorf("⚠️  PARSE: RawContent is too short (%d bytes, min %d)\n"+
				"  → The content extraction may be partially broken.\n"+
				"  → Job URL: %s", contentLen, cfg.MinContentLength, firstJob.ApplyURL)
		} else {
			t.Logf("   Content:  %d bytes", contentLen)
		}

		if failures == 0 {
			t.Logf("✅ All parse checks passed")
		} else {
			t.Logf("❌ %d parse check(s) failed — crawler needs fixing", failures)
		}
	})
}

// isNetworkError checks if an error is a network/DNS/connectivity issue
// (as opposed to a parsing or HTTP status error from a reachable server).
func isNetworkError(err error) bool {
	if err == nil {
		return false
	}
	msg := err.Error()
	networkPatterns := []string{
		"no such host",
		"dial tcp",
		"connection refused",
		"network is unreachable",
		"i/o timeout",
		"TLS handshake timeout",
		"operation not permitted",
		"connect: operation not permitted",
	}
	for _, pattern := range networkPatterns {
		if strings.Contains(msg, pattern) {
			return true
		}
	}
	return false
}

// FormatHealthReport generates a summary table from multiple test results.
// This can be called from a top-level aggregation test.
func FormatHealthReport(results []HealthResult) string {
	var sb strings.Builder
	sb.WriteString(fmt.Sprintf("%-15s | %-10s | %-10s | %-10s | %s\n",
		"Platform", "Discover", "Fetch", "Content", "Status"))
	sb.WriteString(strings.Repeat("-", 70) + "\n")
	for _, r := range results {
		discoverStr := fmt.Sprintf("%d jobs", r.DiscoverCount)
		fetchStr := "✓"
		contentStr := fmt.Sprintf("%d B", r.ContentLength)
		status := "✅ HEALTHY"

		if r.Skipped {
			status = "⏭ SKIPPED"
			discoverStr = "-"
			fetchStr = "-"
			contentStr = "-"
		} else if r.Error != nil {
			status = "❌ BROKEN"
			if r.DiscoverCount == 0 {
				discoverStr = "FAIL"
				fetchStr = "-"
				contentStr = "-"
			} else if r.ContentLength == 0 {
				fetchStr = "FAIL"
				contentStr = "-"
			}
		} else if r.DiscoverCount == 0 {
			status = "⚠️  DEGRADED"
			discoverStr = "0 ⚠️"
		}

		sb.WriteString(fmt.Sprintf("%-15s | %-10s | %-10s | %-10s | %s\n",
			r.Platform, discoverStr, fetchStr, contentStr, status))
	}
	return sb.String()
}

// HealthResult is a structured result from a crawler health check.
type HealthResult struct {
	Platform      string
	DiscoverCount int
	ContentLength int
	Error         error
	Skipped       bool
}