feat: implement multiple new job crawlers and add corresponding test suites
This commit is contained in:
@@ -0,0 +1,326 @@
|
||||
// Package crawlertest provides a standardized integration test harness for
|
||||
// validating that each SiteCrawler still functions against its real target URL.
|
||||
//
|
||||
// Each crawler package should have a TestXxx_LiveIntegration test that calls
|
||||
// crawlertest.RunLiveIntegration(). This tests three real-world failure modes:
|
||||
//
|
||||
// 1. DISCOVER STAGE - Can we still hit the search URL and get job listings?
|
||||
// Detects: search API endpoint moved, query param format changed, auth required.
|
||||
//
|
||||
// 2. FETCH STAGE - Can we still fetch a single job detail page?
|
||||
// Detects: detail URL pattern changed, page returns 403/404, rate limiting.
|
||||
//
|
||||
// 3. PARSE STAGE - Can we still extract structured data from the response?
|
||||
// Detects: HTML/JSON schema changed, title/company/content extraction broken.
|
||||
package crawlertest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hireme/hireme/internal/crawler"
|
||||
)
|
||||
|
||||
// LiveTestConfig defines what a live integration test expects from a crawler.
|
||||
type LiveTestConfig struct {
|
||||
// PlatformName is the human-readable platform name (e.g., "Amazon", "Google").
|
||||
PlatformName string
|
||||
|
||||
// Crawler is the SiteCrawler instance to test.
|
||||
Crawler crawler.SiteCrawler
|
||||
|
||||
// TargetURL is the real search/listing URL to test discovery against.
|
||||
TargetURL string
|
||||
|
||||
// ExpectedCanonicalPrefix is the prefix each CanonicalID should start with
|
||||
// (e.g., "amazon:", "google:"). Empty string skips this check.
|
||||
ExpectedCanonicalPrefix string
|
||||
|
||||
// ExpectedCompany is the expected Company field in FetchJobDetail results.
|
||||
// Empty string skips this check.
|
||||
ExpectedCompany string
|
||||
|
||||
// MinDiscoverCount is the minimum number of jobs DiscoverJobs should return.
|
||||
// Defaults to 1 if not set.
|
||||
MinDiscoverCount int
|
||||
|
||||
// MinContentLength is the minimum byte length of RawContent in FetchJobDetail.
|
||||
// Defaults to 100 if not set.
|
||||
MinContentLength int
|
||||
|
||||
// Timeout is the context timeout for the entire test. Defaults to 30s.
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// RunLiveIntegration runs a standardized 3-stage integration test against real
|
||||
// targets. It gracefully skips on network errors (offline/sandbox) but FAILS
|
||||
// loudly when the crawler reaches the site but can't parse data correctly.
|
||||
//
|
||||
// This is the test that tells you "the site changed and our crawler is broken".
|
||||
//
|
||||
// Usage in each crawler package's _test.go:
|
||||
//
|
||||
// func TestAmazonCrawler_LiveIntegration(t *testing.T) {
|
||||
// crawlertest.RunLiveIntegration(t, crawlertest.LiveTestConfig{
|
||||
// PlatformName: "Amazon",
|
||||
// Crawler: NewAmazonCrawler(),
|
||||
// TargetURL: "https://www.amazon.jobs/en/search.json?...",
|
||||
// ExpectedCanonicalPrefix: "amazon:",
|
||||
// ExpectedCompany: "Amazon",
|
||||
// })
|
||||
// }
|
||||
func RunLiveIntegration(t *testing.T, cfg LiveTestConfig) {
|
||||
t.Helper()
|
||||
|
||||
// Apply defaults
|
||||
if cfg.MinDiscoverCount <= 0 {
|
||||
cfg.MinDiscoverCount = 1
|
||||
}
|
||||
if cfg.MinContentLength <= 0 {
|
||||
cfg.MinContentLength = 100
|
||||
}
|
||||
if cfg.Timeout <= 0 {
|
||||
cfg.Timeout = 30 * time.Second
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout)
|
||||
defer cancel()
|
||||
|
||||
// =========================================================================
|
||||
// STAGE 1: DISCOVER — Can we reach the search URL and get job listings?
|
||||
// =========================================================================
|
||||
t.Run("Stage1_Discover", func(t *testing.T) {
|
||||
t.Logf("[%s] Discovering jobs from: %s", cfg.PlatformName, cfg.TargetURL)
|
||||
|
||||
discovered, err := cfg.Crawler.DiscoverJobs(ctx, cfg.TargetURL)
|
||||
if err != nil {
|
||||
if isNetworkError(err) {
|
||||
t.Skipf("SKIP (offline): %v", err)
|
||||
}
|
||||
t.Fatalf("❌ DISCOVER FAILED: %v\n"+
|
||||
" → The search URL may have changed format or the API endpoint moved.\n"+
|
||||
" → Target URL: %s", err, cfg.TargetURL)
|
||||
}
|
||||
|
||||
if len(discovered) == 0 {
|
||||
t.Fatalf("❌ DISCOVER RETURNED 0 JOBS\n"+
|
||||
" → The search results page format may have changed.\n"+
|
||||
" → Check if the target URL still returns results in a browser.\n"+
|
||||
" → Target URL: %s", cfg.TargetURL)
|
||||
}
|
||||
|
||||
if len(discovered) < cfg.MinDiscoverCount {
|
||||
t.Errorf("⚠️ DISCOVER LOW COUNT: got %d jobs, expected at least %d\n"+
|
||||
" → The search API may be returning fewer results than expected.",
|
||||
len(discovered), cfg.MinDiscoverCount)
|
||||
}
|
||||
|
||||
t.Logf("✅ Discovered %d jobs", len(discovered))
|
||||
|
||||
// Validate first result structure
|
||||
first := discovered[0]
|
||||
if first.ApplyURL == "" {
|
||||
t.Errorf("❌ First discovered job has empty ApplyURL")
|
||||
}
|
||||
if first.CanonicalID == "" {
|
||||
t.Errorf("❌ First discovered job has empty CanonicalID")
|
||||
}
|
||||
if cfg.ExpectedCanonicalPrefix != "" && !strings.HasPrefix(first.CanonicalID, cfg.ExpectedCanonicalPrefix) {
|
||||
t.Errorf("❌ CANONICAL ID FORMAT CHANGED: got %q, expected prefix %q\n"+
|
||||
" → The job ID extraction pattern may need updating.",
|
||||
first.CanonicalID, cfg.ExpectedCanonicalPrefix)
|
||||
}
|
||||
|
||||
t.Logf(" First job: ID=%s URL=%s", first.CanonicalID, first.ApplyURL)
|
||||
})
|
||||
|
||||
// Re-discover for stage 2 (subtests get their own scope)
|
||||
discovered, err := cfg.Crawler.DiscoverJobs(ctx, cfg.TargetURL)
|
||||
if err != nil || len(discovered) == 0 {
|
||||
// Stage 1 already reported the failure
|
||||
return
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// STAGE 2: FETCH — Can we fetch a job detail page?
|
||||
// =========================================================================
|
||||
firstJob := discovered[0]
|
||||
var detail *crawler.ExtractedJob
|
||||
|
||||
t.Run("Stage2_Fetch", func(t *testing.T) {
|
||||
t.Logf("[%s] Fetching job detail: %s", cfg.PlatformName, firstJob.ApplyURL)
|
||||
|
||||
var fetchErr error
|
||||
detail, fetchErr = cfg.Crawler.FetchJobDetail(ctx, firstJob.ApplyURL)
|
||||
if fetchErr != nil {
|
||||
if isNetworkError(fetchErr) {
|
||||
t.Skipf("SKIP (offline): %v", fetchErr)
|
||||
}
|
||||
t.Fatalf("❌ FETCH FAILED: %v\n"+
|
||||
" → The job detail URL pattern may have changed.\n"+
|
||||
" → The page may now require JavaScript rendering.\n"+
|
||||
" → Job URL: %s", fetchErr, firstJob.ApplyURL)
|
||||
}
|
||||
|
||||
if detail == nil {
|
||||
t.Fatalf("❌ FETCH RETURNED nil ExtractedJob")
|
||||
}
|
||||
|
||||
t.Logf("✅ Fetched job detail successfully")
|
||||
})
|
||||
|
||||
if detail == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// STAGE 3: PARSE — Can we extract structured data from the response?
|
||||
// =========================================================================
|
||||
t.Run("Stage3_Parse", func(t *testing.T) {
|
||||
t.Logf("[%s] Validating parsed fields", cfg.PlatformName)
|
||||
failures := 0
|
||||
|
||||
// Title
|
||||
if strings.TrimSpace(detail.Title) == "" {
|
||||
t.Errorf("❌ PARSE: Title is empty\n"+
|
||||
" → The title extraction selector/pattern may have changed.\n"+
|
||||
" → Job URL: %s", firstJob.ApplyURL)
|
||||
failures++
|
||||
} else {
|
||||
t.Logf(" Title: %s", detail.Title)
|
||||
}
|
||||
|
||||
// Company
|
||||
if strings.TrimSpace(detail.Company) == "" {
|
||||
t.Errorf("❌ PARSE: Company is empty\n"+
|
||||
" → The company extraction logic may have changed.")
|
||||
failures++
|
||||
} else {
|
||||
t.Logf(" Company: %s", detail.Company)
|
||||
}
|
||||
|
||||
if cfg.ExpectedCompany != "" && !strings.Contains(detail.Company, cfg.ExpectedCompany) {
|
||||
t.Errorf("⚠️ PARSE: Company mismatch: got %q, expected to contain %q\n"+
|
||||
" → The company name format may have changed.",
|
||||
detail.Company, cfg.ExpectedCompany)
|
||||
}
|
||||
|
||||
// Location
|
||||
if strings.TrimSpace(detail.Location) == "" {
|
||||
t.Errorf("⚠️ PARSE: Location is empty\n"+
|
||||
" → Location extraction may have changed.")
|
||||
} else {
|
||||
t.Logf(" Location: %s", detail.Location)
|
||||
}
|
||||
|
||||
// CanonicalID
|
||||
if detail.CanonicalID == "" {
|
||||
t.Errorf("❌ PARSE: CanonicalID is empty")
|
||||
failures++
|
||||
}
|
||||
|
||||
// ApplyURL
|
||||
if detail.ApplyURL == "" {
|
||||
t.Errorf("❌ PARSE: ApplyURL is empty")
|
||||
failures++
|
||||
}
|
||||
|
||||
// RawContent (the critical one for downstream LLM analysis)
|
||||
contentLen := len(detail.RawContent)
|
||||
if contentLen == 0 {
|
||||
t.Errorf("❌ PARSE: RawContent is EMPTY\n"+
|
||||
" → The content extraction is completely broken.\n"+
|
||||
" → The page structure or API response schema has changed.\n"+
|
||||
" → Job URL: %s", firstJob.ApplyURL)
|
||||
failures++
|
||||
} else if contentLen < cfg.MinContentLength {
|
||||
t.Errorf("⚠️ PARSE: RawContent is too short (%d bytes, min %d)\n"+
|
||||
" → The content extraction may be partially broken.\n"+
|
||||
" → Job URL: %s", contentLen, cfg.MinContentLength, firstJob.ApplyURL)
|
||||
} else {
|
||||
t.Logf(" Content: %d bytes", contentLen)
|
||||
}
|
||||
|
||||
if failures == 0 {
|
||||
t.Logf("✅ All parse checks passed")
|
||||
} else {
|
||||
t.Logf("❌ %d parse check(s) failed — crawler needs fixing", failures)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// isNetworkError checks if an error is a network/DNS/connectivity issue
|
||||
// (as opposed to a parsing or HTTP status error from a reachable server).
|
||||
func isNetworkError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
msg := err.Error()
|
||||
networkPatterns := []string{
|
||||
"no such host",
|
||||
"dial tcp",
|
||||
"connection refused",
|
||||
"network is unreachable",
|
||||
"i/o timeout",
|
||||
"TLS handshake timeout",
|
||||
"operation not permitted",
|
||||
"connect: operation not permitted",
|
||||
}
|
||||
for _, pattern := range networkPatterns {
|
||||
if strings.Contains(msg, pattern) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// FormatHealthReport generates a summary table from multiple test results.
|
||||
// This can be called from a top-level aggregation test.
|
||||
func FormatHealthReport(results []HealthResult) string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("%-15s | %-10s | %-10s | %-10s | %s\n",
|
||||
"Platform", "Discover", "Fetch", "Content", "Status"))
|
||||
sb.WriteString(strings.Repeat("-", 70) + "\n")
|
||||
for _, r := range results {
|
||||
discoverStr := fmt.Sprintf("%d jobs", r.DiscoverCount)
|
||||
fetchStr := "✓"
|
||||
contentStr := fmt.Sprintf("%d B", r.ContentLength)
|
||||
status := "✅ HEALTHY"
|
||||
|
||||
if r.Skipped {
|
||||
status = "⏭ SKIPPED"
|
||||
discoverStr = "-"
|
||||
fetchStr = "-"
|
||||
contentStr = "-"
|
||||
} else if r.Error != nil {
|
||||
status = "❌ BROKEN"
|
||||
if r.DiscoverCount == 0 {
|
||||
discoverStr = "FAIL"
|
||||
fetchStr = "-"
|
||||
contentStr = "-"
|
||||
} else if r.ContentLength == 0 {
|
||||
fetchStr = "FAIL"
|
||||
contentStr = "-"
|
||||
}
|
||||
} else if r.DiscoverCount == 0 {
|
||||
status = "⚠️ DEGRADED"
|
||||
discoverStr = "0 ⚠️"
|
||||
}
|
||||
|
||||
sb.WriteString(fmt.Sprintf("%-15s | %-10s | %-10s | %-10s | %s\n",
|
||||
r.Platform, discoverStr, fetchStr, contentStr, status))
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// HealthResult is a structured result from a crawler health check.
|
||||
type HealthResult struct {
|
||||
Platform string
|
||||
DiscoverCount int
|
||||
ContentLength int
|
||||
Error error
|
||||
Skipped bool
|
||||
}
|
||||
Reference in New Issue
Block a user