57 lines
1.6 KiB
Go
57 lines
1.6 KiB
Go
package meta
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestMetaCrawler_EndToEnd(t *testing.T) {
|
|
if testing.Short() {
|
|
t.Skip("Skipping long-running Meta crawler network test in short mode")
|
|
}
|
|
|
|
c := NewMetaCrawler()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
|
defer cancel()
|
|
|
|
testURL := "https://www.metacareers.com/jobsearch/?offices[0]=Seattle%2C%20WA&roles[0]=Full%20time%20employment"
|
|
|
|
t.Logf("Discovering jobs from: %s", testURL)
|
|
discovered, err := c.DiscoverJobs(ctx, testURL)
|
|
if err != nil {
|
|
t.Fatalf("Failed to discover jobs: %v", err)
|
|
}
|
|
|
|
if len(discovered) == 0 {
|
|
t.Fatalf("Expected to find at least 1 job, found 0")
|
|
}
|
|
|
|
t.Logf("Successfully discovered %d jobs", len(discovered))
|
|
|
|
// Fetch details for the first discovered job
|
|
firstJob := discovered[0]
|
|
t.Logf("Fetching details for job: %s", firstJob.ApplyURL)
|
|
|
|
detail, err := c.FetchJobDetail(ctx, firstJob.ApplyURL)
|
|
if err != nil {
|
|
t.Fatalf("Failed to fetch job details: %v", err)
|
|
}
|
|
|
|
if detail.Title == "" || detail.Title == "Unknown Meta Title" {
|
|
t.Errorf("Expected a valid job title, got %s", detail.Title)
|
|
}
|
|
|
|
if len(detail.RawContent) < 100 {
|
|
t.Errorf("Expected RawContent to contain the job description, but it was too short (%d bytes)", len(detail.RawContent))
|
|
}
|
|
|
|
if detail.CanonicalID == "unknown" || detail.CanonicalID == "" {
|
|
t.Errorf("Failed to extract canonical ID from %s", firstJob.ApplyURL)
|
|
}
|
|
|
|
t.Logf("Extracted Title: %s", detail.Title)
|
|
t.Logf("Extracted CanonicalID: %s", detail.CanonicalID)
|
|
t.Logf("Raw Content Length: %d", len(detail.RawContent))
|
|
}
|