118 lines
4.4 KiB
Go
118 lines
4.4 KiB
Go
package google
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestCanHandle(t *testing.T) {
|
|
crawler := NewGoogleCrawler()
|
|
|
|
if !crawler.CanHandle("google_jobs") {
|
|
t.Errorf("expected CanHandle('google_jobs') = true")
|
|
}
|
|
if !crawler.CanHandle("google") {
|
|
t.Errorf("expected CanHandle('google') = true")
|
|
}
|
|
if !crawler.CanHandle("https://www.google.com/about/careers/applications/") {
|
|
t.Errorf("expected CanHandle for Google careers URL = true")
|
|
}
|
|
if crawler.CanHandle("linkedin") {
|
|
t.Errorf("expected CanHandle('linkedin') = false")
|
|
}
|
|
}
|
|
|
|
func TestDefaultTestTargetURL(t *testing.T) {
|
|
if !strings.Contains(DefaultTestTargetURL, "employment_type=FULL_TIME") {
|
|
t.Errorf("DefaultTestTargetURL missing employment_type filter")
|
|
}
|
|
if !strings.Contains(DefaultTestTargetURL, "location=Seattle%2C%20WA%2C%20USA") {
|
|
t.Errorf("DefaultTestTargetURL missing Seattle location filter")
|
|
}
|
|
}
|
|
|
|
// TestGoogleCrawler_RealNetworkCrawling tests DiscoverJobs and FetchJobDetail
|
|
func TestGoogleCrawler_RealNetworkCrawling(t *testing.T) {
|
|
crawler := NewGoogleCrawler()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
|
defer cancel()
|
|
|
|
// 1. Step 1: Discover jobs
|
|
discovered, err := crawler.DiscoverJobs(ctx, DefaultTestTargetURL)
|
|
if err != nil {
|
|
t.Logf("Notice: Network fetch error (expected in offline sandbox): %v", err)
|
|
return
|
|
}
|
|
|
|
if len(discovered) == 0 {
|
|
t.Errorf("⚠️ YELLOW FLAG: 0 jobs discovered from Google Careers target URL (%s).", DefaultTestTargetURL)
|
|
return
|
|
}
|
|
|
|
t.Logf("Successfully discovered %d job links", len(discovered))
|
|
|
|
// 2. Step 3: Fetch detail for first discovered job link
|
|
first := discovered[0]
|
|
if first.CanonicalID == "" || !strings.HasPrefix(first.CanonicalID, "google:") {
|
|
t.Errorf("invalid CanonicalID: %q", first.CanonicalID)
|
|
}
|
|
|
|
detail, err := crawler.FetchJobDetail(ctx, first.ApplyURL)
|
|
if err != nil {
|
|
t.Logf("Notice: Detail fetch error: %v", err)
|
|
return
|
|
}
|
|
|
|
if strings.TrimSpace(detail.Title) == "" {
|
|
t.Errorf("detail job has empty Title")
|
|
}
|
|
if detail.Company == "" {
|
|
t.Errorf("detail job has empty Company")
|
|
}
|
|
if strings.TrimSpace(detail.RawContent) == "" {
|
|
t.Errorf("detail job has empty RawContent")
|
|
}
|
|
}
|
|
|
|
func TestExtractFromAFData(t *testing.T) {
|
|
// Realistic AF_initDataCallback payload mimicking Google Careers HTML
|
|
html := `<html><script class="ds:0">AF_initDataCallback({key: 'ds:0', hash: '1', data:[["141612485990851270","Software Engineer Manager, Modem NPI","https://apply.url",[null,"responsibilities"],[null,"qualifications"],"projects/gweb-careers-proto/tenants/xxx/companies/yyy",null,"Google","en-US",[["Shanghai, China",["Shanghai, China"],"Shanghai",null,"Shanghai","CN"]],[null,"about the job"],[2]]], sideChannel: {}});</script></html>`
|
|
|
|
title, company, location := extractFromAFData(html)
|
|
|
|
if title != "Software Engineer Manager, Modem NPI" {
|
|
t.Errorf("title = %q, want 'Software Engineer Manager, Modem NPI'", title)
|
|
}
|
|
if company != "Google" {
|
|
t.Errorf("company = %q, want 'Google'", company)
|
|
}
|
|
if location != "Shanghai, China" {
|
|
t.Errorf("location = %q, want 'Shanghai, China'", location)
|
|
}
|
|
|
|
// Test with multiple locations and different company
|
|
html2 := `<script class="ds:0">AF_initDataCallback({key: 'ds:0', hash: '1', data:[["123","Strategy Associate, YouTube","https://apply.url",[null,"resp"],[null,"quals"],"projects/x",null,"YouTube","en-US",[["San Bruno, CA, USA",["addr"],"San Bruno","94066","CA","US"],["New York, NY, USA",["addr"],"New York",null,"NY","US"]],[null,"about"]]], sideChannel: {}});</script>`
|
|
|
|
title2, company2, location2 := extractFromAFData(html2)
|
|
|
|
if title2 != "Strategy Associate, YouTube" {
|
|
t.Errorf("title2 = %q, want 'Strategy Associate, YouTube'", title2)
|
|
}
|
|
if company2 != "YouTube" {
|
|
t.Errorf("company2 = %q, want 'YouTube'", company2)
|
|
}
|
|
if location2 != "San Bruno, CA, USA; New York, NY, USA" {
|
|
t.Errorf("location2 = %q, want 'San Bruno, CA, USA; New York, NY, USA'", location2)
|
|
}
|
|
|
|
// Test with DeepMind company
|
|
html3 := `<script class="ds:0">AF_initDataCallback({key: 'ds:0', hash: '1', data:[["456","Research Scientist","https://apply.url",[null,"resp"],[null,"quals"],"projects/x",null,"Google DeepMind","en-US",[["London, UK",["addr"],"London",null,"England","GB"]],[null,"about"]]], sideChannel: {}});</script>`
|
|
|
|
_, company3, _ := extractFromAFData(html3)
|
|
if company3 != "Google DeepMind" {
|
|
t.Errorf("company3 = %q, want 'Google DeepMind'", company3)
|
|
}
|
|
}
|