feat: initialize project structure with SQLite database, crawler framework, and LLM-based JD analysis integration

2026-08-12 01:13:53 -07:00
commit 51b9468737
27 changed files with 3422 additions and 0 deletions
@@ -0,0 +1,117 @@
+package google
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestCanHandle(t *testing.T) {
+	crawler := NewGoogleCrawler()
+
+	if !crawler.CanHandle("google_jobs") {
+		t.Errorf("expected CanHandle('google_jobs') = true")
+	}
+	if !crawler.CanHandle("google") {
+		t.Errorf("expected CanHandle('google') = true")
+	}
+	if !crawler.CanHandle("https://www.google.com/about/careers/applications/") {
+		t.Errorf("expected CanHandle for Google careers URL = true")
+	}
+	if crawler.CanHandle("linkedin") {
+		t.Errorf("expected CanHandle('linkedin') = false")
+	}
+}
+
+func TestDefaultTestTargetURL(t *testing.T) {
+	if !strings.Contains(DefaultTestTargetURL, "employment_type=FULL_TIME") {
+		t.Errorf("DefaultTestTargetURL missing employment_type filter")
+	}
+	if !strings.Contains(DefaultTestTargetURL, "location=Seattle%2C%20WA%2C%20USA") {
+		t.Errorf("DefaultTestTargetURL missing Seattle location filter")
+	}
+}
+
+// TestGoogleCrawler_RealNetworkCrawling tests DiscoverJobs and FetchJobDetail
+func TestGoogleCrawler_RealNetworkCrawling(t *testing.T) {
+	crawler := NewGoogleCrawler()
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+	defer cancel()
+
+	// 1. Step 1: Discover jobs
+	discovered, err := crawler.DiscoverJobs(ctx, DefaultTestTargetURL)
+	if err != nil {
+		t.Logf("Notice: Network fetch error (expected in offline sandbox): %v", err)
+		return
+	}
+
+	if len(discovered) == 0 {
+		t.Errorf("⚠️ YELLOW FLAG: 0 jobs discovered from Google Careers target URL (%s).", DefaultTestTargetURL)
+		return
+	}
+
+	t.Logf("Successfully discovered %d job links", len(discovered))
+
+	// 2. Step 3: Fetch detail for first discovered job link
+	first := discovered[0]
+	if first.CanonicalID == "" || !strings.HasPrefix(first.CanonicalID, "google:") {
+		t.Errorf("invalid CanonicalID: %q", first.CanonicalID)
+	}
+
+	detail, err := crawler.FetchJobDetail(ctx, first.ApplyURL)
+	if err != nil {
+		t.Logf("Notice: Detail fetch error: %v", err)
+		return
+	}
+
+	if strings.TrimSpace(detail.Title) == "" {
+		t.Errorf("detail job has empty Title")
+	}
+	if detail.Company == "" {
+		t.Errorf("detail job has empty Company")
+	}
+	if strings.TrimSpace(detail.RawContent) == "" {
+		t.Errorf("detail job has empty RawContent")
+	}
+}
+
+func TestExtractFromAFData(t *testing.T) {
+	// Realistic AF_initDataCallback payload mimicking Google Careers HTML
+	html := `<html><script class="ds:0">AF_initDataCallback({key: 'ds:0', hash: '1', data:[["141612485990851270","Software Engineer Manager, Modem NPI","https://apply.url",[null,"responsibilities"],[null,"qualifications"],"projects/gweb-careers-proto/tenants/xxx/companies/yyy",null,"Google","en-US",[["Shanghai, China",["Shanghai, China"],"Shanghai",null,"Shanghai","CN"]],[null,"about the job"],[2]]], sideChannel: {}});</script></html>`
+
+	title, company, location := extractFromAFData(html)
+
+	if title != "Software Engineer Manager, Modem NPI" {
+		t.Errorf("title = %q, want 'Software Engineer Manager, Modem NPI'", title)
+	}
+	if company != "Google" {
+		t.Errorf("company = %q, want 'Google'", company)
+	}
+	if location != "Shanghai, China" {
+		t.Errorf("location = %q, want 'Shanghai, China'", location)
+	}
+
+	// Test with multiple locations and different company
+	html2 := `<script class="ds:0">AF_initDataCallback({key: 'ds:0', hash: '1', data:[["123","Strategy Associate, YouTube","https://apply.url",[null,"resp"],[null,"quals"],"projects/x",null,"YouTube","en-US",[["San Bruno, CA, USA",["addr"],"San Bruno","94066","CA","US"],["New York, NY, USA",["addr"],"New York",null,"NY","US"]],[null,"about"]]], sideChannel: {}});</script>`
+
+	title2, company2, location2 := extractFromAFData(html2)
+
+	if title2 != "Strategy Associate, YouTube" {
+		t.Errorf("title2 = %q, want 'Strategy Associate, YouTube'", title2)
+	}
+	if company2 != "YouTube" {
+		t.Errorf("company2 = %q, want 'YouTube'", company2)
+	}
+	if location2 != "San Bruno, CA, USA; New York, NY, USA" {
+		t.Errorf("location2 = %q, want 'San Bruno, CA, USA; New York, NY, USA'", location2)
+	}
+
+	// Test with DeepMind company
+	html3 := `<script class="ds:0">AF_initDataCallback({key: 'ds:0', hash: '1', data:[["456","Research Scientist","https://apply.url",[null,"resp"],[null,"quals"],"projects/x",null,"Google DeepMind","en-US",[["London, UK",["addr"],"London",null,"England","GB"]],[null,"about"]]], sideChannel: {}});</script>`
+
+	_, company3, _ := extractFromAFData(html3)
+	if company3 != "Google DeepMind" {
+		t.Errorf("company3 = %q, want 'Google DeepMind'", company3)
+	}
+}