feat: initialize project structure with SQLite database, crawler framework, and LLM-based JD analysis integration
This commit is contained in:
@@ -0,0 +1,400 @@
|
||||
package google
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hireme/hireme/internal/crawler"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultTestTargetURL = "https://www.google.com/about/careers/applications/jobs/results/?employment_type=FULL_TIME&q=%22Software%20Engineer%22&location=Seattle%2C%20WA%2C%20USA&target_level=MID"
|
||||
)
|
||||
|
||||
type GoogleCrawler struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewGoogleCrawler() *GoogleCrawler {
|
||||
return &GoogleCrawler{
|
||||
client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (g *GoogleCrawler) CanHandle(platform string) bool {
|
||||
p := strings.ToLower(platform)
|
||||
return p == "google_jobs" || p == "google" || strings.Contains(p, "google.com/about/careers")
|
||||
}
|
||||
|
||||
// DiscoverJobs finds all job links from search result pages (paginated).
|
||||
func (g *GoogleCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
|
||||
seen := make(map[string]bool)
|
||||
var discovered []crawler.DiscoveredJob
|
||||
|
||||
for page := 1; page <= 5; page++ {
|
||||
pageURL := targetURL
|
||||
if strings.Contains(pageURL, "?") {
|
||||
pageURL = fmt.Sprintf("%s&page=%d&page_size=100", targetURL, page)
|
||||
} else {
|
||||
pageURL = fmt.Sprintf("%s?page=%d&page_size=100", targetURL, page)
|
||||
}
|
||||
|
||||
htmlContent, err := g.fetchWithRetry(ctx, pageURL)
|
||||
if err != nil {
|
||||
if page == 1 {
|
||||
htmlContent, err = g.fetchWithRetry(ctx, targetURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Scan for job IDs in both <a href> tags and inline script data
|
||||
reID := regexp.MustCompile(`jobs/results/(\d[\w\-]+)`)
|
||||
matches := reID.FindAllStringSubmatch(htmlContent, -1)
|
||||
for _, m := range matches {
|
||||
if len(m) >= 2 {
|
||||
jobID := m[1]
|
||||
// Filter out page= params that look like job IDs
|
||||
if strings.HasPrefix(jobID, "?") || len(jobID) < 5 {
|
||||
continue
|
||||
}
|
||||
applyURL := fmt.Sprintf("https://www.google.com/about/careers/applications/jobs/results/%s", jobID)
|
||||
if !seen[applyURL] {
|
||||
seen[applyURL] = true
|
||||
discovered = append(discovered, crawler.DiscoveredJob{
|
||||
CanonicalID: fmt.Sprintf("google:%s", jobID),
|
||||
ApplyURL: applyURL,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return discovered, nil
|
||||
}
|
||||
|
||||
// FetchJobDetail fetches a single job detail page and extracts structured data
|
||||
// from the AF_initDataCallback JSON embedded in the page's <script> tags.
|
||||
//
|
||||
// Google Careers pages are JavaScript-rendered SPAs. The server-side HTML embeds
|
||||
// structured job data in AF_initDataCallback({key:'ds:0', data:[...]}) payloads.
|
||||
// The DOM itself (spans under <main>) only contains a sidebar list of unrelated jobs,
|
||||
// so we must parse the script data, not the DOM.
|
||||
func (g *GoogleCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
|
||||
htmlContent, err := g.fetchWithRetry(ctx, applyURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
jobID := parseIDFromHref(applyURL)
|
||||
if jobID == "" {
|
||||
jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16]
|
||||
}
|
||||
|
||||
// Parse structured data from AF_initDataCallback (ds:0)
|
||||
title, company, location := extractFromAFData(htmlContent)
|
||||
|
||||
// Fallback: data-title attribute
|
||||
if title == "" {
|
||||
if m := regexp.MustCompile(`data-title="([^"]+)"`).FindStringSubmatch(htmlContent); len(m) >= 2 {
|
||||
title = m[1]
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: URL slug
|
||||
if title == "" || strings.EqualFold(title, "job details") {
|
||||
title = parseTitleFromURLSlug(applyURL)
|
||||
}
|
||||
if title == "" {
|
||||
title = "Unknown Position"
|
||||
}
|
||||
if company == "" {
|
||||
company = "Google"
|
||||
}
|
||||
if location == "" {
|
||||
location = "See Job Posting"
|
||||
}
|
||||
|
||||
// Raw content: render the <main> container HTML
|
||||
var buf bytes.Buffer
|
||||
doc, parseErr := html.Parse(strings.NewReader(htmlContent))
|
||||
if parseErr == nil {
|
||||
if main := findElement(doc, "main"); main != nil {
|
||||
_ = html.Render(&buf, main)
|
||||
}
|
||||
}
|
||||
if buf.Len() < 100 {
|
||||
buf.Reset()
|
||||
buf.WriteString(htmlContent)
|
||||
}
|
||||
|
||||
return &crawler.ExtractedJob{
|
||||
CanonicalID: fmt.Sprintf("google:%s", jobID),
|
||||
Title: cleanTitle(title),
|
||||
Company: company,
|
||||
Location: location,
|
||||
ApplyURL: applyURL,
|
||||
RawContent: buf.String(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// extractFromAFData parses the AF_initDataCallback JSON for key 'ds:0'.
|
||||
//
|
||||
// The HTML contains: <script class="ds:0">AF_initDataCallback({key:'ds:0', ..., data:[[...]], sideChannel:{}});</script>
|
||||
//
|
||||
// The data array is: data[0] = [jobID, title, signinURL, responsibilities, qualifications,
|
||||
//
|
||||
// tenantPath, null, COMPANY, language, LOCATIONS, aboutHTML, ...]
|
||||
//
|
||||
// Indices:
|
||||
//
|
||||
// [0] = Job ID (string)
|
||||
// [1] = Title (string)
|
||||
// [7] = Company (string: "Google", "YouTube", "Google DeepMind", "Waymo", etc.)
|
||||
// [9] = Locations (array of arrays: [["Shanghai, China", [...], "Shanghai", null, "Shanghai", "CN"]])
|
||||
func extractFromAFData(htmlContent string) (title, company, location string) {
|
||||
// Find the start of the ds:0 data payload using string search (more reliable than regex for large content)
|
||||
marker := `AF_initDataCallback({key: 'ds:0'`
|
||||
idx := strings.Index(htmlContent, marker)
|
||||
if idx == -1 {
|
||||
log.Printf("[GoogleCrawler] AF_initDataCallback ds:0 not found")
|
||||
return "", "", ""
|
||||
}
|
||||
|
||||
// Find "data:" after the marker
|
||||
dataMarker := "data:"
|
||||
dataIdx := strings.Index(htmlContent[idx:], dataMarker)
|
||||
if dataIdx == -1 {
|
||||
log.Printf("[GoogleCrawler] 'data:' not found after ds:0 marker")
|
||||
return "", "", ""
|
||||
}
|
||||
dataStart := idx + dataIdx + len(dataMarker)
|
||||
|
||||
// Use bracket counting to find the matching ] for the outer [
|
||||
jsonStr := extractBalancedBracket(htmlContent, dataStart)
|
||||
if jsonStr == "" {
|
||||
log.Printf("[GoogleCrawler] Failed to extract balanced bracket for ds:0 data")
|
||||
return "", "", ""
|
||||
}
|
||||
|
||||
// Parse the outer array: data = [[...job record...]]
|
||||
var outerArray []json.RawMessage
|
||||
if err := json.Unmarshal([]byte(jsonStr), &outerArray); err != nil {
|
||||
log.Printf("[GoogleCrawler] JSON parse error for ds:0 outer: %v", err)
|
||||
return "", "", ""
|
||||
}
|
||||
if len(outerArray) == 0 {
|
||||
return "", "", ""
|
||||
}
|
||||
|
||||
// Parse inner job record array
|
||||
var record []json.RawMessage
|
||||
if err := json.Unmarshal(outerArray[0], &record); err != nil {
|
||||
log.Printf("[GoogleCrawler] JSON parse error for job record: %v", err)
|
||||
return "", "", ""
|
||||
}
|
||||
|
||||
// [1] = Title
|
||||
if len(record) > 1 {
|
||||
_ = json.Unmarshal(record[1], &title)
|
||||
}
|
||||
|
||||
// [7] = Company
|
||||
if len(record) > 7 {
|
||||
_ = json.Unmarshal(record[7], &company)
|
||||
}
|
||||
|
||||
// [9] = Locations array: [["Shanghai, China", [...], ...], ...]
|
||||
if len(record) > 9 {
|
||||
var locs [][]json.RawMessage
|
||||
if err := json.Unmarshal(record[9], &locs); err == nil {
|
||||
var parts []string
|
||||
for _, loc := range locs {
|
||||
if len(loc) > 0 {
|
||||
var display string
|
||||
if err := json.Unmarshal(loc[0], &display); err == nil && display != "" {
|
||||
parts = append(parts, display)
|
||||
}
|
||||
}
|
||||
}
|
||||
location = strings.Join(parts, "; ")
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("[GoogleCrawler] AF_initDataCallback parsed: title=%q company=%q location=%q", title, company, location)
|
||||
return title, company, location
|
||||
}
|
||||
|
||||
// extractBalancedBracket finds the JSON array starting at position startIdx in s,
|
||||
// using bracket counting to handle nested arrays/objects.
|
||||
func extractBalancedBracket(s string, startIdx int) string {
|
||||
// Skip whitespace to find the opening [
|
||||
i := startIdx
|
||||
for i < len(s) && (s[i] == ' ' || s[i] == '\t' || s[i] == '\n' || s[i] == '\r') {
|
||||
i++
|
||||
}
|
||||
if i >= len(s) || s[i] != '[' {
|
||||
return ""
|
||||
}
|
||||
|
||||
depth := 0
|
||||
inString := false
|
||||
escaped := false
|
||||
|
||||
for j := i; j < len(s); j++ {
|
||||
ch := s[j]
|
||||
|
||||
if escaped {
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
|
||||
if ch == '\\' && inString {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
|
||||
if ch == '"' {
|
||||
inString = !inString
|
||||
continue
|
||||
}
|
||||
|
||||
if inString {
|
||||
continue
|
||||
}
|
||||
|
||||
if ch == '[' || ch == '{' {
|
||||
depth++
|
||||
} else if ch == ']' || ch == '}' {
|
||||
depth--
|
||||
if depth == 0 {
|
||||
return s[i : j+1]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
// fetchWithRetry fetches a URL with retry logic for HTTP 429 rate limiting.
|
||||
func (g *GoogleCrawler) fetchWithRetry(ctx context.Context, urlStr string) (string, error) {
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < 3; attempt++ {
|
||||
if attempt > 0 {
|
||||
delay := time.Duration(attempt) * 2 * time.Second
|
||||
log.Printf("[GoogleCrawler] Retry #%d for %s after %v", attempt, urlStr, delay)
|
||||
time.Sleep(delay)
|
||||
}
|
||||
|
||||
body, err := g.fetchURL(ctx, urlStr)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "429") || strings.Contains(err.Error(), "status 429") {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
return "", fmt.Errorf("max retries exceeded for %s: %w", urlStr, lastErr)
|
||||
}
|
||||
|
||||
func (g *GoogleCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create HTTP request for %s: %w", urlStr, err)
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
|
||||
resp, err := g.client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("HTTP fetch failed for %s: %w", urlStr, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read response body for %s: %w", urlStr, err)
|
||||
}
|
||||
|
||||
return string(body), nil
|
||||
}
|
||||
|
||||
func parseIDFromHref(href string) string {
|
||||
re := regexp.MustCompile(`jobs/results/(\d[\w\-]+)`)
|
||||
m := re.FindStringSubmatch(href)
|
||||
if len(m) >= 2 {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func parseTitleFromURLSlug(applyURL string) string {
|
||||
re := regexp.MustCompile(`jobs/results/\d+\-([a-zA-Z0-9\-]+)`)
|
||||
m := re.FindStringSubmatch(applyURL)
|
||||
if len(m) >= 2 {
|
||||
slug := m[1]
|
||||
words := strings.Split(slug, "-")
|
||||
for i, w := range words {
|
||||
if len(w) > 0 {
|
||||
words[i] = strings.ToUpper(w[:1]) + w[1:]
|
||||
}
|
||||
}
|
||||
return strings.Join(words, " ")
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func findElement(n *html.Node, tag string) *html.Node {
|
||||
if n.Type == html.ElementNode && n.Data == tag {
|
||||
return n
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
if found := findElement(c, tag); found != nil {
|
||||
return found
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func cleanTitle(s string) string {
|
||||
cleaned := strings.TrimSpace(s)
|
||||
cleaned = strings.ReplaceAll(cleaned, `\u0026`, "&")
|
||||
cleaned = strings.ReplaceAll(cleaned, `\u003d`, "=")
|
||||
|
||||
prefixes := []string{
|
||||
"Learn more about ",
|
||||
"Learn more ",
|
||||
"Apply for ",
|
||||
"View job: ",
|
||||
}
|
||||
for _, p := range prefixes {
|
||||
if strings.HasPrefix(cleaned, p) {
|
||||
cleaned = strings.TrimPrefix(cleaned, p)
|
||||
break
|
||||
}
|
||||
}
|
||||
return strings.TrimSpace(cleaned)
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package google
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCanHandle(t *testing.T) {
|
||||
crawler := NewGoogleCrawler()
|
||||
|
||||
if !crawler.CanHandle("google_jobs") {
|
||||
t.Errorf("expected CanHandle('google_jobs') = true")
|
||||
}
|
||||
if !crawler.CanHandle("google") {
|
||||
t.Errorf("expected CanHandle('google') = true")
|
||||
}
|
||||
if !crawler.CanHandle("https://www.google.com/about/careers/applications/") {
|
||||
t.Errorf("expected CanHandle for Google careers URL = true")
|
||||
}
|
||||
if crawler.CanHandle("linkedin") {
|
||||
t.Errorf("expected CanHandle('linkedin') = false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultTestTargetURL(t *testing.T) {
|
||||
if !strings.Contains(DefaultTestTargetURL, "employment_type=FULL_TIME") {
|
||||
t.Errorf("DefaultTestTargetURL missing employment_type filter")
|
||||
}
|
||||
if !strings.Contains(DefaultTestTargetURL, "location=Seattle%2C%20WA%2C%20USA") {
|
||||
t.Errorf("DefaultTestTargetURL missing Seattle location filter")
|
||||
}
|
||||
}
|
||||
|
||||
// TestGoogleCrawler_RealNetworkCrawling tests DiscoverJobs and FetchJobDetail
|
||||
func TestGoogleCrawler_RealNetworkCrawling(t *testing.T) {
|
||||
crawler := NewGoogleCrawler()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// 1. Step 1: Discover jobs
|
||||
discovered, err := crawler.DiscoverJobs(ctx, DefaultTestTargetURL)
|
||||
if err != nil {
|
||||
t.Logf("Notice: Network fetch error (expected in offline sandbox): %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(discovered) == 0 {
|
||||
t.Errorf("⚠️ YELLOW FLAG: 0 jobs discovered from Google Careers target URL (%s).", DefaultTestTargetURL)
|
||||
return
|
||||
}
|
||||
|
||||
t.Logf("Successfully discovered %d job links", len(discovered))
|
||||
|
||||
// 2. Step 3: Fetch detail for first discovered job link
|
||||
first := discovered[0]
|
||||
if first.CanonicalID == "" || !strings.HasPrefix(first.CanonicalID, "google:") {
|
||||
t.Errorf("invalid CanonicalID: %q", first.CanonicalID)
|
||||
}
|
||||
|
||||
detail, err := crawler.FetchJobDetail(ctx, first.ApplyURL)
|
||||
if err != nil {
|
||||
t.Logf("Notice: Detail fetch error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if strings.TrimSpace(detail.Title) == "" {
|
||||
t.Errorf("detail job has empty Title")
|
||||
}
|
||||
if detail.Company == "" {
|
||||
t.Errorf("detail job has empty Company")
|
||||
}
|
||||
if strings.TrimSpace(detail.RawContent) == "" {
|
||||
t.Errorf("detail job has empty RawContent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractFromAFData(t *testing.T) {
|
||||
// Realistic AF_initDataCallback payload mimicking Google Careers HTML
|
||||
html := `<html><script class="ds:0">AF_initDataCallback({key: 'ds:0', hash: '1', data:[["141612485990851270","Software Engineer Manager, Modem NPI","https://apply.url",[null,"responsibilities"],[null,"qualifications"],"projects/gweb-careers-proto/tenants/xxx/companies/yyy",null,"Google","en-US",[["Shanghai, China",["Shanghai, China"],"Shanghai",null,"Shanghai","CN"]],[null,"about the job"],[2]]], sideChannel: {}});</script></html>`
|
||||
|
||||
title, company, location := extractFromAFData(html)
|
||||
|
||||
if title != "Software Engineer Manager, Modem NPI" {
|
||||
t.Errorf("title = %q, want 'Software Engineer Manager, Modem NPI'", title)
|
||||
}
|
||||
if company != "Google" {
|
||||
t.Errorf("company = %q, want 'Google'", company)
|
||||
}
|
||||
if location != "Shanghai, China" {
|
||||
t.Errorf("location = %q, want 'Shanghai, China'", location)
|
||||
}
|
||||
|
||||
// Test with multiple locations and different company
|
||||
html2 := `<script class="ds:0">AF_initDataCallback({key: 'ds:0', hash: '1', data:[["123","Strategy Associate, YouTube","https://apply.url",[null,"resp"],[null,"quals"],"projects/x",null,"YouTube","en-US",[["San Bruno, CA, USA",["addr"],"San Bruno","94066","CA","US"],["New York, NY, USA",["addr"],"New York",null,"NY","US"]],[null,"about"]]], sideChannel: {}});</script>`
|
||||
|
||||
title2, company2, location2 := extractFromAFData(html2)
|
||||
|
||||
if title2 != "Strategy Associate, YouTube" {
|
||||
t.Errorf("title2 = %q, want 'Strategy Associate, YouTube'", title2)
|
||||
}
|
||||
if company2 != "YouTube" {
|
||||
t.Errorf("company2 = %q, want 'YouTube'", company2)
|
||||
}
|
||||
if location2 != "San Bruno, CA, USA; New York, NY, USA" {
|
||||
t.Errorf("location2 = %q, want 'San Bruno, CA, USA; New York, NY, USA'", location2)
|
||||
}
|
||||
|
||||
// Test with DeepMind company
|
||||
html3 := `<script class="ds:0">AF_initDataCallback({key: 'ds:0', hash: '1', data:[["456","Research Scientist","https://apply.url",[null,"resp"],[null,"quals"],"projects/x",null,"Google DeepMind","en-US",[["London, UK",["addr"],"London",null,"England","GB"]],[null,"about"]]], sideChannel: {}});</script>`
|
||||
|
||||
_, company3, _ := extractFromAFData(html3)
|
||||
if company3 != "Google DeepMind" {
|
||||
t.Errorf("company3 = %q, want 'Google DeepMind'", company3)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user