package google import ( "bytes" "context" "crypto/sha256" "encoding/json" "fmt" "io" "log" "net/http" "regexp" "strings" "time" md "github.com/JohannesKaufmann/html-to-markdown/v2" "github.com/hireme/hireme/internal/crawler" "golang.org/x/net/html" ) const ( DefaultTestTargetURL = "https://www.google.com/about/careers/applications/jobs/results/?employment_type=FULL_TIME&q=%22Software%20Engineer%22&location=Seattle%2C%20WA%2C%20USA&target_level=MID" ) type GoogleCrawler struct { client *http.Client } func NewGoogleCrawler() *GoogleCrawler { return &GoogleCrawler{ client: &http.Client{ Timeout: 30 * time.Second, }, } } func (g *GoogleCrawler) CanHandle(platform string) bool { p := strings.ToLower(platform) return p == "google_jobs" || p == "google" || strings.Contains(p, "google.com/about/careers") } // DiscoverJobs finds all job links from search result pages (paginated). func (g *GoogleCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { seen := make(map[string]bool) var discovered []crawler.DiscoveredJob for page := 1; page <= 5; page++ { pageURL := targetURL if strings.Contains(pageURL, "?") { pageURL = fmt.Sprintf("%s&page=%d&page_size=100", targetURL, page) } else { pageURL = fmt.Sprintf("%s?page=%d&page_size=100", targetURL, page) } htmlContent, err := g.fetchWithRetry(ctx, pageURL) if err != nil { if page == 1 { htmlContent, err = g.fetchWithRetry(ctx, targetURL) if err != nil { return nil, err } } else { break } } // Scan for job IDs in both tags and inline script data reID := regexp.MustCompile(`jobs/results/(\d[\w\-]+)`) matches := reID.FindAllStringSubmatch(htmlContent, -1) for _, m := range matches { if len(m) >= 2 { jobID := m[1] // Filter out page= params that look like job IDs if strings.HasPrefix(jobID, "?") || len(jobID) < 5 { continue } applyURL := fmt.Sprintf("https://www.google.com/about/careers/applications/jobs/results/%s", jobID) if !seen[applyURL] { seen[applyURL] = true discovered = append(discovered, crawler.DiscoveredJob{ CanonicalID: fmt.Sprintf("google:%s", jobID), ApplyURL: applyURL, }) } } } } return discovered, nil } // FetchJobDetail fetches a single job detail page and extracts structured data // from the AF_initDataCallback JSON embedded in the page's // // The data array is: data[0] = [jobID, title, signinURL, responsibilities, qualifications, // // tenantPath, null, COMPANY, language, LOCATIONS, aboutHTML, ...] // // Indices: // // [0] = Job ID (string) // [1] = Title (string) // [7] = Company (string: "Google", "YouTube", "Google DeepMind", "Waymo", etc.) // [9] = Locations (array of arrays: [["Shanghai, China", [...], "Shanghai", null, "Shanghai", "CN"]]) func extractFromAFData(htmlContent string) (title, company, location string) { // Find the start of the ds:0 data payload using string search (more reliable than regex for large content) marker := `AF_initDataCallback({key: 'ds:0'` idx := strings.Index(htmlContent, marker) if idx == -1 { log.Printf("[GoogleCrawler] AF_initDataCallback ds:0 not found") return "", "", "" } // Find "data:" after the marker dataMarker := "data:" dataIdx := strings.Index(htmlContent[idx:], dataMarker) if dataIdx == -1 { log.Printf("[GoogleCrawler] 'data:' not found after ds:0 marker") return "", "", "" } dataStart := idx + dataIdx + len(dataMarker) // Use bracket counting to find the matching ] for the outer [ jsonStr := extractBalancedBracket(htmlContent, dataStart) if jsonStr == "" { log.Printf("[GoogleCrawler] Failed to extract balanced bracket for ds:0 data") return "", "", "" } // Parse the outer array: data = [[...job record...]] var outerArray []json.RawMessage if err := json.Unmarshal([]byte(jsonStr), &outerArray); err != nil { log.Printf("[GoogleCrawler] JSON parse error for ds:0 outer: %v", err) return "", "", "" } if len(outerArray) == 0 { return "", "", "" } // Parse inner job record array var record []json.RawMessage if err := json.Unmarshal(outerArray[0], &record); err != nil { log.Printf("[GoogleCrawler] JSON parse error for job record: %v", err) return "", "", "" } // [1] = Title if len(record) > 1 { _ = json.Unmarshal(record[1], &title) } // [7] = Company if len(record) > 7 { _ = json.Unmarshal(record[7], &company) } // [9] = Locations array: [["Shanghai, China", [...], ...], ...] if len(record) > 9 { var locs [][]json.RawMessage if err := json.Unmarshal(record[9], &locs); err == nil { var parts []string for _, loc := range locs { if len(loc) > 0 { var display string if err := json.Unmarshal(loc[0], &display); err == nil && display != "" { parts = append(parts, display) } } } location = strings.Join(parts, "; ") } } log.Printf("[GoogleCrawler] AF_initDataCallback parsed: title=%q company=%q location=%q", title, company, location) return title, company, location } // extractBalancedBracket finds the JSON array starting at position startIdx in s, // using bracket counting to handle nested arrays/objects. func extractBalancedBracket(s string, startIdx int) string { // Skip whitespace to find the opening [ i := startIdx for i < len(s) && (s[i] == ' ' || s[i] == '\t' || s[i] == '\n' || s[i] == '\r') { i++ } if i >= len(s) || s[i] != '[' { return "" } depth := 0 inString := false escaped := false for j := i; j < len(s); j++ { ch := s[j] if escaped { escaped = false continue } if ch == '\\' && inString { escaped = true continue } if ch == '"' { inString = !inString continue } if inString { continue } switch ch { case '[', '{': depth++ case ']', '}': depth-- if depth == 0 { return s[i : j+1] } } } return "" } // fetchWithRetry fetches a URL with retry logic for HTTP 429 rate limiting. func (g *GoogleCrawler) fetchWithRetry(ctx context.Context, urlStr string) (string, error) { var lastErr error for attempt := 0; attempt < 3; attempt++ { if attempt > 0 { delay := time.Duration(attempt) * 2 * time.Second log.Printf("[GoogleCrawler] Retry #%d for %s after %v", attempt, urlStr, delay) time.Sleep(delay) } body, err := g.fetchURL(ctx, urlStr) if err != nil { if strings.Contains(err.Error(), "429") || strings.Contains(err.Error(), "status 429") { lastErr = err continue } return "", err } return body, nil } return "", fmt.Errorf("max retries exceeded for %s: %w", urlStr, lastErr) } func (g *GoogleCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) { req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) if err != nil { return "", fmt.Errorf("failed to create HTTP request for %s: %w", urlStr, err) } req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") req.Header.Set("Accept-Language", "en-US,en;q=0.9") resp, err := g.client.Do(req) if err != nil { return "", fmt.Errorf("HTTP fetch failed for %s: %w", urlStr, err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr) } body, err := io.ReadAll(resp.Body) if err != nil { return "", fmt.Errorf("failed to read response body for %s: %w", urlStr, err) } return string(body), nil } func parseIDFromHref(href string) string { re := regexp.MustCompile(`jobs/results/(\d[\w\-]+)`) m := re.FindStringSubmatch(href) if len(m) >= 2 { return m[1] } return "" } func parseTitleFromURLSlug(applyURL string) string { re := regexp.MustCompile(`jobs/results/\d+\-([a-zA-Z0-9\-]+)`) m := re.FindStringSubmatch(applyURL) if len(m) >= 2 { slug := m[1] words := strings.Split(slug, "-") for i, w := range words { if len(w) > 0 { words[i] = strings.ToUpper(w[:1]) + w[1:] } } return strings.Join(words, " ") } return "" } func findElement(n *html.Node, tag string) *html.Node { if n.Type == html.ElementNode && n.Data == tag { return n } for c := n.FirstChild; c != nil; c = c.NextSibling { if found := findElement(c, tag); found != nil { return found } } return nil } func cleanTitle(s string) string { cleaned := strings.TrimSpace(s) cleaned = strings.ReplaceAll(cleaned, `\u0026`, "&") cleaned = strings.ReplaceAll(cleaned, `\u003d`, "=") prefixes := []string{ "Learn more about ", "Learn more ", "Apply for ", "View job: ", } for _, p := range prefixes { if strings.HasPrefix(cleaned, p) { cleaned = strings.TrimPrefix(cleaned, p) break } } return strings.TrimSpace(cleaned) }