hireme/pkg/crawler/google/google.go

package google

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"regexp"
	"strings"
	"time"

	md "github.com/JohannesKaufmann/html-to-markdown/v2"
	"github.com/hireme/hireme/internal/crawler"
	"golang.org/x/net/html"
)

const (
	DefaultTestTargetURL = "https://www.google.com/about/careers/applications/jobs/results/?employment_type=FULL_TIME&q=%22Software%20Engineer%22&location=Seattle%2C%20WA%2C%20USA&target_level=MID"
)

type GoogleCrawler struct {
	client *http.Client
}

func NewGoogleCrawler() *GoogleCrawler {
	return &GoogleCrawler{
		client: &http.Client{
			Timeout: 30 * time.Second,
		},
	}
}

func (g *GoogleCrawler) CanHandle(platform string) bool {
	p := strings.ToLower(platform)
	return p == "google_jobs" || p == "google" || strings.Contains(p, "google.com/about/careers")
}

// DiscoverJobs finds all job links from search result pages (paginated).
func (g *GoogleCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
	seen := make(map[string]bool)
	var discovered []crawler.DiscoveredJob

	for page := 1; page <= 5; page++ {
		pageURL := targetURL
		if strings.Contains(pageURL, "?") {
			pageURL = fmt.Sprintf("%s&page=%d&page_size=100", targetURL, page)
		} else {
			pageURL = fmt.Sprintf("%s?page=%d&page_size=100", targetURL, page)
		}

		htmlContent, err := g.fetchWithRetry(ctx, pageURL)
		if err != nil {
			if page == 1 {
				htmlContent, err = g.fetchWithRetry(ctx, targetURL)
				if err != nil {
					return nil, err
				}
			} else {
				break
			}
		}

		// Scan for job IDs in both <a href> tags and inline script data
		reID := regexp.MustCompile(`jobs/results/(\d[\w\-]+)`)
		matches := reID.FindAllStringSubmatch(htmlContent, -1)
		for _, m := range matches {
			if len(m) >= 2 {
				jobID := m[1]
				// Filter out page= params that look like job IDs
				if strings.HasPrefix(jobID, "?") || len(jobID) < 5 {
					continue
				}
				applyURL := fmt.Sprintf("https://www.google.com/about/careers/applications/jobs/results/%s", jobID)
				if !seen[applyURL] {
					seen[applyURL] = true
					discovered = append(discovered, crawler.DiscoveredJob{
						CanonicalID: fmt.Sprintf("google:%s", jobID),
						ApplyURL:    applyURL,
					})
				}
			}
		}
	}

	return discovered, nil
}

// FetchJobDetail fetches a single job detail page and extracts structured data
// from the AF_initDataCallback JSON embedded in the page's <script> tags.
//
// Google Careers pages are JavaScript-rendered SPAs. The server-side HTML embeds
// structured job data in AF_initDataCallback({key:'ds:0', data:[...]}) payloads.
// The DOM itself (spans under <main>) only contains a sidebar list of unrelated jobs,
// so we must parse the script data, not the DOM.
func (g *GoogleCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
	htmlContent, err := g.fetchWithRetry(ctx, applyURL)
	if err != nil {
		return nil, err
	}

	jobID := parseIDFromHref(applyURL)
	if jobID == "" {
		jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16]
	}

	// Parse structured data from AF_initDataCallback (ds:0)
	title, company, location := extractFromAFData(htmlContent)

	// Fallback: data-title attribute
	if title == "" {
		if m := regexp.MustCompile(`data-title="([^"]+)"`).FindStringSubmatch(htmlContent); len(m) >= 2 {
			title = m[1]
		}
	}

	// Fallback: URL slug
	if title == "" || strings.EqualFold(title, "job details") {
		title = parseTitleFromURLSlug(applyURL)
	}
	if title == "" {
		title = "Unknown Position"
	}
	if company == "" {
		company = "Google"
	}
	if location == "" {
		location = "See Job Posting"
	}

	// Raw content: extract <main> HTML, convert to Markdown for LLM efficiency
	var rawMarkdown string
	doc, parseErr := html.Parse(strings.NewReader(htmlContent))
	if parseErr == nil {
		if mainEl := findElement(doc, "main"); mainEl != nil {
			var buf bytes.Buffer
			_ = html.Render(&buf, mainEl)
			converted, convErr := md.ConvertString(buf.String())
			if convErr == nil && len(converted) > 50 {
				rawMarkdown = converted
			}
		}
	}
	if rawMarkdown == "" {
		// Fallback: convert the whole page
		converted, convErr := md.ConvertString(htmlContent)
		if convErr == nil {
			rawMarkdown = converted
		} else {
			rawMarkdown = htmlContent
		}
	}

	return &crawler.ExtractedJob{
		CanonicalID: fmt.Sprintf("google:%s", jobID),
		Title:       cleanTitle(title),
		Company:     company,
		Location:    location,
		ApplyURL:    applyURL,
		RawContent:  rawMarkdown,
	}, nil
}

// extractFromAFData parses the AF_initDataCallback JSON for key 'ds:0'.
//
// The HTML contains: <script class="ds:0">AF_initDataCallback({key:'ds:0', ..., data:[[...]], sideChannel:{}});</script>
//
// The data array is: data[0] = [jobID, title, signinURL, responsibilities, qualifications,
//
//	tenantPath, null, COMPANY, language, LOCATIONS, aboutHTML, ...]
//
// Indices:
//
//	[0]  = Job ID (string)
//	[1]  = Title (string)
//	[7]  = Company (string: "Google", "YouTube", "Google DeepMind", "Waymo", etc.)
//	[9]  = Locations (array of arrays: [["Shanghai, China", [...], "Shanghai", null, "Shanghai", "CN"]])
func extractFromAFData(htmlContent string) (title, company, location string) {
	// Find the start of the ds:0 data payload using string search (more reliable than regex for large content)
	marker := `AF_initDataCallback({key: 'ds:0'`
	idx := strings.Index(htmlContent, marker)
	if idx == -1 {
		log.Printf("[GoogleCrawler] AF_initDataCallback ds:0 not found")
		return "", "", ""
	}

	// Find "data:" after the marker
	dataMarker := "data:"
	dataIdx := strings.Index(htmlContent[idx:], dataMarker)
	if dataIdx == -1 {
		log.Printf("[GoogleCrawler] 'data:' not found after ds:0 marker")
		return "", "", ""
	}
	dataStart := idx + dataIdx + len(dataMarker)

	// Use bracket counting to find the matching ] for the outer [
	jsonStr := extractBalancedBracket(htmlContent, dataStart)
	if jsonStr == "" {
		log.Printf("[GoogleCrawler] Failed to extract balanced bracket for ds:0 data")
		return "", "", ""
	}

	// Parse the outer array: data = [[...job record...]]
	var outerArray []json.RawMessage
	if err := json.Unmarshal([]byte(jsonStr), &outerArray); err != nil {
		log.Printf("[GoogleCrawler] JSON parse error for ds:0 outer: %v", err)
		return "", "", ""
	}
	if len(outerArray) == 0 {
		return "", "", ""
	}

	// Parse inner job record array
	var record []json.RawMessage
	if err := json.Unmarshal(outerArray[0], &record); err != nil {
		log.Printf("[GoogleCrawler] JSON parse error for job record: %v", err)
		return "", "", ""
	}

	// [1] = Title
	if len(record) > 1 {
		_ = json.Unmarshal(record[1], &title)
	}

	// [7] = Company
	if len(record) > 7 {
		_ = json.Unmarshal(record[7], &company)
	}

	// [9] = Locations array: [["Shanghai, China", [...], ...], ...]
	if len(record) > 9 {
		var locs [][]json.RawMessage
		if err := json.Unmarshal(record[9], &locs); err == nil {
			var parts []string
			for _, loc := range locs {
				if len(loc) > 0 {
					var display string
					if err := json.Unmarshal(loc[0], &display); err == nil && display != "" {
						parts = append(parts, display)
					}
				}
			}
			location = strings.Join(parts, "; ")
		}
	}

	log.Printf("[GoogleCrawler] AF_initDataCallback parsed: title=%q company=%q location=%q", title, company, location)
	return title, company, location
}

// extractBalancedBracket finds the JSON array starting at position startIdx in s,
// using bracket counting to handle nested arrays/objects.
func extractBalancedBracket(s string, startIdx int) string {
	// Skip whitespace to find the opening [
	i := startIdx
	for i < len(s) && (s[i] == ' ' || s[i] == '\t' || s[i] == '\n' || s[i] == '\r') {
		i++
	}
	if i >= len(s) || s[i] != '[' {
		return ""
	}

	depth := 0
	inString := false
	escaped := false

	for j := i; j < len(s); j++ {
		ch := s[j]

		if escaped {
			escaped = false
			continue
		}

		if ch == '\\' && inString {
			escaped = true
			continue
		}

		if ch == '"' {
			inString = !inString
			continue
		}

		if inString {
			continue
		}

		switch ch {
		case '[', '{':
			depth++
		case ']', '}':
			depth--
			if depth == 0 {
				return s[i : j+1]
			}
		}
	}

	return ""
}

// fetchWithRetry fetches a URL with retry logic for HTTP 429 rate limiting.
func (g *GoogleCrawler) fetchWithRetry(ctx context.Context, urlStr string) (string, error) {
	var lastErr error
	for attempt := 0; attempt < 3; attempt++ {
		if attempt > 0 {
			delay := time.Duration(attempt) * 2 * time.Second
			log.Printf("[GoogleCrawler] Retry #%d for %s after %v", attempt, urlStr, delay)
			time.Sleep(delay)
		}

		body, err := g.fetchURL(ctx, urlStr)
		if err != nil {
			if strings.Contains(err.Error(), "429") || strings.Contains(err.Error(), "status 429") {
				lastErr = err
				continue
			}
			return "", err
		}
		return body, nil
	}
	return "", fmt.Errorf("max retries exceeded for %s: %w", urlStr, lastErr)
}

func (g *GoogleCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) {
	req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
	if err != nil {
		return "", fmt.Errorf("failed to create HTTP request for %s: %w", urlStr, err)
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
	req.Header.Set("Accept-Language", "en-US,en;q=0.9")

	resp, err := g.client.Do(req)
	if err != nil {
		return "", fmt.Errorf("HTTP fetch failed for %s: %w", urlStr, err)
	}
	defer func() { _ = resp.Body.Close() }()

	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr)
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", fmt.Errorf("failed to read response body for %s: %w", urlStr, err)
	}

	return string(body), nil
}

func parseIDFromHref(href string) string {
	re := regexp.MustCompile(`jobs/results/(\d[\w\-]+)`)
	m := re.FindStringSubmatch(href)
	if len(m) >= 2 {
		return m[1]
	}
	return ""
}

func parseTitleFromURLSlug(applyURL string) string {
	re := regexp.MustCompile(`jobs/results/\d+\-([a-zA-Z0-9\-]+)`)
	m := re.FindStringSubmatch(applyURL)
	if len(m) >= 2 {
		slug := m[1]
		words := strings.Split(slug, "-")
		for i, w := range words {
			if len(w) > 0 {
				words[i] = strings.ToUpper(w[:1]) + w[1:]
			}
		}
		return strings.Join(words, " ")
	}
	return ""
}

func findElement(n *html.Node, tag string) *html.Node {
	if n.Type == html.ElementNode && n.Data == tag {
		return n
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		if found := findElement(c, tag); found != nil {
			return found
		}
	}
	return nil
}

func cleanTitle(s string) string {
	cleaned := strings.TrimSpace(s)
	cleaned = strings.ReplaceAll(cleaned, `\u0026`, "&")
	cleaned = strings.ReplaceAll(cleaned, `\u003d`, "=")

	prefixes := []string{
		"Learn more about ",
		"Learn more ",
		"Apply for ",
		"View job: ",
	}
	for _, p := range prefixes {
		if strings.HasPrefix(cleaned, p) {
			cleaned = strings.TrimPrefix(cleaned, p)
			break
		}
	}
	return strings.TrimSpace(cleaned)
}