hireme/pkg/crawler/uber/uber.go

package uber

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"

	"git.pengzhan.dev/haopengzhan/hireme/internal/crawler"
	"git.pengzhan.dev/haopengzhan/hireme/internal/logger"
)

type UberCrawler struct {
	client *http.Client
}

func NewUberCrawler() *UberCrawler {
	return NewUberCrawlerWithClient(&http.Client{
		Timeout: 30 * time.Second,
	})
}

func NewUberCrawlerWithClient(client *http.Client) *UberCrawler {
	return &UberCrawler{
		client: client,
	}
}

func (c *UberCrawler) CanHandle(platform string) bool {
	p := strings.ToLower(platform)
	return p == "uber" || strings.Contains(p, "uber.com/careers") || strings.Contains(p, "uber.com/global/en/careers")
}

type uberSearchPayload struct {
	Limit  int                    `json:"limit"`
	Page   int                    `json:"page"`
	Params map[string]interface{} `json:"params"`
}

type uberSearchResponse struct {
	Status string `json:"status"`
	Data   struct {
		TotalResults int           `json:"totalResults"`
		Results      []uberJobItem `json:"results"`
	} `json:"data"`
}

type uberJobItem struct {
	ID                 int64    `json:"id"`
	Title              string   `json:"title"`
	Description        string   `json:"description"`
	Location           string   `json:"location"`
	City               string   `json:"city"`
	Country            string   `json:"countryName"`
	Department         string   `json:"department"`
	Team               string   `json:"team"`
	LineOfBusiness     string   `json:"lineOfBusiness"`
	AllLocations       []string `json:"allLocations"`
	CreationDate       string   `json:"creationDate"`
	WhatYouWillDo      string   `json:"whatYouWillDo"`
	BasicQualifications string   `json:"basicQualifications"`
}

func (c *UberCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
	seen := make(map[string]bool)
	var discovered []crawler.DiscoveredJob

	apiURL := "https://www.uber.com/api/loadSearchJobsResults"

	keyword := "Software Engineer"
	u, err := url.Parse(targetURL)
	if err == nil && u.Query().Get("query") != "" {
		keyword = u.Query().Get("query")
	}

	pageSize := 20
	for page := 0; page < 5; page++ {
		reqPayload := uberSearchPayload{
			Limit: pageSize,
			Page:  page,
			Params: map[string]interface{}{
				"query": keyword,
			},
		}

		payloadBytes, err := json.Marshal(reqPayload)
		if err != nil {
			return nil, err
		}

		req, err := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewBuffer(payloadBytes))
		if err != nil {
			if page == 0 {
				return nil, err
			}
			break
		}

		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
		req.Header.Set("x-csrf-token", "x")

		resp, err := c.client.Do(req)
		if err != nil {
			if page == 0 {
				return nil, err
			}
			break
		}
		defer func() { _ = resp.Body.Close() }()

		body, err := io.ReadAll(resp.Body)
		if err != nil {
			break
		}

		var searchResp uberSearchResponse
		if err := json.Unmarshal(body, &searchResp); err != nil {
			if page == 0 {
				return nil, fmt.Errorf("failed to parse Uber search response: %w", err)
			}
			break
		}

		results := searchResp.Data.Results
		if len(results) == 0 {
			break
		}

		newFound := false
		for _, job := range results {
			if job.ID == 0 {
				continue
			}

			jobIDStr := fmt.Sprintf("%d", job.ID)
			applyURL := fmt.Sprintf("https://www.uber.com/global/en/careers/list/%s/", jobIDStr)

			if !seen[applyURL] {
				seen[applyURL] = true
				newFound = true
				discovered = append(discovered, crawler.DiscoveredJob{
					CanonicalID: fmt.Sprintf("uber:%s", jobIDStr),
					ApplyURL:    applyURL,
				})
			}
		}

		if !newFound || len(results) < pageSize {
			break
		}
	}

	logger.Log.Infow("Uber discovery finished", "found_count", len(discovered))
	return discovered, nil
}

func (c *UberCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
	jobID := parseUberJobID(applyURL)
	if jobID == "" {
		jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16]
	}

	// Try API endpoint: https://www.uber.com/api/loadJobDetails?id={id}
	detailAPI := fmt.Sprintf("https://www.uber.com/api/loadJobDetails?id=%s", jobID)
	req, err := http.NewRequestWithContext(ctx, "GET", detailAPI, nil)
	if err == nil {
		req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
		req.Header.Set("x-csrf-token", "x")
		resp, err := c.client.Do(req)
		if err == nil {
			defer func() { _ = resp.Body.Close() }()
			body, _ := io.ReadAll(resp.Body)
			var detailResp struct {
				Data uberJobItem `json:"data"`
			}
			if err := json.Unmarshal(body, &detailResp); err == nil && detailResp.Data.Title != "" {
				j := detailResp.Data
				var sb strings.Builder
				if j.Description != "" {
					sb.WriteString("## Description\n\n")
					sb.WriteString(cleanHTML(j.Description))
					sb.WriteString("\n\n")
				}
				if j.WhatYouWillDo != "" {
					sb.WriteString("## What You'll Do\n\n")
					sb.WriteString(cleanHTML(j.WhatYouWillDo))
					sb.WriteString("\n\n")
				}
				if j.BasicQualifications != "" {
					sb.WriteString("## Basic Qualifications\n\n")
					sb.WriteString(cleanHTML(j.BasicQualifications))
					sb.WriteString("\n\n")
				}

				loc := j.Location
				if loc == "" && j.City != "" {
					loc = fmt.Sprintf("%s, %s", j.City, j.Country)
				}
				if loc == "" {
					loc = "Seattle, WA"
				}

				return &crawler.ExtractedJob{
					CanonicalID: fmt.Sprintf("uber:%s", jobID),
					Title:       j.Title,
					Company:     "Uber",
					Location:    loc,
					ApplyURL:    applyURL,
					RawContent:  sb.String(),
				}, nil
			}
		}
	}

	// Fallback to HTML
	htmlBody, err := c.fetchURL(ctx, applyURL)
	if err != nil {
		return nil, err
	}

	title := extractRegex(htmlBody, `<h1[^>]*>(.*?)</h1>`)
	if title == "" {
		title = "Software Engineer - Uber"
	}

	return &crawler.ExtractedJob{
		CanonicalID: fmt.Sprintf("uber:%s", jobID),
		Title:       cleanHTML(title),
		Company:     "Uber",
		Location:    "Seattle, WA",
		ApplyURL:    applyURL,
		RawContent:  cleanHTML(htmlBody),
	}, nil
}

func (c *UberCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) {
	req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
	if err != nil {
		return "", fmt.Errorf("failed to create request: %w", err)
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
	req.Header.Set("Accept", "application/json, text/html, */*")

	resp, err := c.client.Do(req)
	if err != nil {
		return "", fmt.Errorf("HTTP GET failed: %w", err)
	}
	defer func() { _ = resp.Body.Close() }()

	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr)
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", fmt.Errorf("failed to read response: %w", err)
	}

	return string(body), nil
}

func parseUberJobID(u string) string {
	re := regexp.MustCompile(`/list/([0-9A-Za-z]+)`)
	m := re.FindStringSubmatch(u)
	if len(m) >= 2 {
		return m[1]
	}
	return ""
}

func extractRegex(s, pattern string) string {
	re := regexp.MustCompile(pattern)
	m := re.FindStringSubmatch(s)
	if len(m) >= 2 {
		return m[1]
	}
	return ""
}

func cleanHTML(s string) string {
	s = strings.ReplaceAll(s, "<b>", "**")
	s = strings.ReplaceAll(s, "</b>", "**")
	s = strings.ReplaceAll(s, "<strong>", "**")
	s = strings.ReplaceAll(s, "</strong>", "**")
	s = strings.ReplaceAll(s, "<br>", "\n")
	s = strings.ReplaceAll(s, "<br/>", "\n")
	s = strings.ReplaceAll(s, "<br />", "\n")
	s = strings.ReplaceAll(s, "<li>", "- ")
	s = strings.ReplaceAll(s, "</li>", "\n")
	s = strings.ReplaceAll(s, "<ul>", "\n")
	s = strings.ReplaceAll(s, "</ul>", "\n")
	s = strings.ReplaceAll(s, "<p>", "\n")
	s = strings.ReplaceAll(s, "</p>", "\n")
	reTags := regexp.MustCompile(`<[^>]*>`)
	s = reTags.ReplaceAllString(s, "")
	return strings.TrimSpace(s)
}