package uber import ( "bytes" "context" "crypto/sha256" "encoding/json" "fmt" "io" "net/http" "net/url" "regexp" "strings" "time" "github.com/hireme/hireme/internal/crawler" "github.com/hireme/hireme/internal/logger" ) type UberCrawler struct { client *http.Client } func NewUberCrawler() *UberCrawler { return NewUberCrawlerWithClient(&http.Client{ Timeout: 30 * time.Second, }) } func NewUberCrawlerWithClient(client *http.Client) *UberCrawler { return &UberCrawler{ client: client, } } func (c *UberCrawler) CanHandle(platform string) bool { p := strings.ToLower(platform) return p == "uber" || strings.Contains(p, "uber.com/careers") || strings.Contains(p, "uber.com/global/en/careers") } type uberSearchPayload struct { Limit int `json:"limit"` Page int `json:"page"` Params map[string]interface{} `json:"params"` } type uberSearchResponse struct { Status string `json:"status"` Data struct { TotalResults int `json:"totalResults"` Results []uberJobItem `json:"results"` } `json:"data"` } type uberJobItem struct { ID int64 `json:"id"` Title string `json:"title"` Description string `json:"description"` Location string `json:"location"` City string `json:"city"` Country string `json:"countryName"` Department string `json:"department"` Team string `json:"team"` LineOfBusiness string `json:"lineOfBusiness"` AllLocations []string `json:"allLocations"` CreationDate string `json:"creationDate"` WhatYouWillDo string `json:"whatYouWillDo"` BasicQualifications string `json:"basicQualifications"` } func (c *UberCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { seen := make(map[string]bool) var discovered []crawler.DiscoveredJob apiURL := "https://www.uber.com/api/loadSearchJobsResults" keyword := "Software Engineer" u, err := url.Parse(targetURL) if err == nil && u.Query().Get("query") != "" { keyword = u.Query().Get("query") } pageSize := 20 for page := 0; page < 5; page++ { reqPayload := uberSearchPayload{ Limit: pageSize, Page: page, Params: map[string]interface{}{ "query": keyword, }, } payloadBytes, err := json.Marshal(reqPayload) if err != nil { return nil, err } req, err := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewBuffer(payloadBytes)) if err != nil { if page == 0 { return nil, err } break } req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") req.Header.Set("x-csrf-token", "x") resp, err := c.client.Do(req) if err != nil { if page == 0 { return nil, err } break } defer func() { _ = resp.Body.Close() }() body, err := io.ReadAll(resp.Body) if err != nil { break } var searchResp uberSearchResponse if err := json.Unmarshal(body, &searchResp); err != nil { if page == 0 { return nil, fmt.Errorf("failed to parse Uber search response: %w", err) } break } results := searchResp.Data.Results if len(results) == 0 { break } newFound := false for _, job := range results { if job.ID == 0 { continue } jobIDStr := fmt.Sprintf("%d", job.ID) applyURL := fmt.Sprintf("https://www.uber.com/global/en/careers/list/%s/", jobIDStr) if !seen[applyURL] { seen[applyURL] = true newFound = true discovered = append(discovered, crawler.DiscoveredJob{ CanonicalID: fmt.Sprintf("uber:%s", jobIDStr), ApplyURL: applyURL, }) } } if !newFound || len(results) < pageSize { break } } logger.Log.Infow("Uber discovery finished", "found_count", len(discovered)) return discovered, nil } func (c *UberCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { jobID := parseUberJobID(applyURL) if jobID == "" { jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16] } // Try API endpoint: https://www.uber.com/api/loadJobDetails?id={id} detailAPI := fmt.Sprintf("https://www.uber.com/api/loadJobDetails?id=%s", jobID) req, err := http.NewRequestWithContext(ctx, "GET", detailAPI, nil) if err == nil { req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") req.Header.Set("x-csrf-token", "x") resp, err := c.client.Do(req) if err == nil { defer func() { _ = resp.Body.Close() }() body, _ := io.ReadAll(resp.Body) var detailResp struct { Data uberJobItem `json:"data"` } if err := json.Unmarshal(body, &detailResp); err == nil && detailResp.Data.Title != "" { j := detailResp.Data var sb strings.Builder if j.Description != "" { sb.WriteString("## Description\n\n") sb.WriteString(cleanHTML(j.Description)) sb.WriteString("\n\n") } if j.WhatYouWillDo != "" { sb.WriteString("## What You'll Do\n\n") sb.WriteString(cleanHTML(j.WhatYouWillDo)) sb.WriteString("\n\n") } if j.BasicQualifications != "" { sb.WriteString("## Basic Qualifications\n\n") sb.WriteString(cleanHTML(j.BasicQualifications)) sb.WriteString("\n\n") } loc := j.Location if loc == "" && j.City != "" { loc = fmt.Sprintf("%s, %s", j.City, j.Country) } if loc == "" { loc = "Seattle, WA" } return &crawler.ExtractedJob{ CanonicalID: fmt.Sprintf("uber:%s", jobID), Title: j.Title, Company: "Uber", Location: loc, ApplyURL: applyURL, RawContent: sb.String(), }, nil } } } // Fallback to HTML htmlBody, err := c.fetchURL(ctx, applyURL) if err != nil { return nil, err } title := extractRegex(htmlBody, `
", "\n") s = strings.ReplaceAll(s, "
", "\n") reTags := regexp.MustCompile(`<[^>]*>`) s = reTags.ReplaceAllString(s, "") return strings.TrimSpace(s) }