feat: implement multiple new job crawlers and add corresponding test suites
This commit is contained in:
@@ -0,0 +1,306 @@
|
||||
package uber
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hireme/hireme/internal/crawler"
|
||||
"github.com/hireme/hireme/internal/logger"
|
||||
)
|
||||
|
||||
type UberCrawler struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewUberCrawler() *UberCrawler {
|
||||
return NewUberCrawlerWithClient(&http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
}
|
||||
|
||||
func NewUberCrawlerWithClient(client *http.Client) *UberCrawler {
|
||||
return &UberCrawler{
|
||||
client: client,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *UberCrawler) CanHandle(platform string) bool {
|
||||
p := strings.ToLower(platform)
|
||||
return p == "uber" || strings.Contains(p, "uber.com/careers") || strings.Contains(p, "uber.com/global/en/careers")
|
||||
}
|
||||
|
||||
type uberSearchPayload struct {
|
||||
Limit int `json:"limit"`
|
||||
Page int `json:"page"`
|
||||
Params map[string]interface{} `json:"params"`
|
||||
}
|
||||
|
||||
type uberSearchResponse struct {
|
||||
Status string `json:"status"`
|
||||
Data struct {
|
||||
TotalResults int `json:"totalResults"`
|
||||
Results []uberJobItem `json:"results"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type uberJobItem struct {
|
||||
ID int64 `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Location string `json:"location"`
|
||||
City string `json:"city"`
|
||||
Country string `json:"countryName"`
|
||||
Department string `json:"department"`
|
||||
Team string `json:"team"`
|
||||
LineOfBusiness string `json:"lineOfBusiness"`
|
||||
AllLocations []string `json:"allLocations"`
|
||||
CreationDate string `json:"creationDate"`
|
||||
WhatYouWillDo string `json:"whatYouWillDo"`
|
||||
BasicQualifications string `json:"basicQualifications"`
|
||||
}
|
||||
|
||||
func (c *UberCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
|
||||
seen := make(map[string]bool)
|
||||
var discovered []crawler.DiscoveredJob
|
||||
|
||||
apiURL := "https://www.uber.com/api/loadSearchJobsResults"
|
||||
|
||||
keyword := "Software Engineer"
|
||||
u, err := url.Parse(targetURL)
|
||||
if err == nil && u.Query().Get("query") != "" {
|
||||
keyword = u.Query().Get("query")
|
||||
}
|
||||
|
||||
pageSize := 20
|
||||
for page := 0; page < 5; page++ {
|
||||
reqPayload := uberSearchPayload{
|
||||
Limit: pageSize,
|
||||
Page: page,
|
||||
Params: map[string]interface{}{
|
||||
"query": keyword,
|
||||
},
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(reqPayload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewBuffer(payloadBytes))
|
||||
if err != nil {
|
||||
if page == 0 {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("x-csrf-token", "x")
|
||||
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
if page == 0 {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
||||
var searchResp uberSearchResponse
|
||||
if err := json.Unmarshal(body, &searchResp); err != nil {
|
||||
if page == 0 {
|
||||
return nil, fmt.Errorf("failed to parse Uber search response: %w", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
results := searchResp.Data.Results
|
||||
if len(results) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
newFound := false
|
||||
for _, job := range results {
|
||||
if job.ID == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
jobIDStr := fmt.Sprintf("%d", job.ID)
|
||||
applyURL := fmt.Sprintf("https://www.uber.com/global/en/careers/list/%s/", jobIDStr)
|
||||
|
||||
if !seen[applyURL] {
|
||||
seen[applyURL] = true
|
||||
newFound = true
|
||||
discovered = append(discovered, crawler.DiscoveredJob{
|
||||
CanonicalID: fmt.Sprintf("uber:%s", jobIDStr),
|
||||
ApplyURL: applyURL,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if !newFound || len(results) < pageSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
logger.Log.Infow("Uber discovery finished", "found_count", len(discovered))
|
||||
return discovered, nil
|
||||
}
|
||||
|
||||
func (c *UberCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
|
||||
jobID := parseUberJobID(applyURL)
|
||||
if jobID == "" {
|
||||
jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16]
|
||||
}
|
||||
|
||||
// Try API endpoint: https://www.uber.com/api/loadJobDetails?id={id}
|
||||
detailAPI := fmt.Sprintf("https://www.uber.com/api/loadJobDetails?id=%s", jobID)
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", detailAPI, nil)
|
||||
if err == nil {
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("x-csrf-token", "x")
|
||||
resp, err := c.client.Do(req)
|
||||
if err == nil {
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var detailResp struct {
|
||||
Data uberJobItem `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(body, &detailResp); err == nil && detailResp.Data.Title != "" {
|
||||
j := detailResp.Data
|
||||
var sb strings.Builder
|
||||
if j.Description != "" {
|
||||
sb.WriteString("## Description\n\n")
|
||||
sb.WriteString(cleanHTML(j.Description))
|
||||
sb.WriteString("\n\n")
|
||||
}
|
||||
if j.WhatYouWillDo != "" {
|
||||
sb.WriteString("## What You'll Do\n\n")
|
||||
sb.WriteString(cleanHTML(j.WhatYouWillDo))
|
||||
sb.WriteString("\n\n")
|
||||
}
|
||||
if j.BasicQualifications != "" {
|
||||
sb.WriteString("## Basic Qualifications\n\n")
|
||||
sb.WriteString(cleanHTML(j.BasicQualifications))
|
||||
sb.WriteString("\n\n")
|
||||
}
|
||||
|
||||
loc := j.Location
|
||||
if loc == "" && j.City != "" {
|
||||
loc = fmt.Sprintf("%s, %s", j.City, j.Country)
|
||||
}
|
||||
if loc == "" {
|
||||
loc = "Seattle, WA"
|
||||
}
|
||||
|
||||
return &crawler.ExtractedJob{
|
||||
CanonicalID: fmt.Sprintf("uber:%s", jobID),
|
||||
Title: j.Title,
|
||||
Company: "Uber",
|
||||
Location: loc,
|
||||
ApplyURL: applyURL,
|
||||
RawContent: sb.String(),
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to HTML
|
||||
htmlBody, err := c.fetchURL(ctx, applyURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
title := extractRegex(htmlBody, `<h1[^>]*>(.*?)</h1>`)
|
||||
if title == "" {
|
||||
title = "Software Engineer - Uber"
|
||||
}
|
||||
|
||||
return &crawler.ExtractedJob{
|
||||
CanonicalID: fmt.Sprintf("uber:%s", jobID),
|
||||
Title: cleanHTML(title),
|
||||
Company: "Uber",
|
||||
Location: "Seattle, WA",
|
||||
ApplyURL: applyURL,
|
||||
RawContent: cleanHTML(htmlBody),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *UberCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "application/json, text/html, */*")
|
||||
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("HTTP GET failed: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
return string(body), nil
|
||||
}
|
||||
|
||||
func parseUberJobID(u string) string {
|
||||
re := regexp.MustCompile(`/list/([0-9A-Za-z]+)`)
|
||||
m := re.FindStringSubmatch(u)
|
||||
if len(m) >= 2 {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractRegex(s, pattern string) string {
|
||||
re := regexp.MustCompile(pattern)
|
||||
m := re.FindStringSubmatch(s)
|
||||
if len(m) >= 2 {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func cleanHTML(s string) string {
|
||||
s = strings.ReplaceAll(s, "<b>", "**")
|
||||
s = strings.ReplaceAll(s, "</b>", "**")
|
||||
s = strings.ReplaceAll(s, "<strong>", "**")
|
||||
s = strings.ReplaceAll(s, "</strong>", "**")
|
||||
s = strings.ReplaceAll(s, "<br>", "\n")
|
||||
s = strings.ReplaceAll(s, "<br/>", "\n")
|
||||
s = strings.ReplaceAll(s, "<br />", "\n")
|
||||
s = strings.ReplaceAll(s, "<li>", "- ")
|
||||
s = strings.ReplaceAll(s, "</li>", "\n")
|
||||
s = strings.ReplaceAll(s, "<ul>", "\n")
|
||||
s = strings.ReplaceAll(s, "</ul>", "\n")
|
||||
s = strings.ReplaceAll(s, "<p>", "\n")
|
||||
s = strings.ReplaceAll(s, "</p>", "\n")
|
||||
reTags := regexp.MustCompile(`<[^>]*>`)
|
||||
s = reTags.ReplaceAllString(s, "")
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package uber
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/hireme/hireme/internal/crawler/crawlertest"
|
||||
)
|
||||
|
||||
type roundTripFunc func(req *http.Request) *http.Response
|
||||
|
||||
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return f(req), nil
|
||||
}
|
||||
|
||||
func TestUberCrawler_DiscoverAndFetch(t *testing.T) {
|
||||
mockClient := &http.Client{
|
||||
Transport: roundTripFunc(func(r *http.Request) *http.Response {
|
||||
if strings.Contains(r.URL.Path, "loadSearchJobsResults") {
|
||||
body := `{
|
||||
"status": "success",
|
||||
"data": {
|
||||
"totalResults": 1,
|
||||
"results": [
|
||||
{
|
||||
"id": 139820,
|
||||
"title": "Software Engineer II - Core Compute & Storage",
|
||||
"location": "Seattle, WA",
|
||||
"city": "Seattle",
|
||||
"countryName": "USA"
|
||||
}
|
||||
]
|
||||
}
|
||||
}`
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Body: io.NopCloser(bytes.NewBufferString(body)),
|
||||
Header: make(http.Header),
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Contains(r.URL.Path, "loadJobDetails") {
|
||||
body := `{
|
||||
"data": {
|
||||
"id": 139820,
|
||||
"title": "Software Engineer II - Core Compute & Storage",
|
||||
"description": "<p>Uber Infra powers global dispatch and real-time computation.</p>",
|
||||
"whatYouWillDo": "<li>Architect scalable storage solutions using Go and Java.</li>",
|
||||
"basicQualifications": "<li>2+ years of backend software development experience.</li>",
|
||||
"location": "Seattle, WA"
|
||||
}
|
||||
}`
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Body: io.NopCloser(bytes.NewBufferString(body)),
|
||||
Header: make(http.Header),
|
||||
}
|
||||
}
|
||||
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusNotFound,
|
||||
Body: io.NopCloser(bytes.NewBufferString("not found")),
|
||||
Header: make(http.Header),
|
||||
}
|
||||
}),
|
||||
}
|
||||
|
||||
c := NewUberCrawlerWithClient(mockClient)
|
||||
if !c.CanHandle("uber") {
|
||||
t.Errorf("CanHandle should support uber")
|
||||
}
|
||||
|
||||
discovered, err := c.DiscoverJobs(context.Background(), "https://www.uber.com/global/en/careers/search/?query=Software+Engineer")
|
||||
if err != nil {
|
||||
t.Fatalf("DiscoverJobs failed: %v", err)
|
||||
}
|
||||
|
||||
if len(discovered) != 1 {
|
||||
t.Fatalf("expected 1 discovered job, got %d", len(discovered))
|
||||
}
|
||||
|
||||
if discovered[0].CanonicalID != "uber:139820" {
|
||||
t.Errorf("unexpected canonical ID: %s", discovered[0].CanonicalID)
|
||||
}
|
||||
|
||||
extracted, err := c.FetchJobDetail(context.Background(), "https://www.uber.com/global/en/careers/list/139820/")
|
||||
if err != nil {
|
||||
t.Fatalf("FetchJobDetail failed: %v", err)
|
||||
}
|
||||
|
||||
if extracted.Title != "Software Engineer II - Core Compute & Storage" {
|
||||
t.Errorf("unexpected title: %s", extracted.Title)
|
||||
}
|
||||
if extracted.Company != "Uber" {
|
||||
t.Errorf("unexpected company: %s", extracted.Company)
|
||||
}
|
||||
if extracted.Location != "Seattle, WA" {
|
||||
t.Errorf("unexpected location: %s", extracted.Location)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUberCrawler_LiveIntegration(t *testing.T) {
|
||||
crawlertest.RunLiveIntegration(t, crawlertest.LiveTestConfig{
|
||||
PlatformName: "Uber",
|
||||
Crawler: NewUberCrawler(),
|
||||
TargetURL: "https://www.uber.com/global/en/careers/search/?query=Software+Engineer",
|
||||
ExpectedCanonicalPrefix: "uber:",
|
||||
ExpectedCompany: "Uber",
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user