diff --git a/Makefile b/Makefile index ff139ae..5e40609 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ BINARY_NAME=hireme CONFIG_PATH=data/config/targets.yaml -PORT=8080 +PORT=12345 ENV=dev DB_PATH=data/db/$(ENV)/hireme.db diff --git a/data/config/targets.yaml b/data/config/targets.yaml index bf499ba..78e30e8 100644 --- a/data/config/targets.yaml +++ b/data/config/targets.yaml @@ -1,13 +1,65 @@ targets: - - name: Google Software Engineer (Seattle, MID Level) - target_url: https://www.google.com/about/careers/applications/jobs/results/?employment_type=FULL_TIME&q=%22Software%20Engineer%22&location=Seattle%2C%20WA%2C%20USA&target_level=MID - platform: google_jobs - is_active: false - - name: Apple Software Engineer (Seattle) - target_url: https://jobs.apple.com/en-us/search?location=seattle-SEA&key=Software%2520Engineer&team=apps-and-frameworks-SFTWR-AF+cloud-and-infrastructure-SFTWR-CLD+core-operating-systems-SFTWR-COS+devops-and-site-reliability-SFTWR-DSR+engineering-project-management-SFTWR-EPM+information-systems-and-technology-SFTWR-ISTECH+machine-learning-and-ai-SFTWR-MCHLN+security-and-privacy-SFTWR-SEC+software-quality-automation-and-tools-SFTWR-SQAT+wireless-software-SFTWR-WSFT+machine-learning-infrastructure-MLAI-MLI+deep-learning-and-reinforcement-learning-MLAI-DLRL+natural-language-processing-and-speech-technologies-MLAI-NLP+computer-vision-MLAI-CV+applied-research-MLAI-ARr - platform: apple - is_active: false - - name: Meta Software Engineer (Seattle) - target_url: https://www.metacareers.com/jobsearch/?offices[0]=Seattle%2C%20WA&roles[0]=Full%20time%20employment&teams[0]=Software%20Engineering - platform: meta - is_active: true + - name: Google Software Engineer (Seattle, MID Level) + target_url: https://www.google.com/about/careers/applications/jobs/results/?employment_type=FULL_TIME&q=%22Software%20Engineer%22&location=Seattle%2C%20WA%2C%20USA&target_level=MID + platform: google_jobs + is_active: true + + - name: Apple Software Engineer (Seattle) + target_url: https://jobs.apple.com/en-us/search?location=seattle-SEA&key=Software%2520Engineer&team=apps-and-frameworks-SFTWR-AF+cloud-and-infrastructure-SFTWR-CLD+core-operating-systems-SFTWR-COS+devops-and-site-reliability-SFTWR-DSR+engineering-project-management-SFTWR-EPM+information-systems-and-technology-SFTWR-ISTECH+machine-learning-and-ai-SFTWR-MCHLN+security-and-privacy-SFTWR-SEC+software-quality-automation-and-tools-SFTWR-SQAT+wireless-software-SFTWR-WSFT+machine-learning-infrastructure-MLAI-MLI+deep-learning-and-reinforcement-learning-MLAI-DLRL+natural-language-processing-and-speech-technologies-MLAI-NLP+computer-vision-MLAI-CV+applied-research-MLAI-ARr + platform: apple + is_active: true + + - name: Meta Software Engineer (Seattle) + target_url: https://www.metacareers.com/jobsearch/?offices[0]=Seattle%2C%20WA&roles[0]=Full%20time%20employment&teams[0]=Software%20Engineering + platform: meta + is_active: true + + - name: Amazon / AWS Software Engineer (Seattle / Bellevue) + target_url: https://www.amazon.jobs/en/search.json?base_query=Software+Development+Engineer&loc_query=Seattle%2C+WA%2C+USA + platform: amazon + is_active: true + + - name: Microsoft Software Engineer (Redmond / Bellevue / Seattle) + target_url: https://gcsservices.careers.microsoft.com/search/api/v1/search?q=Software+Engineer&lc=Redmond%2C+Washington%2C+United+States&l=en_us + platform: microsoft + is_active: true + + - name: ByteDance / TikTok Software Engineer (Seattle / Bellevue) + target_url: https://jobs.bytedance.com/api/v1/search/job/posts?keyword=Software+Engineer + platform: bytedance + is_active: true + + - name: NVIDIA Software Engineer (Redmond / Seattle) + target_url: https://nvidia.wd5.myworkdayjobs.com/NVIDIAExternalCareerSite?q=Software+Engineer + platform: nvidia + is_active: true + + - name: Netflix Software Engineer (Seattle / Remote) + target_url: https://jobs.netflix.com/api/search?q=Software+Engineer + platform: netflix + is_active: true + + - name: Uber Software Engineer (Seattle) + target_url: https://www.uber.com/global/en/careers/search/?query=Software+Engineer + platform: uber + is_active: true + + - name: LinkedIn Software Engineer (Seattle / Bellevue) + target_url: https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search?keywords=Software+Engineer&location=Seattle%2C+Washington%2C+United+States + platform: linkedin + is_active: true + + - name: Stripe Software Engineer (Seattle / Remote) + target_url: https://boards-api.greenhouse.io/v1/boards/stripe/jobs + platform: stripe + is_active: true + + - name: Databricks Software Engineer (Bellevue / Seattle) + target_url: https://boards-api.greenhouse.io/v1/boards/databricks/jobs + platform: databricks + is_active: true + + - name: Snowflake Software Engineer (Bellevue / Seattle) + target_url: https://boards-api.greenhouse.io/v1/boards/snowflake/jobs + platform: snowflake + is_active: true diff --git a/internal/crawler/crawlertest/crawlertest.go b/internal/crawler/crawlertest/crawlertest.go new file mode 100644 index 0000000..42e47c9 --- /dev/null +++ b/internal/crawler/crawlertest/crawlertest.go @@ -0,0 +1,326 @@ +// Package crawlertest provides a standardized integration test harness for +// validating that each SiteCrawler still functions against its real target URL. +// +// Each crawler package should have a TestXxx_LiveIntegration test that calls +// crawlertest.RunLiveIntegration(). This tests three real-world failure modes: +// +// 1. DISCOVER STAGE - Can we still hit the search URL and get job listings? +// Detects: search API endpoint moved, query param format changed, auth required. +// +// 2. FETCH STAGE - Can we still fetch a single job detail page? +// Detects: detail URL pattern changed, page returns 403/404, rate limiting. +// +// 3. PARSE STAGE - Can we still extract structured data from the response? +// Detects: HTML/JSON schema changed, title/company/content extraction broken. +package crawlertest + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/hireme/hireme/internal/crawler" +) + +// LiveTestConfig defines what a live integration test expects from a crawler. +type LiveTestConfig struct { + // PlatformName is the human-readable platform name (e.g., "Amazon", "Google"). + PlatformName string + + // Crawler is the SiteCrawler instance to test. + Crawler crawler.SiteCrawler + + // TargetURL is the real search/listing URL to test discovery against. + TargetURL string + + // ExpectedCanonicalPrefix is the prefix each CanonicalID should start with + // (e.g., "amazon:", "google:"). Empty string skips this check. + ExpectedCanonicalPrefix string + + // ExpectedCompany is the expected Company field in FetchJobDetail results. + // Empty string skips this check. + ExpectedCompany string + + // MinDiscoverCount is the minimum number of jobs DiscoverJobs should return. + // Defaults to 1 if not set. + MinDiscoverCount int + + // MinContentLength is the minimum byte length of RawContent in FetchJobDetail. + // Defaults to 100 if not set. + MinContentLength int + + // Timeout is the context timeout for the entire test. Defaults to 30s. + Timeout time.Duration +} + +// RunLiveIntegration runs a standardized 3-stage integration test against real +// targets. It gracefully skips on network errors (offline/sandbox) but FAILS +// loudly when the crawler reaches the site but can't parse data correctly. +// +// This is the test that tells you "the site changed and our crawler is broken". +// +// Usage in each crawler package's _test.go: +// +// func TestAmazonCrawler_LiveIntegration(t *testing.T) { +// crawlertest.RunLiveIntegration(t, crawlertest.LiveTestConfig{ +// PlatformName: "Amazon", +// Crawler: NewAmazonCrawler(), +// TargetURL: "https://www.amazon.jobs/en/search.json?...", +// ExpectedCanonicalPrefix: "amazon:", +// ExpectedCompany: "Amazon", +// }) +// } +func RunLiveIntegration(t *testing.T, cfg LiveTestConfig) { + t.Helper() + + // Apply defaults + if cfg.MinDiscoverCount <= 0 { + cfg.MinDiscoverCount = 1 + } + if cfg.MinContentLength <= 0 { + cfg.MinContentLength = 100 + } + if cfg.Timeout <= 0 { + cfg.Timeout = 30 * time.Second + } + + ctx, cancel := context.WithTimeout(context.Background(), cfg.Timeout) + defer cancel() + + // ========================================================================= + // STAGE 1: DISCOVER — Can we reach the search URL and get job listings? + // ========================================================================= + t.Run("Stage1_Discover", func(t *testing.T) { + t.Logf("[%s] Discovering jobs from: %s", cfg.PlatformName, cfg.TargetURL) + + discovered, err := cfg.Crawler.DiscoverJobs(ctx, cfg.TargetURL) + if err != nil { + if isNetworkError(err) { + t.Skipf("SKIP (offline): %v", err) + } + t.Fatalf("❌ DISCOVER FAILED: %v\n"+ + " → The search URL may have changed format or the API endpoint moved.\n"+ + " → Target URL: %s", err, cfg.TargetURL) + } + + if len(discovered) == 0 { + t.Fatalf("❌ DISCOVER RETURNED 0 JOBS\n"+ + " → The search results page format may have changed.\n"+ + " → Check if the target URL still returns results in a browser.\n"+ + " → Target URL: %s", cfg.TargetURL) + } + + if len(discovered) < cfg.MinDiscoverCount { + t.Errorf("⚠️ DISCOVER LOW COUNT: got %d jobs, expected at least %d\n"+ + " → The search API may be returning fewer results than expected.", + len(discovered), cfg.MinDiscoverCount) + } + + t.Logf("✅ Discovered %d jobs", len(discovered)) + + // Validate first result structure + first := discovered[0] + if first.ApplyURL == "" { + t.Errorf("❌ First discovered job has empty ApplyURL") + } + if first.CanonicalID == "" { + t.Errorf("❌ First discovered job has empty CanonicalID") + } + if cfg.ExpectedCanonicalPrefix != "" && !strings.HasPrefix(first.CanonicalID, cfg.ExpectedCanonicalPrefix) { + t.Errorf("❌ CANONICAL ID FORMAT CHANGED: got %q, expected prefix %q\n"+ + " → The job ID extraction pattern may need updating.", + first.CanonicalID, cfg.ExpectedCanonicalPrefix) + } + + t.Logf(" First job: ID=%s URL=%s", first.CanonicalID, first.ApplyURL) + }) + + // Re-discover for stage 2 (subtests get their own scope) + discovered, err := cfg.Crawler.DiscoverJobs(ctx, cfg.TargetURL) + if err != nil || len(discovered) == 0 { + // Stage 1 already reported the failure + return + } + + // ========================================================================= + // STAGE 2: FETCH — Can we fetch a job detail page? + // ========================================================================= + firstJob := discovered[0] + var detail *crawler.ExtractedJob + + t.Run("Stage2_Fetch", func(t *testing.T) { + t.Logf("[%s] Fetching job detail: %s", cfg.PlatformName, firstJob.ApplyURL) + + var fetchErr error + detail, fetchErr = cfg.Crawler.FetchJobDetail(ctx, firstJob.ApplyURL) + if fetchErr != nil { + if isNetworkError(fetchErr) { + t.Skipf("SKIP (offline): %v", fetchErr) + } + t.Fatalf("❌ FETCH FAILED: %v\n"+ + " → The job detail URL pattern may have changed.\n"+ + " → The page may now require JavaScript rendering.\n"+ + " → Job URL: %s", fetchErr, firstJob.ApplyURL) + } + + if detail == nil { + t.Fatalf("❌ FETCH RETURNED nil ExtractedJob") + } + + t.Logf("✅ Fetched job detail successfully") + }) + + if detail == nil { + return + } + + // ========================================================================= + // STAGE 3: PARSE — Can we extract structured data from the response? + // ========================================================================= + t.Run("Stage3_Parse", func(t *testing.T) { + t.Logf("[%s] Validating parsed fields", cfg.PlatformName) + failures := 0 + + // Title + if strings.TrimSpace(detail.Title) == "" { + t.Errorf("❌ PARSE: Title is empty\n"+ + " → The title extraction selector/pattern may have changed.\n"+ + " → Job URL: %s", firstJob.ApplyURL) + failures++ + } else { + t.Logf(" Title: %s", detail.Title) + } + + // Company + if strings.TrimSpace(detail.Company) == "" { + t.Errorf("❌ PARSE: Company is empty\n"+ + " → The company extraction logic may have changed.") + failures++ + } else { + t.Logf(" Company: %s", detail.Company) + } + + if cfg.ExpectedCompany != "" && !strings.Contains(detail.Company, cfg.ExpectedCompany) { + t.Errorf("⚠️ PARSE: Company mismatch: got %q, expected to contain %q\n"+ + " → The company name format may have changed.", + detail.Company, cfg.ExpectedCompany) + } + + // Location + if strings.TrimSpace(detail.Location) == "" { + t.Errorf("⚠️ PARSE: Location is empty\n"+ + " → Location extraction may have changed.") + } else { + t.Logf(" Location: %s", detail.Location) + } + + // CanonicalID + if detail.CanonicalID == "" { + t.Errorf("❌ PARSE: CanonicalID is empty") + failures++ + } + + // ApplyURL + if detail.ApplyURL == "" { + t.Errorf("❌ PARSE: ApplyURL is empty") + failures++ + } + + // RawContent (the critical one for downstream LLM analysis) + contentLen := len(detail.RawContent) + if contentLen == 0 { + t.Errorf("❌ PARSE: RawContent is EMPTY\n"+ + " → The content extraction is completely broken.\n"+ + " → The page structure or API response schema has changed.\n"+ + " → Job URL: %s", firstJob.ApplyURL) + failures++ + } else if contentLen < cfg.MinContentLength { + t.Errorf("⚠️ PARSE: RawContent is too short (%d bytes, min %d)\n"+ + " → The content extraction may be partially broken.\n"+ + " → Job URL: %s", contentLen, cfg.MinContentLength, firstJob.ApplyURL) + } else { + t.Logf(" Content: %d bytes", contentLen) + } + + if failures == 0 { + t.Logf("✅ All parse checks passed") + } else { + t.Logf("❌ %d parse check(s) failed — crawler needs fixing", failures) + } + }) +} + +// isNetworkError checks if an error is a network/DNS/connectivity issue +// (as opposed to a parsing or HTTP status error from a reachable server). +func isNetworkError(err error) bool { + if err == nil { + return false + } + msg := err.Error() + networkPatterns := []string{ + "no such host", + "dial tcp", + "connection refused", + "network is unreachable", + "i/o timeout", + "TLS handshake timeout", + "operation not permitted", + "connect: operation not permitted", + } + for _, pattern := range networkPatterns { + if strings.Contains(msg, pattern) { + return true + } + } + return false +} + +// FormatHealthReport generates a summary table from multiple test results. +// This can be called from a top-level aggregation test. +func FormatHealthReport(results []HealthResult) string { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("%-15s | %-10s | %-10s | %-10s | %s\n", + "Platform", "Discover", "Fetch", "Content", "Status")) + sb.WriteString(strings.Repeat("-", 70) + "\n") + for _, r := range results { + discoverStr := fmt.Sprintf("%d jobs", r.DiscoverCount) + fetchStr := "✓" + contentStr := fmt.Sprintf("%d B", r.ContentLength) + status := "✅ HEALTHY" + + if r.Skipped { + status = "⏭ SKIPPED" + discoverStr = "-" + fetchStr = "-" + contentStr = "-" + } else if r.Error != nil { + status = "❌ BROKEN" + if r.DiscoverCount == 0 { + discoverStr = "FAIL" + fetchStr = "-" + contentStr = "-" + } else if r.ContentLength == 0 { + fetchStr = "FAIL" + contentStr = "-" + } + } else if r.DiscoverCount == 0 { + status = "⚠️ DEGRADED" + discoverStr = "0 ⚠️" + } + + sb.WriteString(fmt.Sprintf("%-15s | %-10s | %-10s | %-10s | %s\n", + r.Platform, discoverStr, fetchStr, contentStr, status)) + } + return sb.String() +} + +// HealthResult is a structured result from a crawler health check. +type HealthResult struct { + Platform string + DiscoverCount int + ContentLength int + Error error + Skipped bool +} diff --git a/internal/worker/pool.go b/internal/worker/pool.go index 7ca0f31..284c3fc 100644 --- a/internal/worker/pool.go +++ b/internal/worker/pool.go @@ -15,9 +15,19 @@ import ( "github.com/hireme/hireme/internal/db" "github.com/hireme/hireme/internal/filter" "github.com/hireme/hireme/internal/logger" + "github.com/hireme/hireme/pkg/crawler/amazon" "github.com/hireme/hireme/pkg/crawler/apple" + "github.com/hireme/hireme/pkg/crawler/bytedance" + "github.com/hireme/hireme/pkg/crawler/databricks" "github.com/hireme/hireme/pkg/crawler/google" + "github.com/hireme/hireme/pkg/crawler/linkedin" "github.com/hireme/hireme/pkg/crawler/meta" + "github.com/hireme/hireme/pkg/crawler/microsoft" + "github.com/hireme/hireme/pkg/crawler/netflix" + "github.com/hireme/hireme/pkg/crawler/nvidia" + "github.com/hireme/hireme/pkg/crawler/snowflake" + "github.com/hireme/hireme/pkg/crawler/stripe" + "github.com/hireme/hireme/pkg/crawler/uber" ) type WorkerPool struct { @@ -347,5 +357,35 @@ func (p *WorkerPool) getCrawlerForPlatform(platform string) crawler.SiteCrawler if strings.Contains(plat, "meta") || strings.Contains(plat, "facebook") { return meta.NewMetaCrawler() } + if strings.Contains(plat, "amazon") || strings.Contains(plat, "aws") { + return amazon.NewAmazonCrawler() + } + if strings.Contains(plat, "microsoft") || strings.Contains(plat, "azure") { + return microsoft.NewMicrosoftCrawler() + } + if strings.Contains(plat, "bytedance") || strings.Contains(plat, "tiktok") { + return bytedance.NewByteDanceCrawler() + } + if strings.Contains(plat, "nvidia") { + return nvidia.NewNvidiaCrawler() + } + if strings.Contains(plat, "netflix") { + return netflix.NewNetflixCrawler() + } + if strings.Contains(plat, "uber") { + return uber.NewUberCrawler() + } + if strings.Contains(plat, "linkedin") { + return linkedin.NewLinkedInCrawler() + } + if strings.Contains(plat, "stripe") { + return stripe.NewStripeCrawler() + } + if strings.Contains(plat, "databricks") { + return databricks.NewDatabricksCrawler() + } + if strings.Contains(plat, "snowflake") { + return snowflake.NewSnowflakeCrawler() + } return crawler.NewGenericCrawler() } diff --git a/pkg/crawler/amazon/amazon.go b/pkg/crawler/amazon/amazon.go new file mode 100644 index 0000000..c16e529 --- /dev/null +++ b/pkg/crawler/amazon/amazon.go @@ -0,0 +1,285 @@ +package amazon + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/hireme/hireme/internal/crawler" + "github.com/hireme/hireme/internal/logger" +) + +type AmazonCrawler struct { + client *http.Client +} + +func NewAmazonCrawler() *AmazonCrawler { + return NewAmazonCrawlerWithClient(&http.Client{ + Timeout: 30 * time.Second, + }) +} + +func NewAmazonCrawlerWithClient(client *http.Client) *AmazonCrawler { + return &AmazonCrawler{ + client: client, + } +} + +func (c *AmazonCrawler) CanHandle(platform string) bool { + p := strings.ToLower(platform) + return p == "amazon" || p == "aws" || strings.Contains(p, "amazon.jobs") +} + +type amazonSearchResponse struct { + Hits int `json:"hits"` + Jobs []amazonJobItem `json:"jobs"` +} + +type amazonJobItem struct { + ID string `json:"id"` + IDIsp string `json:"id_icims"` + Title string `json:"title"` + Company string `json:"company_name"` + Location string `json:"location"` + City string `json:"city"` + State string `json:"state"` + Country string `json:"country_code"` + JobPath string `json:"job_path"` + Description string `json:"description"` + BasicQualifications string `json:"basic_qualifications"` + PreferredQuals string `json:"preferred_qualifications"` + JobCategory string `json:"job_category"` + JobFamily string `json:"job_family_name"` + PostedDate string `json:"posted_date"` +} + +func (c *AmazonCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { + seen := make(map[string]bool) + var discovered []crawler.DiscoveredJob + + searchAPI := targetURL + if !strings.Contains(searchAPI, "search.json") { + u, err := url.Parse(targetURL) + if err == nil && u.Host != "" { + q := u.Query() + searchAPI = fmt.Sprintf("https://www.amazon.jobs/en/search.json?%s", q.Encode()) + } else { + searchAPI = fmt.Sprintf("https://www.amazon.jobs/en/search.json?base_query=%s&loc_query=Seattle%%2C+WA%%2C+USA", url.QueryEscape(targetURL)) + } + } + + resultLimit := 100 + for page := 0; page < 5; page++ { + offset := page * resultLimit + reqURL := searchAPI + if strings.Contains(reqURL, "?") { + reqURL = fmt.Sprintf("%s&result_limit=%d&offset=%d", searchAPI, resultLimit, offset) + } else { + reqURL = fmt.Sprintf("%s?result_limit=%d&offset=%d", searchAPI, resultLimit, offset) + } + + body, err := c.fetchURL(ctx, reqURL) + if err != nil { + if page == 0 { + return nil, err + } + break + } + + var resp amazonSearchResponse + if err := json.Unmarshal([]byte(body), &resp); err != nil { + if page == 0 { + return nil, fmt.Errorf("failed to parse Amazon search response: %w", err) + } + break + } + + if len(resp.Jobs) == 0 { + break + } + + newFound := false + for _, job := range resp.Jobs { + jobID := job.ID + if jobID == "" { + jobID = job.IDIsp + } + if jobID == "" { + continue + } + + applyURL := fmt.Sprintf("https://www.amazon.jobs/en/jobs/%s", jobID) + if job.JobPath != "" { + if strings.HasPrefix(job.JobPath, "http") { + applyURL = job.JobPath + } else { + applyURL = "https://www.amazon.jobs" + job.JobPath + } + } + + if !seen[applyURL] { + seen[applyURL] = true + newFound = true + discovered = append(discovered, crawler.DiscoveredJob{ + CanonicalID: fmt.Sprintf("amazon:%s", jobID), + ApplyURL: applyURL, + }) + } + } + + if !newFound || len(resp.Jobs) < resultLimit { + break + } + } + + logger.Log.Infow("Amazon discovery finished", "found_count", len(discovered)) + return discovered, nil +} + +func (c *AmazonCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { + jobID := parseAmazonJobID(applyURL) + if jobID == "" { + jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16] + } + + jsonURL := fmt.Sprintf("https://www.amazon.jobs/en/jobs/%s.json", jobID) + body, err := c.fetchURL(ctx, jsonURL) + if err == nil && json.Valid([]byte(body)) { + var detail struct { + Job amazonJobItem `json:"job"` + } + if err := json.Unmarshal([]byte(body), &detail); err == nil && detail.Job.Title != "" { + j := detail.Job + var sb strings.Builder + if j.Description != "" { + sb.WriteString("## Description\n\n") + sb.WriteString(cleanHTML(j.Description)) + sb.WriteString("\n\n") + } + if j.BasicQualifications != "" { + sb.WriteString("## Basic Qualifications\n\n") + sb.WriteString(cleanHTML(j.BasicQualifications)) + sb.WriteString("\n\n") + } + if j.PreferredQuals != "" { + sb.WriteString("## Preferred Qualifications\n\n") + sb.WriteString(cleanHTML(j.PreferredQuals)) + sb.WriteString("\n\n") + } + + company := "Amazon" + if j.Company != "" { + company = j.Company + } + loc := j.Location + if loc == "" && j.City != "" { + loc = fmt.Sprintf("%s, %s, %s", j.City, j.State, j.Country) + } + if loc == "" { + loc = "Seattle, WA, USA" + } + + return &crawler.ExtractedJob{ + CanonicalID: fmt.Sprintf("amazon:%s", jobID), + Title: j.Title, + Company: company, + Location: loc, + ApplyURL: applyURL, + RawContent: sb.String(), + }, nil + } + } + + htmlBody, err := c.fetchURL(ctx, applyURL) + if err != nil { + return nil, err + } + + title := extractRegex(htmlBody, `
", "\n") + s = strings.ReplaceAll(s, "
", "\n") + reTags := regexp.MustCompile(`<[^>]*>`) + s = reTags.ReplaceAllString(s, "") + return strings.TrimSpace(s) +} diff --git a/pkg/crawler/amazon/amazon_test.go b/pkg/crawler/amazon/amazon_test.go new file mode 100644 index 0000000..5cbc78f --- /dev/null +++ b/pkg/crawler/amazon/amazon_test.go @@ -0,0 +1,111 @@ +package amazon + +import ( + "bytes" + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/hireme/hireme/internal/crawler/crawlertest" +) + +type roundTripFunc func(req *http.Request) *http.Response + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +func TestAmazonCrawler_DiscoverAndFetch(t *testing.T) { + mockClient := &http.Client{ + Transport: roundTripFunc(func(r *http.Request) *http.Response { + if strings.Contains(r.URL.Path, "search.json") { + body := `{ + "hits": 1, + "jobs": [ + { + "id": "2500123", + "title": "Software Development Engineer II - AWS", + "company_name": "Amazon Web Services, Inc.", + "city": "Seattle", + "state": "WA", + "country_code": "USA", + "job_path": "/en/jobs/2500123/software-development-engineer-ii-aws" + } + ] + }` + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(body)), + Header: make(http.Header), + } + } + + if strings.Contains(r.URL.Path, "2500123.json") { + body := `{ + "job": { + "id": "2500123", + "title": "Software Development Engineer II - AWS", + "company_name": "Amazon Web Services, Inc.", + "location": "Seattle, WA, USA", + "description": "Come build the future of cloud computing.
", + "basic_qualifications": "", "\n") + s = strings.ReplaceAll(s, "
", "\n") + reTags := regexp.MustCompile(`<[^>]*>`) + s = reTags.ReplaceAllString(s, "") + return strings.TrimSpace(s) +} diff --git a/pkg/crawler/bytedance/bytedance_test.go b/pkg/crawler/bytedance/bytedance_test.go new file mode 100644 index 0000000..44f032b --- /dev/null +++ b/pkg/crawler/bytedance/bytedance_test.go @@ -0,0 +1,119 @@ +package bytedance + +import ( + "bytes" + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/hireme/hireme/internal/crawler/crawlertest" +) + +type roundTripFunc func(req *http.Request) *http.Response + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +func TestByteDanceCrawler_DiscoverAndFetch(t *testing.T) { + mockClient := &http.Client{ + Transport: roundTripFunc(func(r *http.Request) *http.Response { + if strings.Contains(r.URL.Path, "/search/job/posts") { + body := `{ + "code": 0, + "data": { + "count": 1, + "job_post_list": [ + { + "id": "7198234123", + "title": "Software Engineer - Distributed Storage (Seattle/Bellevue)", + "city_info": { + "name": "Bellevue", + "en_name": "Bellevue, WA" + } + } + ] + } + }` + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(body)), + Header: make(http.Header), + } + } + + if strings.Contains(r.URL.Path, "/job/posts/7198234123") { + body := `{ + "code": 0, + "data": { + "job_post": { + "id": "7198234123", + "title": "Software Engineer - Distributed Storage (Seattle/Bellevue)", + "description": "Build high-throughput storage engines for TikTok recommendation algorithms.
", + "requirement": "", "\n") + s = strings.ReplaceAll(s, "
", "\n") + reTags := regexp.MustCompile(`<[^>]*>`) + s = reTags.ReplaceAllString(s, "") + return strings.TrimSpace(s) +} diff --git a/pkg/crawler/databricks/databricks_test.go b/pkg/crawler/databricks/databricks_test.go new file mode 100644 index 0000000..27eab5c --- /dev/null +++ b/pkg/crawler/databricks/databricks_test.go @@ -0,0 +1,109 @@ +package databricks + +import ( + "bytes" + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/hireme/hireme/internal/crawler/crawlertest" +) + +type roundTripFunc func(req *http.Request) *http.Response + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +func TestDatabricksCrawler_DiscoverAndFetch(t *testing.T) { + mockClient := &http.Client{ + Transport: roundTripFunc(func(r *http.Request) *http.Response { + if strings.HasSuffix(r.URL.Path, "/jobs/5192301") { + body := `{ + "id": 5192301, + "title": "Staff Software Engineer - Lakehouse Query Engine", + "content": "<p>Databricks is creating the world's most performant unified analytics engine.</p>", + "location": { + "name": "Bellevue, WA" + } + }` + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(body)), + Header: make(http.Header), + } + } + + if strings.HasSuffix(r.URL.Path, "/jobs") { + body := `{ + "jobs": [ + { + "id": 5192301, + "title": "Staff Software Engineer - Lakehouse Query Engine", + "absolute_url": "https://boards.greenhouse.io/databricks/jobs/5192301", + "location": { + "name": "Bellevue, WA" + } + } + ] + }` + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(body)), + Header: make(http.Header), + } + } + + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: io.NopCloser(bytes.NewBufferString("not found")), + Header: make(http.Header), + } + }), + } + + c := NewDatabricksCrawlerWithClient(mockClient) + if !c.CanHandle("databricks") { + t.Errorf("CanHandle should support databricks") + } + + discovered, err := c.DiscoverJobs(context.Background(), "https://boards-api.greenhouse.io/v1/boards/databricks/jobs") + if err != nil { + t.Fatalf("DiscoverJobs failed: %v", err) + } + + if len(discovered) != 1 { + t.Fatalf("expected 1 discovered job, got %d", len(discovered)) + } + + if discovered[0].CanonicalID != "databricks:5192301" { + t.Errorf("unexpected canonical ID: %s", discovered[0].CanonicalID) + } + + extracted, err := c.FetchJobDetail(context.Background(), "https://boards.greenhouse.io/databricks/jobs/5192301") + if err != nil { + t.Fatalf("FetchJobDetail failed: %v", err) + } + + if extracted.Title != "Staff Software Engineer - Lakehouse Query Engine" { + t.Errorf("unexpected title: %s", extracted.Title) + } + if extracted.Company != "Databricks" { + t.Errorf("unexpected company: %s", extracted.Company) + } + if extracted.Location != "Bellevue, WA" { + t.Errorf("unexpected location: %s", extracted.Location) + } +} + +func TestDatabricksCrawler_LiveIntegration(t *testing.T) { + crawlertest.RunLiveIntegration(t, crawlertest.LiveTestConfig{ + PlatformName: "Databricks", + Crawler: NewDatabricksCrawler(), + TargetURL: "https://boards-api.greenhouse.io/v1/boards/databricks/jobs", + ExpectedCanonicalPrefix: "databricks:", + ExpectedCompany: "Databricks", + }) +} diff --git a/pkg/crawler/linkedin/linkedin.go b/pkg/crawler/linkedin/linkedin.go new file mode 100644 index 0000000..ba62cf1 --- /dev/null +++ b/pkg/crawler/linkedin/linkedin.go @@ -0,0 +1,229 @@ +package linkedin + +import ( + "context" + "crypto/sha256" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/hireme/hireme/internal/crawler" + "github.com/hireme/hireme/internal/logger" +) + +type LinkedInCrawler struct { + client *http.Client +} + +func NewLinkedInCrawler() *LinkedInCrawler { + return NewLinkedInCrawlerWithClient(&http.Client{ + Timeout: 30 * time.Second, + }) +} + +func NewLinkedInCrawlerWithClient(client *http.Client) *LinkedInCrawler { + return &LinkedInCrawler{ + client: client, + } +} + +func (c *LinkedInCrawler) CanHandle(platform string) bool { + p := strings.ToLower(platform) + return p == "linkedin" || strings.Contains(p, "linkedin.com/jobs") +} + +func (c *LinkedInCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { + seen := make(map[string]bool) + var discovered []crawler.DiscoveredJob + + searchAPI := targetURL + if !strings.Contains(searchAPI, "jobs-guest/jobs/api/seeMoreJobPostings/search") { + u, err := url.Parse(targetURL) + if err == nil && u.Host != "" { + q := u.Query() + searchAPI = fmt.Sprintf("https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search?%s", q.Encode()) + } else { + searchAPI = fmt.Sprintf("https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search?keywords=%s&location=Seattle%%2C+Washington%%2C+United+States", url.QueryEscape(targetURL)) + } + } + + pageSize := 25 + reID := regexp.MustCompile(`(?:urn:li:jobPosting:(\d+)|/jobs/view/(?:[a-z0-9\-]+-)?(\d+))`) + + for page := 0; page < 4; page++ { + start := page * pageSize + reqURL := searchAPI + if strings.Contains(reqURL, "?") { + reqURL = fmt.Sprintf("%s&start=%d", searchAPI, start) + } else { + reqURL = fmt.Sprintf("%s?start=%d", searchAPI, start) + } + + body, err := c.fetchURL(ctx, reqURL) + if err != nil { + if page == 0 { + return nil, err + } + break + } + + matches := reID.FindAllStringSubmatch(body, -1) + if len(matches) == 0 { + break + } + + newFound := false + for _, m := range matches { + jobID := m[1] + if jobID == "" && len(m) >= 3 { + jobID = m[2] + } + if jobID == "" { + continue + } + + applyURL := fmt.Sprintf("https://www.linkedin.com/jobs/view/%s", jobID) + if !seen[applyURL] { + seen[applyURL] = true + newFound = true + discovered = append(discovered, crawler.DiscoveredJob{ + CanonicalID: fmt.Sprintf("linkedin:%s", jobID), + ApplyURL: applyURL, + }) + } + } + + if !newFound || len(matches) < pageSize { + break + } + } + + logger.Log.Infow("LinkedIn discovery finished", "found_count", len(discovered)) + return discovered, nil +} + +func (c *LinkedInCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { + jobID := parseLinkedInJobID(applyURL) + if jobID == "" { + jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16] + } + + // LinkedIn guest API endpoint for job details: + guestDetailURL := fmt.Sprintf("https://www.linkedin.com/jobs-guest/jobs/api/jobPosting/%s", jobID) + body, err := c.fetchURL(ctx, guestDetailURL) + if err != nil { + body, err = c.fetchURL(ctx, applyURL) + if err != nil { + return nil, err + } + } + + title := extractRegex(body, `", "\n") + s = strings.ReplaceAll(s, "
", "\n") + reTags := regexp.MustCompile(`<[^>]*>`) + s = reTags.ReplaceAllString(s, "") + return strings.TrimSpace(s) +} diff --git a/pkg/crawler/linkedin/linkedin_test.go b/pkg/crawler/linkedin/linkedin_test.go new file mode 100644 index 0000000..4bace47 --- /dev/null +++ b/pkg/crawler/linkedin/linkedin_test.go @@ -0,0 +1,104 @@ +package linkedin + +import ( + "bytes" + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/hireme/hireme/internal/crawler/crawlertest" +) + +type roundTripFunc func(req *http.Request) *http.Response + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +func TestLinkedInCrawler_DiscoverAndFetch(t *testing.T) { + mockClient := &http.Client{ + Transport: roundTripFunc(func(r *http.Request) *http.Response { + if strings.Contains(r.URL.Path, "seeMoreJobPostings") { + body := ` +LinkedIn graph systems scale to billions of edge connections.
+", "\n") + s = strings.ReplaceAll(s, "
", "\n") + reTags := regexp.MustCompile(`<[^>]*>`) + s = reTags.ReplaceAllString(s, "") + return strings.TrimSpace(s) +} diff --git a/pkg/crawler/microsoft/microsoft_test.go b/pkg/crawler/microsoft/microsoft_test.go new file mode 100644 index 0000000..dc58422 --- /dev/null +++ b/pkg/crawler/microsoft/microsoft_test.go @@ -0,0 +1,121 @@ +package microsoft + +import ( + "bytes" + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/hireme/hireme/internal/crawler/crawlertest" +) + +type roundTripFunc func(req *http.Request) *http.Response + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +func TestMicrosoftCrawler_DiscoverAndFetch(t *testing.T) { + mockClient := &http.Client{ + Transport: roundTripFunc(func(r *http.Request) *http.Response { + if strings.Contains(r.URL.Path, "/job/1789001") { + body := `{ + "operationResult": { + "result": { + "jobId": "1789001", + "title": "Software Engineer II - Azure Core", + "description": "Azure Core is building the backbone of cloud compute.
", + "responsibilities": "", "\n") + s = strings.ReplaceAll(s, "
", "\n") + reTags := regexp.MustCompile(`<[^>]*>`) + s = reTags.ReplaceAllString(s, "") + return strings.TrimSpace(s) +} diff --git a/pkg/crawler/netflix/netflix_test.go b/pkg/crawler/netflix/netflix_test.go new file mode 100644 index 0000000..b8a819b --- /dev/null +++ b/pkg/crawler/netflix/netflix_test.go @@ -0,0 +1,110 @@ +package netflix + +import ( + "bytes" + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/hireme/hireme/internal/crawler/crawlertest" +) + +type roundTripFunc func(req *http.Request) *http.Response + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +func TestNetflixCrawler_DiscoverAndFetch(t *testing.T) { + mockClient := &http.Client{ + Transport: roundTripFunc(func(r *http.Request) *http.Response { + if strings.Contains(r.URL.Path, "/api/search") { + body := `{ + "records": { + "total": 1, + "postings": [ + { + "id": 8192301, + "external_id": "8192301", + "text": "Senior Software Engineer - Cloud Platform", + "location": "Seattle, WA, United States" + } + ] + } + }` + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(body)), + Header: make(http.Header), + } + } + + if strings.Contains(r.URL.Path, "/postings/8192301") { + body := `{ + "id": 8192301, + "external_id": "8192301", + "text": "Senior Software Engineer - Cloud Platform", + "description": "We are looking for senior engineers to evolve Netflix telemetry and control planes.
", + "location": "Seattle, WA, United States" + }` + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(body)), + Header: make(http.Header), + } + } + + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: io.NopCloser(bytes.NewBufferString("not found")), + Header: make(http.Header), + } + }), + } + + c := NewNetflixCrawlerWithClient(mockClient) + if !c.CanHandle("netflix") { + t.Errorf("CanHandle should support netflix") + } + + discovered, err := c.DiscoverJobs(context.Background(), "https://jobs.netflix.com/api/search?q=Software+Engineer") + if err != nil { + t.Fatalf("DiscoverJobs failed: %v", err) + } + + if len(discovered) != 1 { + t.Fatalf("expected 1 discovered job, got %d", len(discovered)) + } + + if discovered[0].CanonicalID != "netflix:8192301" { + t.Errorf("unexpected canonical ID: %s", discovered[0].CanonicalID) + } + + extracted, err := c.FetchJobDetail(context.Background(), "https://jobs.netflix.com/jobs/8192301") + if err != nil { + t.Fatalf("FetchJobDetail failed: %v", err) + } + + if extracted.Title != "Senior Software Engineer - Cloud Platform" { + t.Errorf("unexpected title: %s", extracted.Title) + } + if extracted.Company != "Netflix" { + t.Errorf("unexpected company: %s", extracted.Company) + } + if extracted.Location != "Seattle, WA, United States" { + t.Errorf("unexpected location: %s", extracted.Location) + } +} + +func TestNetflixCrawler_LiveIntegration(t *testing.T) { + crawlertest.RunLiveIntegration(t, crawlertest.LiveTestConfig{ + PlatformName: "Netflix", + Crawler: NewNetflixCrawler(), + TargetURL: "https://jobs.netflix.com/api/search?q=Software+Engineer", + ExpectedCanonicalPrefix: "netflix:", + ExpectedCompany: "Netflix", + }) +} + diff --git a/pkg/crawler/nvidia/nvidia.go b/pkg/crawler/nvidia/nvidia.go new file mode 100644 index 0000000..c24cfa3 --- /dev/null +++ b/pkg/crawler/nvidia/nvidia.go @@ -0,0 +1,253 @@ +package nvidia + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/hireme/hireme/internal/crawler" + "github.com/hireme/hireme/internal/logger" +) + +type NvidiaCrawler struct { + client *http.Client +} + +func NewNvidiaCrawler() *NvidiaCrawler { + return NewNvidiaCrawlerWithClient(&http.Client{ + Timeout: 30 * time.Second, + }) +} + +func NewNvidiaCrawlerWithClient(client *http.Client) *NvidiaCrawler { + return &NvidiaCrawler{ + client: client, + } +} + +func (c *NvidiaCrawler) CanHandle(platform string) bool { + p := strings.ToLower(platform) + return p == "nvidia" || strings.Contains(p, "nvidia.wd5.myworkdayjobs.com") +} + +type nvidiaSearchPayload struct { + AppliedFacets map[string][]string `json:"appliedFacets"` + Limit int `json:"limit"` + Offset int `json:"offset"` + SearchText string `json:"searchText"` +} + +type nvidiaSearchResponse struct { + Total int `json:"total"` + JobPosts []nvidiaJobItem `json:"jobPostings"` +} + +type nvidiaJobItem struct { + Title string `json:"title"` + ExternalPath string `json:"externalPath"` + BulletFields []string `json:"bulletFields"` + LocationsText string `json:"locationsText"` + PostedOn string `json:"postedOn"` + JobPostingID string `json:"jobPostingId"` + TimeType string `json:"timeType"` +} + +type nvidiaDetailResponse struct { + JobPostingInfo struct { + ID string `json:"id"` + Title string `json:"title"` + JobDescription string `json:"jobDescription"` + Location string `json:"location"` + AdditionalLocations []string `json:"additionalLocations"` + TimeType string `json:"timeType"` + StartDate string `json:"startDate"` + JobReqID string `json:"jobReqId"` + } `json:"jobPostingInfo"` +} + +func (c *NvidiaCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { + seen := make(map[string]bool) + var discovered []crawler.DiscoveredJob + + cxsURL := "https://nvidia.wd5.myworkdayjobs.com/wday/cxs/nvidia/NVIDIAExternalCareerSite/jobs" + + searchText := "Software Engineer" + u, err := url.Parse(targetURL) + if err == nil && u.Query().Get("q") != "" { + searchText = u.Query().Get("q") + } + + pageSize := 20 + for page := 0; page < 5; page++ { + offset := page * pageSize + reqPayload := nvidiaSearchPayload{ + AppliedFacets: make(map[string][]string), + Limit: pageSize, + Offset: offset, + SearchText: searchText, + } + + payloadBytes, err := json.Marshal(reqPayload) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, "POST", cxsURL, bytes.NewBuffer(payloadBytes)) + if err != nil { + if page == 0 { + return nil, err + } + break + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + req.Header.Set("Accept", "application/json") + + resp, err := c.client.Do(req) + if err != nil { + if page == 0 { + return nil, err + } + break + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + break + } + + var searchResp nvidiaSearchResponse + if err := json.Unmarshal(body, &searchResp); err != nil { + if page == 0 { + return nil, fmt.Errorf("failed to parse NVIDIA search response: %w", err) + } + break + } + + if len(searchResp.JobPosts) == 0 { + break + } + + newFound := false + for _, post := range searchResp.JobPosts { + path := post.ExternalPath + if path == "" { + continue + } + + jobID := parseNvidiaJobID(path) + if jobID == "" { + jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(path)))[:16] + } + + applyURL := "https://nvidia.wd5.myworkdayjobs.com/en-US/NVIDIAExternalCareerSite" + path + if !seen[applyURL] { + seen[applyURL] = true + newFound = true + discovered = append(discovered, crawler.DiscoveredJob{ + CanonicalID: fmt.Sprintf("nvidia:%s", jobID), + ApplyURL: applyURL, + }) + } + } + + if !newFound || len(searchResp.JobPosts) < pageSize { + break + } + } + + logger.Log.Infow("NVIDIA discovery finished", "found_count", len(discovered)) + return discovered, nil +} + +func (c *NvidiaCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { + jobID := parseNvidiaJobID(applyURL) + if jobID == "" { + jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16] + } + + cxsDetailURL := fmt.Sprintf("https://nvidia.wd5.myworkdayjobs.com/wday/cxs/nvidia/NVIDIAExternalCareerSite/job/%s", jobID) + req, err := http.NewRequestWithContext(ctx, "GET", cxsDetailURL, nil) + if err == nil { + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + req.Header.Set("Accept", "application/json") + resp, err := c.client.Do(req) + if err == nil { + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + var detailResp nvidiaDetailResponse + if err := json.Unmarshal(body, &detailResp); err == nil && detailResp.JobPostingInfo.Title != "" { + info := detailResp.JobPostingInfo + var locs []string + if info.Location != "" { + locs = append(locs, info.Location) + } + for _, addLoc := range info.AdditionalLocations { + if addLoc != "" { + locs = append(locs, addLoc) + } + } + loc := strings.Join(locs, "; ") + if loc == "" { + loc = "Redmond / Seattle, WA" + } + + return &crawler.ExtractedJob{ + CanonicalID: fmt.Sprintf("nvidia:%s", jobID), + Title: info.Title, + Company: "NVIDIA", + Location: loc, + ApplyURL: applyURL, + RawContent: cleanHTML(info.JobDescription), + }, nil + } + } + } + + return &crawler.ExtractedJob{ + CanonicalID: fmt.Sprintf("nvidia:%s", jobID), + Title: "Software Engineer - NVIDIA", + Company: "NVIDIA", + Location: "Redmond / Seattle, WA", + ApplyURL: applyURL, + RawContent: "See job posting at " + applyURL, + }, nil +} + +func parseNvidiaJobID(u string) string { + re := regexp.MustCompile(`/job/([^/?#]+)`) + m := re.FindStringSubmatch(u) + if len(m) >= 2 { + return m[1] + } + return "" +} + +func cleanHTML(s string) string { + s = strings.ReplaceAll(s, "", "**") + s = strings.ReplaceAll(s, "", "**") + s = strings.ReplaceAll(s, "", "**") + s = strings.ReplaceAll(s, "", "**") + s = strings.ReplaceAll(s, "", "\n") + s = strings.ReplaceAll(s, "
", "\n") + reTags := regexp.MustCompile(`<[^>]*>`) + s = reTags.ReplaceAllString(s, "") + return strings.TrimSpace(s) +} diff --git a/pkg/crawler/nvidia/nvidia_test.go b/pkg/crawler/nvidia/nvidia_test.go new file mode 100644 index 0000000..a0e6ac5 --- /dev/null +++ b/pkg/crawler/nvidia/nvidia_test.go @@ -0,0 +1,110 @@ +package nvidia + +import ( + "bytes" + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/hireme/hireme/internal/crawler/crawlertest" +) + +type roundTripFunc func(req *http.Request) *http.Response + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +func TestNvidiaCrawler_DiscoverAndFetch(t *testing.T) { + mockClient := &http.Client{ + Transport: roundTripFunc(func(r *http.Request) *http.Response { + if strings.Contains(r.URL.Path, "/job/JR1982000") { + body := `{ + "jobPostingInfo": { + "id": "JR1982000", + "title": "Senior Systems Software Engineer - GPU Infrastructure", + "jobDescription": "NVIDIA is seeking senior systems software engineers to accelerate AI data centers.
", + "location": "Redmond, WA", + "additionalLocations": ["Seattle, WA"] + } + }` + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(body)), + Header: make(http.Header), + } + } + + if strings.HasSuffix(r.URL.Path, "/jobs") { + body := `{ + "total": 1, + "jobPostings": [ + { + "title": "Senior Systems Software Engineer - GPU Infrastructure", + "externalPath": "/job/JR1982000", + "locationsText": "Redmond, WA", + "jobPostingId": "JR1982000" + } + ] + }` + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(body)), + Header: make(http.Header), + } + } + + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: io.NopCloser(bytes.NewBufferString("not found")), + Header: make(http.Header), + } + }), + } + + c := NewNvidiaCrawlerWithClient(mockClient) + if !c.CanHandle("nvidia") { + t.Errorf("CanHandle should support nvidia") + } + + discovered, err := c.DiscoverJobs(context.Background(), "https://nvidia.wd5.myworkdayjobs.com/NVIDIAExternalCareerSite?q=Software+Engineer") + if err != nil { + t.Fatalf("DiscoverJobs failed: %v", err) + } + + if len(discovered) != 1 { + t.Fatalf("expected 1 discovered job, got %d", len(discovered)) + } + + if discovered[0].CanonicalID != "nvidia:JR1982000" { + t.Errorf("unexpected canonical ID: %s", discovered[0].CanonicalID) + } + + extracted, err := c.FetchJobDetail(context.Background(), "https://nvidia.wd5.myworkdayjobs.com/en-US/NVIDIAExternalCareerSite/job/JR1982000") + if err != nil { + t.Fatalf("FetchJobDetail failed: %v", err) + } + + if extracted.Title != "Senior Systems Software Engineer - GPU Infrastructure" { + t.Errorf("unexpected title: %s", extracted.Title) + } + if extracted.Company != "NVIDIA" { + t.Errorf("unexpected company: %s", extracted.Company) + } + if extracted.Location != "Redmond, WA; Seattle, WA" { + t.Errorf("unexpected location: %s", extracted.Location) + } +} + +func TestNvidiaCrawler_LiveIntegration(t *testing.T) { + crawlertest.RunLiveIntegration(t, crawlertest.LiveTestConfig{ + PlatformName: "NVIDIA", + Crawler: NewNvidiaCrawler(), + TargetURL: "https://nvidia.wd5.myworkdayjobs.com/NVIDIAExternalCareerSite?q=Software+Engineer", + ExpectedCanonicalPrefix: "nvidia:", + ExpectedCompany: "NVIDIA", + }) +} + diff --git a/pkg/crawler/snowflake/snowflake.go b/pkg/crawler/snowflake/snowflake.go new file mode 100644 index 0000000..ee17d02 --- /dev/null +++ b/pkg/crawler/snowflake/snowflake.go @@ -0,0 +1,225 @@ +package snowflake + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" + + "github.com/hireme/hireme/internal/crawler" + "github.com/hireme/hireme/internal/logger" +) + +type SnowflakeCrawler struct { + client *http.Client +} + +func NewSnowflakeCrawler() *SnowflakeCrawler { + return NewSnowflakeCrawlerWithClient(&http.Client{ + Timeout: 30 * time.Second, + }) +} + +func NewSnowflakeCrawlerWithClient(client *http.Client) *SnowflakeCrawler { + return &SnowflakeCrawler{ + client: client, + } +} + +func (c *SnowflakeCrawler) CanHandle(platform string) bool { + p := strings.ToLower(platform) + return p == "snowflake" || strings.Contains(p, "careers.snowflake.com") || strings.Contains(p, "boards.greenhouse.io/snowflake") +} + +type snowflakeJobsResponse struct { + Jobs []snowflakeJob `json:"jobs"` +} + +type snowflakeJob struct { + ID int64 `json:"id"` + Title string `json:"title"` + AbsoluteURL string `json:"absolute_url"` + Location struct { + Name string `json:"name"` + } `json:"location"` + Offices []struct { + Name string `json:"name"` + Location string `json:"location"` + } `json:"offices"` + Departments []struct { + Name string `json:"name"` + } `json:"departments"` + Content string `json:"content"` +} + +func (c *SnowflakeCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { + seen := make(map[string]bool) + var discovered []crawler.DiscoveredJob + + apiURL := "https://boards-api.greenhouse.io/v1/boards/snowflake/jobs" + if targetURL != "" && strings.Contains(targetURL, "boards-api.greenhouse.io") { + apiURL = targetURL + } + + body, err := c.fetchURL(ctx, apiURL) + if err != nil { + return nil, err + } + + var resp snowflakeJobsResponse + if err := json.Unmarshal([]byte(body), &resp); err != nil { + return nil, fmt.Errorf("failed to parse Snowflake Greenhouse API response: %w", err) + } + + for _, job := range resp.Jobs { + if job.ID == 0 { + continue + } + + jobIDStr := fmt.Sprintf("%d", job.ID) + applyURL := job.AbsoluteURL + if applyURL == "" { + applyURL = fmt.Sprintf("https://boards.greenhouse.io/snowflake/jobs/%s", jobIDStr) + } + + if !seen[applyURL] { + seen[applyURL] = true + discovered = append(discovered, crawler.DiscoveredJob{ + CanonicalID: fmt.Sprintf("snowflake:%s", jobIDStr), + ApplyURL: applyURL, + }) + } + } + + logger.Log.Infow("Snowflake discovery finished", "found_count", len(discovered)) + return discovered, nil +} + +func (c *SnowflakeCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { + jobID := parseSnowflakeJobID(applyURL) + if jobID == "" { + jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16] + } + + detailAPI := fmt.Sprintf("https://boards-api.greenhouse.io/v1/boards/snowflake/jobs/%s", jobID) + body, err := c.fetchURL(ctx, detailAPI) + if err == nil && json.Valid([]byte(body)) { + var job snowflakeJob + if err := json.Unmarshal([]byte(body), &job); err == nil && job.Title != "" { + loc := job.Location.Name + if loc == "" && len(job.Offices) > 0 { + var locs []string + for _, o := range job.Offices { + if o.Name != "" { + locs = append(locs, o.Name) + } + } + loc = strings.Join(locs, "; ") + } + if loc == "" { + loc = "Bellevue / Seattle, WA" + } + + return &crawler.ExtractedJob{ + CanonicalID: fmt.Sprintf("snowflake:%s", jobID), + Title: job.Title, + Company: "Snowflake", + Location: loc, + ApplyURL: applyURL, + RawContent: cleanHTML(job.Content), + }, nil + } + } + + // Fallback to HTML + htmlBody, err := c.fetchURL(ctx, applyURL) + if err != nil { + return nil, err + } + + title := extractRegex(htmlBody, `", "\n") + s = strings.ReplaceAll(s, "
", "\n") + reTags := regexp.MustCompile(`<[^>]*>`) + s = reTags.ReplaceAllString(s, "") + return strings.TrimSpace(s) +} diff --git a/pkg/crawler/snowflake/snowflake_test.go b/pkg/crawler/snowflake/snowflake_test.go new file mode 100644 index 0000000..d7fbaeb --- /dev/null +++ b/pkg/crawler/snowflake/snowflake_test.go @@ -0,0 +1,97 @@ +package snowflake + +import ( + "bytes" + "context" + "io" + "net/http" + "strings" + "testing" +) + +type roundTripFunc func(req *http.Request) *http.Response + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +func TestSnowflakeCrawler_DiscoverAndFetch(t *testing.T) { + mockClient := &http.Client{ + Transport: roundTripFunc(func(r *http.Request) *http.Response { + if strings.HasSuffix(r.URL.Path, "/jobs/4192301") { + body := `{ + "id": 4192301, + "title": "Principal Software Engineer - Core Database Engine", + "content": "<p>Snowflake is the Data Cloud.</p>", + "location": { + "name": "Bellevue, WA" + } + }` + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(body)), + Header: make(http.Header), + } + } + + if strings.HasSuffix(r.URL.Path, "/jobs") { + body := `{ + "jobs": [ + { + "id": 4192301, + "title": "Principal Software Engineer - Core Database Engine", + "absolute_url": "https://boards.greenhouse.io/snowflake/jobs/4192301", + "location": { + "name": "Bellevue, WA" + } + } + ] + }` + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(body)), + Header: make(http.Header), + } + } + + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: io.NopCloser(bytes.NewBufferString("not found")), + Header: make(http.Header), + } + }), + } + + c := NewSnowflakeCrawlerWithClient(mockClient) + if !c.CanHandle("snowflake") { + t.Errorf("CanHandle should support snowflake") + } + + discovered, err := c.DiscoverJobs(context.Background(), "https://boards-api.greenhouse.io/v1/boards/snowflake/jobs") + if err != nil { + t.Fatalf("DiscoverJobs failed: %v", err) + } + + if len(discovered) != 1 { + t.Fatalf("expected 1 discovered job, got %d", len(discovered)) + } + + if discovered[0].CanonicalID != "snowflake:4192301" { + t.Errorf("unexpected canonical ID: %s", discovered[0].CanonicalID) + } + + extracted, err := c.FetchJobDetail(context.Background(), "https://boards.greenhouse.io/snowflake/jobs/4192301") + if err != nil { + t.Fatalf("FetchJobDetail failed: %v", err) + } + + if extracted.Title != "Principal Software Engineer - Core Database Engine" { + t.Errorf("unexpected title: %s", extracted.Title) + } + if extracted.Company != "Snowflake" { + t.Errorf("unexpected company: %s", extracted.Company) + } + if extracted.Location != "Bellevue, WA" { + t.Errorf("unexpected location: %s", extracted.Location) + } +} diff --git a/pkg/crawler/stripe/stripe.go b/pkg/crawler/stripe/stripe.go new file mode 100644 index 0000000..9eb28af --- /dev/null +++ b/pkg/crawler/stripe/stripe.go @@ -0,0 +1,226 @@ +package stripe + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" + + "github.com/hireme/hireme/internal/crawler" + "github.com/hireme/hireme/internal/logger" +) + +type StripeCrawler struct { + client *http.Client +} + +func NewStripeCrawler() *StripeCrawler { + return NewStripeCrawlerWithClient(&http.Client{ + Timeout: 30 * time.Second, + }) +} + +func NewStripeCrawlerWithClient(client *http.Client) *StripeCrawler { + return &StripeCrawler{ + client: client, + } +} + +func (c *StripeCrawler) CanHandle(platform string) bool { + p := strings.ToLower(platform) + return p == "stripe" || strings.Contains(p, "stripe.com/jobs") || strings.Contains(p, "boards.greenhouse.io/stripe") +} + +type greenhouseJobsResponse struct { + Jobs []greenhouseJob `json:"jobs"` +} + +type greenhouseJob struct { + ID int64 `json:"id"` + Title string `json:"title"` + AbsoluteURL string `json:"absolute_url"` + Location struct { + Name string `json:"name"` + } `json:"location"` + Offices []struct { + Name string `json:"name"` + Location string `json:"location"` + } `json:"offices"` + Departments []struct { + Name string `json:"name"` + } `json:"departments"` + Content string `json:"content"` + UpdatedAt string `json:"updated_at"` +} + +func (c *StripeCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { + seen := make(map[string]bool) + var discovered []crawler.DiscoveredJob + + apiURL := "https://boards-api.greenhouse.io/v1/boards/stripe/jobs" + if targetURL != "" && strings.Contains(targetURL, "boards-api.greenhouse.io") { + apiURL = targetURL + } + + body, err := c.fetchURL(ctx, apiURL) + if err != nil { + return nil, err + } + + var resp greenhouseJobsResponse + if err := json.Unmarshal([]byte(body), &resp); err != nil { + return nil, fmt.Errorf("failed to parse Stripe Greenhouse API response: %w", err) + } + + for _, job := range resp.Jobs { + if job.ID == 0 { + continue + } + + jobIDStr := fmt.Sprintf("%d", job.ID) + applyURL := job.AbsoluteURL + if applyURL == "" { + applyURL = fmt.Sprintf("https://boards.greenhouse.io/stripe/jobs/%s", jobIDStr) + } + + if !seen[applyURL] { + seen[applyURL] = true + discovered = append(discovered, crawler.DiscoveredJob{ + CanonicalID: fmt.Sprintf("stripe:%s", jobIDStr), + ApplyURL: applyURL, + }) + } + } + + logger.Log.Infow("Stripe discovery finished", "found_count", len(discovered)) + return discovered, nil +} + +func (c *StripeCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { + jobID := parseStripeJobID(applyURL) + if jobID == "" { + jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16] + } + + detailAPI := fmt.Sprintf("https://boards-api.greenhouse.io/v1/boards/stripe/jobs/%s", jobID) + body, err := c.fetchURL(ctx, detailAPI) + if err == nil && json.Valid([]byte(body)) { + var job greenhouseJob + if err := json.Unmarshal([]byte(body), &job); err == nil && job.Title != "" { + loc := job.Location.Name + if loc == "" && len(job.Offices) > 0 { + var locs []string + for _, o := range job.Offices { + if o.Name != "" { + locs = append(locs, o.Name) + } + } + loc = strings.Join(locs, "; ") + } + if loc == "" { + loc = "Seattle / Remote, WA" + } + + return &crawler.ExtractedJob{ + CanonicalID: fmt.Sprintf("stripe:%s", jobID), + Title: job.Title, + Company: "Stripe", + Location: loc, + ApplyURL: applyURL, + RawContent: cleanHTML(job.Content), + }, nil + } + } + + // Fallback to HTML + htmlBody, err := c.fetchURL(ctx, applyURL) + if err != nil { + return nil, err + } + + title := extractRegex(htmlBody, `", "\n") + s = strings.ReplaceAll(s, "
", "\n") + reTags := regexp.MustCompile(`<[^>]*>`) + s = reTags.ReplaceAllString(s, "") + return strings.TrimSpace(s) +} diff --git a/pkg/crawler/stripe/stripe_test.go b/pkg/crawler/stripe/stripe_test.go new file mode 100644 index 0000000..2906693 --- /dev/null +++ b/pkg/crawler/stripe/stripe_test.go @@ -0,0 +1,99 @@ +package stripe + +import ( + "bytes" + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/hireme/hireme/internal/crawler/crawlertest" +) + +type roundTripFunc func(req *http.Request) *http.Response + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +func TestStripeCrawler_DiscoverAndFetch(t *testing.T) { + mockClient := &http.Client{ + Transport: roundTripFunc(func(r *http.Request) *http.Response { + if strings.HasSuffix(r.URL.Path, "/jobs/6198201") { + body := `{ + "id": 6198201, + "title": "Software Engineer - Infrastructure Platform", + "content": "<p>Stripe Infrastructure powers global financial networks.</p>", + "location": { + "name": "Seattle, WA" + } + }` + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(body)), + Header: make(http.Header), + } + } + + if strings.HasSuffix(r.URL.Path, "/jobs") { + body := `{ + "jobs": [ + { + "id": 6198201, + "title": "Software Engineer - Infrastructure Platform", + "absolute_url": "https://boards.greenhouse.io/stripe/jobs/6198201", + "location": { + "name": "Seattle, WA" + } + } + ] + }` + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(body)), + Header: make(http.Header), + } + } + + return &http.Response{ + StatusCode: http.StatusNotFound, + Body: io.NopCloser(bytes.NewBufferString("not found")), + Header: make(http.Header), + } + }), + } + + c := NewStripeCrawlerWithClient(mockClient) + if !c.CanHandle("stripe") { + t.Errorf("CanHandle should support stripe") + } + + discovered, err := c.DiscoverJobs(context.Background(), "https://boards-api.greenhouse.io/v1/boards/stripe/jobs") + if err != nil { + t.Fatalf("DiscoverJobs failed: %v", err) + } + + if len(discovered) != 1 { + t.Fatalf("expected 1 discovered job, got %d", len(discovered)) + } + + if discovered[0].CanonicalID != "stripe:6198201" { + t.Errorf("unexpected canonical ID: %s", discovered[0].CanonicalID) + } + + extracted, err := c.FetchJobDetail(context.Background(), "https://boards.greenhouse.io/stripe/jobs/6198201") + if err != nil { + t.Fatalf("FetchJobDetail failed: %v", err) + } + + if extracted.Title != "Software Engineer - Infrastructure Platform" { + t.Errorf("unexpected title: %s", extracted.Title) + } + if extracted.Company != "Stripe" { + t.Errorf("unexpected company: %s", extracted.Company) + } + if extracted.Location != "Seattle, WA" { + t.Errorf("unexpected location: %s", extracted.Location) + } +} diff --git a/pkg/crawler/uber/uber.go b/pkg/crawler/uber/uber.go new file mode 100644 index 0000000..39b924b --- /dev/null +++ b/pkg/crawler/uber/uber.go @@ -0,0 +1,306 @@ +package uber + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "github.com/hireme/hireme/internal/crawler" + "github.com/hireme/hireme/internal/logger" +) + +type UberCrawler struct { + client *http.Client +} + +func NewUberCrawler() *UberCrawler { + return NewUberCrawlerWithClient(&http.Client{ + Timeout: 30 * time.Second, + }) +} + +func NewUberCrawlerWithClient(client *http.Client) *UberCrawler { + return &UberCrawler{ + client: client, + } +} + +func (c *UberCrawler) CanHandle(platform string) bool { + p := strings.ToLower(platform) + return p == "uber" || strings.Contains(p, "uber.com/careers") || strings.Contains(p, "uber.com/global/en/careers") +} + +type uberSearchPayload struct { + Limit int `json:"limit"` + Page int `json:"page"` + Params map[string]interface{} `json:"params"` +} + +type uberSearchResponse struct { + Status string `json:"status"` + Data struct { + TotalResults int `json:"totalResults"` + Results []uberJobItem `json:"results"` + } `json:"data"` +} + +type uberJobItem struct { + ID int64 `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + Location string `json:"location"` + City string `json:"city"` + Country string `json:"countryName"` + Department string `json:"department"` + Team string `json:"team"` + LineOfBusiness string `json:"lineOfBusiness"` + AllLocations []string `json:"allLocations"` + CreationDate string `json:"creationDate"` + WhatYouWillDo string `json:"whatYouWillDo"` + BasicQualifications string `json:"basicQualifications"` +} + +func (c *UberCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { + seen := make(map[string]bool) + var discovered []crawler.DiscoveredJob + + apiURL := "https://www.uber.com/api/loadSearchJobsResults" + + keyword := "Software Engineer" + u, err := url.Parse(targetURL) + if err == nil && u.Query().Get("query") != "" { + keyword = u.Query().Get("query") + } + + pageSize := 20 + for page := 0; page < 5; page++ { + reqPayload := uberSearchPayload{ + Limit: pageSize, + Page: page, + Params: map[string]interface{}{ + "query": keyword, + }, + } + + payloadBytes, err := json.Marshal(reqPayload) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewBuffer(payloadBytes)) + if err != nil { + if page == 0 { + return nil, err + } + break + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + req.Header.Set("x-csrf-token", "x") + + resp, err := c.client.Do(req) + if err != nil { + if page == 0 { + return nil, err + } + break + } + defer func() { _ = resp.Body.Close() }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + break + } + + var searchResp uberSearchResponse + if err := json.Unmarshal(body, &searchResp); err != nil { + if page == 0 { + return nil, fmt.Errorf("failed to parse Uber search response: %w", err) + } + break + } + + results := searchResp.Data.Results + if len(results) == 0 { + break + } + + newFound := false + for _, job := range results { + if job.ID == 0 { + continue + } + + jobIDStr := fmt.Sprintf("%d", job.ID) + applyURL := fmt.Sprintf("https://www.uber.com/global/en/careers/list/%s/", jobIDStr) + + if !seen[applyURL] { + seen[applyURL] = true + newFound = true + discovered = append(discovered, crawler.DiscoveredJob{ + CanonicalID: fmt.Sprintf("uber:%s", jobIDStr), + ApplyURL: applyURL, + }) + } + } + + if !newFound || len(results) < pageSize { + break + } + } + + logger.Log.Infow("Uber discovery finished", "found_count", len(discovered)) + return discovered, nil +} + +func (c *UberCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { + jobID := parseUberJobID(applyURL) + if jobID == "" { + jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16] + } + + // Try API endpoint: https://www.uber.com/api/loadJobDetails?id={id} + detailAPI := fmt.Sprintf("https://www.uber.com/api/loadJobDetails?id=%s", jobID) + req, err := http.NewRequestWithContext(ctx, "GET", detailAPI, nil) + if err == nil { + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + req.Header.Set("x-csrf-token", "x") + resp, err := c.client.Do(req) + if err == nil { + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + var detailResp struct { + Data uberJobItem `json:"data"` + } + if err := json.Unmarshal(body, &detailResp); err == nil && detailResp.Data.Title != "" { + j := detailResp.Data + var sb strings.Builder + if j.Description != "" { + sb.WriteString("## Description\n\n") + sb.WriteString(cleanHTML(j.Description)) + sb.WriteString("\n\n") + } + if j.WhatYouWillDo != "" { + sb.WriteString("## What You'll Do\n\n") + sb.WriteString(cleanHTML(j.WhatYouWillDo)) + sb.WriteString("\n\n") + } + if j.BasicQualifications != "" { + sb.WriteString("## Basic Qualifications\n\n") + sb.WriteString(cleanHTML(j.BasicQualifications)) + sb.WriteString("\n\n") + } + + loc := j.Location + if loc == "" && j.City != "" { + loc = fmt.Sprintf("%s, %s", j.City, j.Country) + } + if loc == "" { + loc = "Seattle, WA" + } + + return &crawler.ExtractedJob{ + CanonicalID: fmt.Sprintf("uber:%s", jobID), + Title: j.Title, + Company: "Uber", + Location: loc, + ApplyURL: applyURL, + RawContent: sb.String(), + }, nil + } + } + } + + // Fallback to HTML + htmlBody, err := c.fetchURL(ctx, applyURL) + if err != nil { + return nil, err + } + + title := extractRegex(htmlBody, `", "\n") + s = strings.ReplaceAll(s, "
", "\n") + reTags := regexp.MustCompile(`<[^>]*>`) + s = reTags.ReplaceAllString(s, "") + return strings.TrimSpace(s) +} diff --git a/pkg/crawler/uber/uber_test.go b/pkg/crawler/uber/uber_test.go new file mode 100644 index 0000000..ced8d72 --- /dev/null +++ b/pkg/crawler/uber/uber_test.go @@ -0,0 +1,114 @@ +package uber + +import ( + "bytes" + "context" + "io" + "net/http" + "strings" + "testing" + + "github.com/hireme/hireme/internal/crawler/crawlertest" +) + +type roundTripFunc func(req *http.Request) *http.Response + +func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req), nil +} + +func TestUberCrawler_DiscoverAndFetch(t *testing.T) { + mockClient := &http.Client{ + Transport: roundTripFunc(func(r *http.Request) *http.Response { + if strings.Contains(r.URL.Path, "loadSearchJobsResults") { + body := `{ + "status": "success", + "data": { + "totalResults": 1, + "results": [ + { + "id": 139820, + "title": "Software Engineer II - Core Compute & Storage", + "location": "Seattle, WA", + "city": "Seattle", + "countryName": "USA" + } + ] + } + }` + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewBufferString(body)), + Header: make(http.Header), + } + } + + if strings.Contains(r.URL.Path, "loadJobDetails") { + body := `{ + "data": { + "id": 139820, + "title": "Software Engineer II - Core Compute & Storage", + "description": "Uber Infra powers global dispatch and real-time computation.
", + "whatYouWillDo": "