package amazon import ( "context" "crypto/sha256" "encoding/json" "fmt" "io" "net/http" "net/url" "regexp" "strings" "time" "git.pengzhan.dev/haopengzhan/hireme/internal/crawler" "git.pengzhan.dev/haopengzhan/hireme/internal/logger" ) type AmazonCrawler struct { client *http.Client } func NewAmazonCrawler() *AmazonCrawler { return NewAmazonCrawlerWithClient(&http.Client{ Timeout: 30 * time.Second, }) } func NewAmazonCrawlerWithClient(client *http.Client) *AmazonCrawler { return &AmazonCrawler{ client: client, } } func (c *AmazonCrawler) CanHandle(platform string) bool { p := strings.ToLower(platform) return p == "amazon" || p == "aws" || strings.Contains(p, "amazon.jobs") } type amazonSearchResponse struct { Hits int `json:"hits"` Jobs []amazonJobItem `json:"jobs"` } type amazonJobItem struct { ID string `json:"id"` IDIsp string `json:"id_icims"` Title string `json:"title"` Company string `json:"company_name"` Location string `json:"location"` City string `json:"city"` State string `json:"state"` Country string `json:"country_code"` JobPath string `json:"job_path"` Description string `json:"description"` BasicQualifications string `json:"basic_qualifications"` PreferredQuals string `json:"preferred_qualifications"` JobCategory string `json:"job_category"` JobFamily string `json:"job_family_name"` PostedDate string `json:"posted_date"` } func (c *AmazonCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { seen := make(map[string]bool) var discovered []crawler.DiscoveredJob searchAPI := targetURL if !strings.Contains(searchAPI, "search.json") { u, err := url.Parse(targetURL) if err == nil && u.Host != "" { q := u.Query() searchAPI = fmt.Sprintf("https://www.amazon.jobs/en/search.json?%s", q.Encode()) } else { searchAPI = fmt.Sprintf("https://www.amazon.jobs/en/search.json?base_query=%s&loc_query=Seattle%%2C+WA%%2C+USA", url.QueryEscape(targetURL)) } } resultLimit := 100 for page := 0; page < 5; page++ { offset := page * resultLimit reqURL := searchAPI if strings.Contains(reqURL, "?") { reqURL = fmt.Sprintf("%s&result_limit=%d&offset=%d", searchAPI, resultLimit, offset) } else { reqURL = fmt.Sprintf("%s?result_limit=%d&offset=%d", searchAPI, resultLimit, offset) } body, err := c.fetchURL(ctx, reqURL) if err != nil { if page == 0 { return nil, err } break } var resp amazonSearchResponse if err := json.Unmarshal([]byte(body), &resp); err != nil { if page == 0 { return nil, fmt.Errorf("failed to parse Amazon search response: %w", err) } break } if len(resp.Jobs) == 0 { break } newFound := false for _, job := range resp.Jobs { jobID := job.ID if jobID == "" { jobID = job.IDIsp } if jobID == "" { continue } applyURL := fmt.Sprintf("https://www.amazon.jobs/en/jobs/%s", jobID) if job.JobPath != "" { if strings.HasPrefix(job.JobPath, "http") { applyURL = job.JobPath } else { applyURL = "https://www.amazon.jobs" + job.JobPath } } if !seen[applyURL] { seen[applyURL] = true newFound = true discovered = append(discovered, crawler.DiscoveredJob{ CanonicalID: fmt.Sprintf("amazon:%s", jobID), ApplyURL: applyURL, }) } } if !newFound || len(resp.Jobs) < resultLimit { break } } logger.Log.Infow("Amazon discovery finished", "found_count", len(discovered)) return discovered, nil } func (c *AmazonCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { jobID := parseAmazonJobID(applyURL) if jobID == "" { jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16] } jsonURL := fmt.Sprintf("https://www.amazon.jobs/en/jobs/%s.json", jobID) body, err := c.fetchURL(ctx, jsonURL) if err == nil && json.Valid([]byte(body)) { var detail struct { Job amazonJobItem `json:"job"` } if err := json.Unmarshal([]byte(body), &detail); err == nil && detail.Job.Title != "" { j := detail.Job var sb strings.Builder if j.Description != "" { sb.WriteString("## Description\n\n") sb.WriteString(cleanHTML(j.Description)) sb.WriteString("\n\n") } if j.BasicQualifications != "" { sb.WriteString("## Basic Qualifications\n\n") sb.WriteString(cleanHTML(j.BasicQualifications)) sb.WriteString("\n\n") } if j.PreferredQuals != "" { sb.WriteString("## Preferred Qualifications\n\n") sb.WriteString(cleanHTML(j.PreferredQuals)) sb.WriteString("\n\n") } company := "Amazon" if j.Company != "" { company = j.Company } loc := j.Location if loc == "" && j.City != "" { loc = fmt.Sprintf("%s, %s, %s", j.City, j.State, j.Country) } if loc == "" { loc = "Seattle, WA, USA" } return &crawler.ExtractedJob{ CanonicalID: fmt.Sprintf("amazon:%s", jobID), Title: j.Title, Company: company, Location: loc, ApplyURL: applyURL, RawContent: sb.String(), }, nil } } htmlBody, err := c.fetchURL(ctx, applyURL) if err != nil { return nil, err } title := extractRegex(htmlBody, `
", "\n") s = strings.ReplaceAll(s, "
", "\n") reTags := regexp.MustCompile(`<[^>]*>`) s = reTags.ReplaceAllString(s, "") return strings.TrimSpace(s) }