package apple import ( "context" "crypto/sha256" "encoding/json" "fmt" "io" "net/http" "regexp" "strings" "time" "github.com/hireme/hireme/internal/crawler" "github.com/hireme/hireme/internal/logger" ) type AppleCrawler struct { client *http.Client } func NewAppleCrawler() *AppleCrawler { return &AppleCrawler{ client: &http.Client{ Timeout: 30 * time.Second, }, } } func (a *AppleCrawler) CanHandle(platform string) bool { p := strings.ToLower(platform) return p == "apple" || p == "apple_jobs" || strings.Contains(p, "jobs.apple.com") } func (a *AppleCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { seen := make(map[string]bool) var discovered []crawler.DiscoveredJob // Regex to match reID := regexp.MustCompile(`href="(/[^"]*/details/(\d+)[^"]*)"`) for page := 1; page <= 10; page++ { pageURL := targetURL if strings.Contains(pageURL, "?") { pageURL = fmt.Sprintf("%s&page=%d", targetURL, page) } else { pageURL = fmt.Sprintf("%s?page=%d", targetURL, page) } htmlContent, err := a.fetchURL(ctx, pageURL) if err != nil { if page == 1 { return nil, err } break } matches := reID.FindAllStringSubmatch(htmlContent, -1) if len(matches) == 0 { // No more jobs found on this page break } newJobsFound := false for _, m := range matches { if len(m) >= 3 { href := m[1] jobID := m[2] applyURL := href if strings.HasPrefix(applyURL, "/") { applyURL = "https://jobs.apple.com" + applyURL } if !seen[applyURL] { seen[applyURL] = true newJobsFound = true discovered = append(discovered, crawler.DiscoveredJob{ CanonicalID: fmt.Sprintf("apple:%s", jobID), ApplyURL: applyURL, }) } } } if !newJobsFound { break } } return discovered, nil } func (a *AppleCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { htmlContent, err := a.fetchURL(ctx, applyURL) if err != nil { return nil, err } jobID := parseIDFromURL(applyURL) if jobID == "" { jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16] } // Extract window.__staticRouterHydrationData re := regexp.MustCompile(`window\.__staticRouterHydrationData\s*=\s*JSON\.parse\("(.*?)"\);`) match := re.FindStringSubmatch(htmlContent) if len(match) < 2 { return nil, fmt.Errorf("could not find JSON payload in Apple job page") } quotedStr := "\"" + match[1] + "\"" var rawJson string if err := json.Unmarshal([]byte(quotedStr), &rawJson); err != nil { return nil, fmt.Errorf("failed to unmarshal JSON string literal: %w", err) } var payload map[string]interface{} if err := json.Unmarshal([]byte(rawJson), &payload); err != nil { return nil, fmt.Errorf("failed to unmarshal JSON payload: %w", err) } var jobDetails map[string]interface{} loaderData, ok := payload["loaderData"].(map[string]interface{}) if ok { if jdWrapper, ok := loaderData["jobDetails"].(map[string]interface{}); ok { if jd, ok := jdWrapper["jobsData"].(map[string]interface{}); ok { jobDetails = jd } } } if jobDetails == nil { return nil, fmt.Errorf("could not find jobsData in the parsed JSON") } title := getString(jobDetails, "postingTitle", "Unknown Apple Position") company := "Apple" // Try to get location var locationParts []string if locs, ok := jobDetails["locations"].([]interface{}); ok { for _, loc := range locs { if locMap, ok := loc.(map[string]interface{}); ok { if name, ok := locMap["name"].(string); ok && name != "" { locationParts = append(locationParts, name) } } } } location := "Apple Location" if len(locationParts) > 0 { location = strings.Join(locationParts, "; ") } // Construct raw markdown text var sb strings.Builder if summary := getString(jobDetails, "jobSummary", ""); summary != "" { sb.WriteString("## Summary\n\n") sb.WriteString(cleanHTML(summary)) sb.WriteString("\n\n") } if desc := getString(jobDetails, "description", ""); desc != "" { sb.WriteString("## Description\n\n") sb.WriteString(cleanHTML(desc)) sb.WriteString("\n\n") } if minQual := getString(jobDetails, "minimumQualifications", ""); minQual != "" { sb.WriteString("## Minimum Qualifications\n\n") sb.WriteString(cleanHTML(minQual)) sb.WriteString("\n\n") } if prefQual := getString(jobDetails, "preferredQualifications", ""); prefQual != "" { sb.WriteString("## Preferred Qualifications\n\n") sb.WriteString(cleanHTML(prefQual)) sb.WriteString("\n\n") } return &crawler.ExtractedJob{ CanonicalID: fmt.Sprintf("apple:%s", jobID), Title: title, Company: company, Location: location, ApplyURL: applyURL, RawContent: sb.String(), }, nil } func (a *AppleCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) { req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) if err != nil { return "", fmt.Errorf("failed to create HTTP request for %s: %w", urlStr, err) } req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") resp, err := a.client.Do(req) if err != nil { return "", fmt.Errorf("HTTP fetch failed for %s: %w", urlStr, err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { logger.Log.Warnw("Apple crawler HTTP error", "status", resp.StatusCode, "url", urlStr) return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr) } body, err := io.ReadAll(resp.Body) if err != nil { return "", fmt.Errorf("failed to read response body for %s: %w", urlStr, err) } return string(body), nil } func parseIDFromURL(u string) string { re := regexp.MustCompile(`/details/(\d+)`) m := re.FindStringSubmatch(u) if len(m) >= 2 { return m[1] } return "" } func getString(m map[string]interface{}, key, fallback string) string { if val, ok := m[key]; ok { if s, ok := val.(string); ok && s != "" { return s } } return fallback } func cleanHTML(s string) string { s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "
", "\n") s = strings.ReplaceAll(s, "
", "\n") s = strings.ReplaceAll(s, "
  • ", "- ") s = strings.ReplaceAll(s, "
  • ", "\n") s = strings.ReplaceAll(s, "", "\n") s = strings.ReplaceAll(s, "

    ", "\n") s = strings.ReplaceAll(s, "

    ", "\n") return strings.TrimSpace(s) }