From f8bfc4b5e36026f99320445e0c459a171e0b4785 Mon Sep 17 00:00:00 2001 From: haopengzhan Date: Sat, 15 Aug 2026 00:56:20 -0700 Subject: [PATCH] feat: add Apple jobs crawler and implement duplicate-prevention logic for target loading --- data/config/targets.yaml | 4 + internal/db/targets_loader.go | 19 ++- internal/worker/pool.go | 4 + pkg/crawler/apple/apple.go | 250 ++++++++++++++++++++++++++++++++ pkg/crawler/apple/apple_test.go | 66 +++++++++ 5 files changed, 335 insertions(+), 8 deletions(-) create mode 100644 pkg/crawler/apple/apple.go create mode 100644 pkg/crawler/apple/apple_test.go diff --git a/data/config/targets.yaml b/data/config/targets.yaml index d3018f4..2e97241 100644 --- a/data/config/targets.yaml +++ b/data/config/targets.yaml @@ -2,4 +2,8 @@ targets: - name: "Google Software Engineer (Seattle, MID Level)" target_url: "https://www.google.com/about/careers/applications/jobs/results/?employment_type=FULL_TIME&q=%22Software%20Engineer%22&location=Seattle%2C%20WA%2C%20USA&target_level=MID" platform: "google_jobs" + is_active: true + - name: "Apple Software Engineer (Seattle)" + target_url: "https://jobs.apple.com/en-us/search?location=seattle-SEA&key=Software%2520Engineer&team=apps-and-frameworks-SFTWR-AF+cloud-and-infrastructure-SFTWR-CLD+core-operating-systems-SFTWR-COS+devops-and-site-reliability-SFTWR-DSR+engineering-project-management-SFTWR-EPM+information-systems-and-technology-SFTWR-ISTECH+machine-learning-and-ai-SFTWR-MCHLN+security-and-privacy-SFTWR-SEC+software-quality-automation-and-tools-SFTWR-SQAT+wireless-software-SFTWR-WSFT+machine-learning-infrastructure-MLAI-MLI+deep-learning-and-reinforcement-learning-MLAI-DLRL+natural-language-processing-and-speech-technologies-MLAI-NLP+computer-vision-MLAI-CV+applied-research-MLAI-ARr" + platform: "apple" is_active: true \ No newline at end of file diff --git a/internal/db/targets_loader.go b/internal/db/targets_loader.go index 38d7f44..74edd83 100644 --- a/internal/db/targets_loader.go +++ b/internal/db/targets_loader.go @@ -28,20 +28,23 @@ func (db *DB) LoadTargetsFromYAML(filepath string) error { return fmt.Errorf("could not parse target config: %w", err) } - // Check if we already have targets - targets, err := db.GetAllTargets() + // Get existing targets to prevent duplicates + existingTargets, err := db.GetAllTargets() if err != nil { return err } - if len(targets) > 0 { - // Do not overwrite existing targets - return nil + + existingURLs := make(map[string]bool) + for _, t := range existingTargets { + existingURLs[t.TargetURL] = true } for _, t := range config.Targets { - _, err := db.CreateTarget(t.Name, t.TargetURL, t.Platform) - if err != nil { - return fmt.Errorf("failed to insert target %s: %w", t.Name, err) + if !existingURLs[t.TargetURL] { + _, err := db.CreateTarget(t.Name, t.TargetURL, t.Platform) + if err != nil { + return fmt.Errorf("failed to insert target %s: %w", t.Name, err) + } } } return nil diff --git a/internal/worker/pool.go b/internal/worker/pool.go index 17972ef..ce0b7d5 100644 --- a/internal/worker/pool.go +++ b/internal/worker/pool.go @@ -15,6 +15,7 @@ import ( "github.com/hireme/hireme/internal/db" "github.com/hireme/hireme/internal/filter" "github.com/hireme/hireme/internal/logger" + "github.com/hireme/hireme/pkg/crawler/apple" "github.com/hireme/hireme/pkg/crawler/google" ) @@ -292,5 +293,8 @@ func (p *WorkerPool) getCrawlerForPlatform(platform string) crawler.SiteCrawler if strings.Contains(plat, "google") { return google.NewGoogleCrawler() } + if strings.Contains(plat, "apple") { + return apple.NewAppleCrawler() + } return crawler.NewGenericCrawler() } diff --git a/pkg/crawler/apple/apple.go b/pkg/crawler/apple/apple.go new file mode 100644 index 0000000..2549b68 --- /dev/null +++ b/pkg/crawler/apple/apple.go @@ -0,0 +1,250 @@ +package apple + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "time" + + "github.com/hireme/hireme/internal/crawler" + "github.com/hireme/hireme/internal/logger" +) + +type AppleCrawler struct { + client *http.Client +} + +func NewAppleCrawler() *AppleCrawler { + return &AppleCrawler{ + client: &http.Client{ + Timeout: 30 * time.Second, + }, + } +} + +func (a *AppleCrawler) CanHandle(platform string) bool { + p := strings.ToLower(platform) + return p == "apple" || p == "apple_jobs" || strings.Contains(p, "jobs.apple.com") +} + +func (a *AppleCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { + seen := make(map[string]bool) + var discovered []crawler.DiscoveredJob + + // Regex to match + reID := regexp.MustCompile(`href="(/[^"]*/details/(\d+)[^"]*)"`) + + for page := 1; page <= 10; page++ { + pageURL := targetURL + if strings.Contains(pageURL, "?") { + pageURL = fmt.Sprintf("%s&page=%d", targetURL, page) + } else { + pageURL = fmt.Sprintf("%s?page=%d", targetURL, page) + } + + htmlContent, err := a.fetchURL(ctx, pageURL) + if err != nil { + if page == 1 { + return nil, err + } + break + } + + matches := reID.FindAllStringSubmatch(htmlContent, -1) + if len(matches) == 0 { + // No more jobs found on this page + break + } + + newJobsFound := false + for _, m := range matches { + if len(m) >= 3 { + href := m[1] + jobID := m[2] + + applyURL := href + if strings.HasPrefix(applyURL, "/") { + applyURL = "https://jobs.apple.com" + applyURL + } + + if !seen[applyURL] { + seen[applyURL] = true + newJobsFound = true + discovered = append(discovered, crawler.DiscoveredJob{ + CanonicalID: fmt.Sprintf("apple:%s", jobID), + ApplyURL: applyURL, + }) + } + } + } + + if !newJobsFound { + break + } + } + + return discovered, nil +} + +func (a *AppleCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { + htmlContent, err := a.fetchURL(ctx, applyURL) + if err != nil { + return nil, err + } + + jobID := parseIDFromURL(applyURL) + if jobID == "" { + jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16] + } + + // Extract window.__staticRouterHydrationData + re := regexp.MustCompile(`window\.__staticRouterHydrationData\s*=\s*JSON\.parse\("(.*?)"\);`) + match := re.FindStringSubmatch(htmlContent) + if len(match) < 2 { + return nil, fmt.Errorf("could not find JSON payload in Apple job page") + } + + quotedStr := "\"" + match[1] + "\"" + var rawJson string + if err := json.Unmarshal([]byte(quotedStr), &rawJson); err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON string literal: %w", err) + } + + var payload map[string]interface{} + if err := json.Unmarshal([]byte(rawJson), &payload); err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON payload: %w", err) + } + + var jobDetails map[string]interface{} + loaderData, ok := payload["loaderData"].(map[string]interface{}) + if ok { + if jdWrapper, ok := loaderData["jobDetails"].(map[string]interface{}); ok { + if jd, ok := jdWrapper["jobsData"].(map[string]interface{}); ok { + jobDetails = jd + } + } + } + + if jobDetails == nil { + return nil, fmt.Errorf("could not find jobsData in the parsed JSON") + } + + title := getString(jobDetails, "postingTitle", "Unknown Apple Position") + company := "Apple" + + // Try to get location + var locationParts []string + if locs, ok := jobDetails["locations"].([]interface{}); ok { + for _, loc := range locs { + if locMap, ok := loc.(map[string]interface{}); ok { + if name, ok := locMap["name"].(string); ok && name != "" { + locationParts = append(locationParts, name) + } + } + } + } + location := "Apple Location" + if len(locationParts) > 0 { + location = strings.Join(locationParts, "; ") + } + + // Construct raw markdown text + var sb strings.Builder + + if summary := getString(jobDetails, "jobSummary", ""); summary != "" { + sb.WriteString("## Summary\n\n") + sb.WriteString(cleanHTML(summary)) + sb.WriteString("\n\n") + } + if desc := getString(jobDetails, "description", ""); desc != "" { + sb.WriteString("## Description\n\n") + sb.WriteString(cleanHTML(desc)) + sb.WriteString("\n\n") + } + if minQual := getString(jobDetails, "minimumQualifications", ""); minQual != "" { + sb.WriteString("## Minimum Qualifications\n\n") + sb.WriteString(cleanHTML(minQual)) + sb.WriteString("\n\n") + } + if prefQual := getString(jobDetails, "preferredQualifications", ""); prefQual != "" { + sb.WriteString("## Preferred Qualifications\n\n") + sb.WriteString(cleanHTML(prefQual)) + sb.WriteString("\n\n") + } + + return &crawler.ExtractedJob{ + CanonicalID: fmt.Sprintf("apple:%s", jobID), + Title: title, + Company: company, + Location: location, + ApplyURL: applyURL, + RawContent: sb.String(), + }, nil +} + +func (a *AppleCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) { + req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) + if err != nil { + return "", fmt.Errorf("failed to create HTTP request for %s: %w", urlStr, err) + } + + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + + resp, err := a.client.Do(req) + if err != nil { + return "", fmt.Errorf("HTTP fetch failed for %s: %w", urlStr, err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + logger.Log.Warnw("Apple crawler HTTP error", "status", resp.StatusCode, "url", urlStr) + return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body for %s: %w", urlStr, err) + } + + return string(body), nil +} + +func parseIDFromURL(u string) string { + re := regexp.MustCompile(`/details/(\d+)`) + m := re.FindStringSubmatch(u) + if len(m) >= 2 { + return m[1] + } + return "" +} + +func getString(m map[string]interface{}, key, fallback string) string { + if val, ok := m[key]; ok { + if s, ok := val.(string); ok && s != "" { + return s + } + } + return fallback +} + +func cleanHTML(s string) string { + s = strings.ReplaceAll(s, "", "**") + s = strings.ReplaceAll(s, "", "**") + s = strings.ReplaceAll(s, "", "**") + s = strings.ReplaceAll(s, "", "**") + s = strings.ReplaceAll(s, "
", "\n") + s = strings.ReplaceAll(s, "
", "\n") + s = strings.ReplaceAll(s, "
  • ", "- ") + s = strings.ReplaceAll(s, "
    • ", "\n") + s = strings.ReplaceAll(s, "", "\n") + s = strings.ReplaceAll(s, "

    ", "\n") + s = strings.ReplaceAll(s, "

    ", "\n") + return strings.TrimSpace(s) +} diff --git a/pkg/crawler/apple/apple_test.go b/pkg/crawler/apple/apple_test.go new file mode 100644 index 0000000..f54b9fb --- /dev/null +++ b/pkg/crawler/apple/apple_test.go @@ -0,0 +1,66 @@ +package apple + +import ( + "context" + "strings" + "testing" + "time" +) + +func TestAppleCrawler_EndToEnd(t *testing.T) { + if testing.Short() { + t.Skip("skipping end-to-end network test in short mode") + } + + crawler := NewAppleCrawler() + + // Use the default test URL + targetURL := "https://jobs.apple.com/en-us/search?location=seattle-SEA&key=Software%2520Engineer" + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + // 1. Test DiscoverJobs + t.Logf("Discovering jobs from: %s", targetURL) + jobs, err := crawler.DiscoverJobs(ctx, targetURL) + if err != nil { + t.Fatalf("DiscoverJobs failed: %v", err) + } + + if len(jobs) == 0 { + t.Fatalf("DiscoverJobs returned 0 jobs for URL: %s", targetURL) + } + t.Logf("Successfully discovered %d jobs", len(jobs)) + + // 2. Pick the first job and test FetchJobDetail + testJob := jobs[0] + t.Logf("Fetching details for job: %s", testJob.ApplyURL) + + details, err := crawler.FetchJobDetail(ctx, testJob.ApplyURL) + if err != nil { + t.Fatalf("FetchJobDetail failed: %v", err) + } + + if details == nil { + t.Fatal("FetchJobDetail returned nil ExtractedJob") + } + + // 3. Verify extracted details + t.Logf("Extracted Title: %s", details.Title) + t.Logf("Extracted Company: %s", details.Company) + t.Logf("Extracted Location: %s", details.Location) + t.Logf("Raw Content Length: %d", len(details.RawContent)) + + if details.Title == "" || details.Title == "Unknown Apple Position" { + t.Errorf("Expected valid job title, got %q", details.Title) + } + if details.Company != "Apple" { + t.Errorf("Expected company 'Apple', got %q", details.Company) + } + if !strings.HasPrefix(details.CanonicalID, "apple:") { + t.Errorf("Expected CanonicalID to start with 'apple:', got %q", details.CanonicalID) + } + if len(details.RawContent) < 100 { + t.Errorf("Expected RawContent to be substantive, got length %d", len(details.RawContent)) + } +}