feat: add Apple jobs crawler and implement duplicate-prevention logic for target loading

data/config/targets.yaml

@@ -2,4 +2,8 @@ targets:
   - name: "Google Software Engineer (Seattle, MID Level)"
     target_url: "https://www.google.com/about/careers/applications/jobs/results/?employment_type=FULL_TIME&q=%22Software%20Engineer%22&location=Seattle%2C%20WA%2C%20USA&target_level=MID"
     platform: "google_jobs"
+    is_active: true
+  - name: "Apple Software Engineer (Seattle)"
+    target_url: "https://jobs.apple.com/en-us/search?location=seattle-SEA&key=Software%2520Engineer&team=apps-and-frameworks-SFTWR-AF+cloud-and-infrastructure-SFTWR-CLD+core-operating-systems-SFTWR-COS+devops-and-site-reliability-SFTWR-DSR+engineering-project-management-SFTWR-EPM+information-systems-and-technology-SFTWR-ISTECH+machine-learning-and-ai-SFTWR-MCHLN+security-and-privacy-SFTWR-SEC+software-quality-automation-and-tools-SFTWR-SQAT+wireless-software-SFTWR-WSFT+machine-learning-infrastructure-MLAI-MLI+deep-learning-and-reinforcement-learning-MLAI-DLRL+natural-language-processing-and-speech-technologies-MLAI-NLP+computer-vision-MLAI-CV+applied-research-MLAI-ARr"
+    platform: "apple"
     is_active: true

internal/db/targets_loader.go

@@ -28,20 +28,23 @@ func (db *DB) LoadTargetsFromYAML(filepath string) error {
 		return fmt.Errorf("could not parse target config: %w", err)
 	}
 
-	// Check if we already have targets
-	targets, err := db.GetAllTargets()
+	// Get existing targets to prevent duplicates
+	existingTargets, err := db.GetAllTargets()
 	if err != nil {
 		return err
 	}
-	if len(targets) > 0 {
-		// Do not overwrite existing targets
-		return nil
+	
+	existingURLs := make(map[string]bool)
+	for _, t := range existingTargets {
+		existingURLs[t.TargetURL] = true
 	}
 
 	for _, t := range config.Targets {
-		_, err := db.CreateTarget(t.Name, t.TargetURL, t.Platform)
-		if err != nil {
-			return fmt.Errorf("failed to insert target %s: %w", t.Name, err)
+		if !existingURLs[t.TargetURL] {
+			_, err := db.CreateTarget(t.Name, t.TargetURL, t.Platform)
+			if err != nil {
+				return fmt.Errorf("failed to insert target %s: %w", t.Name, err)
+			}
 		}
 	}
 	return nil

internal/worker/pool.go

@@ -15,6 +15,7 @@ import (
 	"github.com/hireme/hireme/internal/db"
 	"github.com/hireme/hireme/internal/filter"
 	"github.com/hireme/hireme/internal/logger"
+	"github.com/hireme/hireme/pkg/crawler/apple"
 	"github.com/hireme/hireme/pkg/crawler/google"
 )
 
@@ -292,5 +293,8 @@ func (p *WorkerPool) getCrawlerForPlatform(platform string) crawler.SiteCrawler
 	if strings.Contains(plat, "google") {
 		return google.NewGoogleCrawler()
 	}
+	if strings.Contains(plat, "apple") {
+		return apple.NewAppleCrawler()
+	}
 	return crawler.NewGenericCrawler()
 }

pkg/crawler/apple/apple.go

@@ -0,0 +1,250 @@
+package apple
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/hireme/hireme/internal/crawler"
+	"github.com/hireme/hireme/internal/logger"
+)
+
+type AppleCrawler struct {
+	client *http.Client
+}
+
+func NewAppleCrawler() *AppleCrawler {
+	return &AppleCrawler{
+		client: &http.Client{
+			Timeout: 30 * time.Second,
+		},
+	}
+}
+
+func (a *AppleCrawler) CanHandle(platform string) bool {
+	p := strings.ToLower(platform)
+	return p == "apple" || p == "apple_jobs" || strings.Contains(p, "jobs.apple.com")
+}
+
+func (a *AppleCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
+	seen := make(map[string]bool)
+	var discovered []crawler.DiscoveredJob
+
+	// Regex to match <a ... href="/en-us/details/200677768...">
+	reID := regexp.MustCompile(`href="(/[^"]*/details/(\d+)[^"]*)"`)
+
+	for page := 1; page <= 10; page++ {
+		pageURL := targetURL
+		if strings.Contains(pageURL, "?") {
+			pageURL = fmt.Sprintf("%s&page=%d", targetURL, page)
+		} else {
+			pageURL = fmt.Sprintf("%s?page=%d", targetURL, page)
+		}
+
+		htmlContent, err := a.fetchURL(ctx, pageURL)
+		if err != nil {
+			if page == 1 {
+				return nil, err
+			}
+			break
+		}
+
+		matches := reID.FindAllStringSubmatch(htmlContent, -1)
+		if len(matches) == 0 {
+			// No more jobs found on this page
+			break
+		}
+
+		newJobsFound := false
+		for _, m := range matches {
+			if len(m) >= 3 {
+				href := m[1]
+				jobID := m[2]
+				
+				applyURL := href
+				if strings.HasPrefix(applyURL, "/") {
+					applyURL = "https://jobs.apple.com" + applyURL
+				}
+				
+				if !seen[applyURL] {
+					seen[applyURL] = true
+					newJobsFound = true
+					discovered = append(discovered, crawler.DiscoveredJob{
+						CanonicalID: fmt.Sprintf("apple:%s", jobID),
+						ApplyURL:    applyURL,
+					})
+				}
+			}
+		}
+		
+		if !newJobsFound {
+			break
+		}
+	}
+
+	return discovered, nil
+}
+
+func (a *AppleCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
+	htmlContent, err := a.fetchURL(ctx, applyURL)
+	if err != nil {
+		return nil, err
+	}
+
+	jobID := parseIDFromURL(applyURL)
+	if jobID == "" {
+		jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16]
+	}
+
+	// Extract window.__staticRouterHydrationData
+	re := regexp.MustCompile(`window\.__staticRouterHydrationData\s*=\s*JSON\.parse\("(.*?)"\);`)
+	match := re.FindStringSubmatch(htmlContent)
+	if len(match) < 2 {
+		return nil, fmt.Errorf("could not find JSON payload in Apple job page")
+	}
+
+	quotedStr := "\"" + match[1] + "\""
+	var rawJson string
+	if err := json.Unmarshal([]byte(quotedStr), &rawJson); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal JSON string literal: %w", err)
+	}
+
+	var payload map[string]interface{}
+	if err := json.Unmarshal([]byte(rawJson), &payload); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal JSON payload: %w", err)
+	}
+
+	var jobDetails map[string]interface{}
+	loaderData, ok := payload["loaderData"].(map[string]interface{})
+	if ok {
+		if jdWrapper, ok := loaderData["jobDetails"].(map[string]interface{}); ok {
+			if jd, ok := jdWrapper["jobsData"].(map[string]interface{}); ok {
+				jobDetails = jd
+			}
+		}
+	}
+
+	if jobDetails == nil {
+		return nil, fmt.Errorf("could not find jobsData in the parsed JSON")
+	}
+
+	title := getString(jobDetails, "postingTitle", "Unknown Apple Position")
+	company := "Apple"
+	
+	// Try to get location
+	var locationParts []string
+	if locs, ok := jobDetails["locations"].([]interface{}); ok {
+		for _, loc := range locs {
+			if locMap, ok := loc.(map[string]interface{}); ok {
+				if name, ok := locMap["name"].(string); ok && name != "" {
+					locationParts = append(locationParts, name)
+				}
+			}
+		}
+	}
+	location := "Apple Location"
+	if len(locationParts) > 0 {
+		location = strings.Join(locationParts, "; ")
+	}
+
+	// Construct raw markdown text
+	var sb strings.Builder
+	
+	if summary := getString(jobDetails, "jobSummary", ""); summary != "" {
+		sb.WriteString("## Summary\n\n")
+		sb.WriteString(cleanHTML(summary))
+		sb.WriteString("\n\n")
+	}
+	if desc := getString(jobDetails, "description", ""); desc != "" {
+		sb.WriteString("## Description\n\n")
+		sb.WriteString(cleanHTML(desc))
+		sb.WriteString("\n\n")
+	}
+	if minQual := getString(jobDetails, "minimumQualifications", ""); minQual != "" {
+		sb.WriteString("## Minimum Qualifications\n\n")
+		sb.WriteString(cleanHTML(minQual))
+		sb.WriteString("\n\n")
+	}
+	if prefQual := getString(jobDetails, "preferredQualifications", ""); prefQual != "" {
+		sb.WriteString("## Preferred Qualifications\n\n")
+		sb.WriteString(cleanHTML(prefQual))
+		sb.WriteString("\n\n")
+	}
+
+	return &crawler.ExtractedJob{
+		CanonicalID: fmt.Sprintf("apple:%s", jobID),
+		Title:       title,
+		Company:     company,
+		Location:    location,
+		ApplyURL:    applyURL,
+		RawContent:  sb.String(),
+	}, nil
+}
+
+func (a *AppleCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to create HTTP request for %s: %w", urlStr, err)
+	}
+
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+
+	resp, err := a.client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("HTTP fetch failed for %s: %w", urlStr, err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode != http.StatusOK {
+		logger.Log.Warnw("Apple crawler HTTP error", "status", resp.StatusCode, "url", urlStr)
+		return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read response body for %s: %w", urlStr, err)
+	}
+
+	return string(body), nil
+}
+
+func parseIDFromURL(u string) string {
+	re := regexp.MustCompile(`/details/(\d+)`)
+	m := re.FindStringSubmatch(u)
+	if len(m) >= 2 {
+		return m[1]
+	}
+	return ""
+}
+
+func getString(m map[string]interface{}, key, fallback string) string {
+	if val, ok := m[key]; ok {
+		if s, ok := val.(string); ok && s != "" {
+			return s
+		}
+	}
+	return fallback
+}
+
+func cleanHTML(s string) string {
+	s = strings.ReplaceAll(s, "<b>", "**")
+	s = strings.ReplaceAll(s, "</b>", "**")
+	s = strings.ReplaceAll(s, "<strong>", "**")
+	s = strings.ReplaceAll(s, "</strong>", "**")
+	s = strings.ReplaceAll(s, "<br>", "\n")
+	s = strings.ReplaceAll(s, "<br/>", "\n")
+	s = strings.ReplaceAll(s, "<li>", "- ")
+	s = strings.ReplaceAll(s, "</li>", "\n")
+	s = strings.ReplaceAll(s, "<ul>", "\n")
+	s = strings.ReplaceAll(s, "</ul>", "\n")
+	s = strings.ReplaceAll(s, "<p>", "\n")
+	s = strings.ReplaceAll(s, "</p>", "\n")
+	return strings.TrimSpace(s)
+}

pkg/crawler/apple/apple_test.go

@@ -0,0 +1,66 @@
+package apple
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestAppleCrawler_EndToEnd(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping end-to-end network test in short mode")
+	}
+
+	crawler := NewAppleCrawler()
+
+	// Use the default test URL
+	targetURL := "https://jobs.apple.com/en-us/search?location=seattle-SEA&key=Software%2520Engineer"
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	// 1. Test DiscoverJobs
+	t.Logf("Discovering jobs from: %s", targetURL)
+	jobs, err := crawler.DiscoverJobs(ctx, targetURL)
+	if err != nil {
+		t.Fatalf("DiscoverJobs failed: %v", err)
+	}
+
+	if len(jobs) == 0 {
+		t.Fatalf("DiscoverJobs returned 0 jobs for URL: %s", targetURL)
+	}
+	t.Logf("Successfully discovered %d jobs", len(jobs))
+
+	// 2. Pick the first job and test FetchJobDetail
+	testJob := jobs[0]
+	t.Logf("Fetching details for job: %s", testJob.ApplyURL)
+
+	details, err := crawler.FetchJobDetail(ctx, testJob.ApplyURL)
+	if err != nil {
+		t.Fatalf("FetchJobDetail failed: %v", err)
+	}
+
+	if details == nil {
+		t.Fatal("FetchJobDetail returned nil ExtractedJob")
+	}
+
+	// 3. Verify extracted details
+	t.Logf("Extracted Title: %s", details.Title)
+	t.Logf("Extracted Company: %s", details.Company)
+	t.Logf("Extracted Location: %s", details.Location)
+	t.Logf("Raw Content Length: %d", len(details.RawContent))
+
+	if details.Title == "" || details.Title == "Unknown Apple Position" {
+		t.Errorf("Expected valid job title, got %q", details.Title)
+	}
+	if details.Company != "Apple" {
+		t.Errorf("Expected company 'Apple', got %q", details.Company)
+	}
+	if !strings.HasPrefix(details.CanonicalID, "apple:") {
+		t.Errorf("Expected CanonicalID to start with 'apple:', got %q", details.CanonicalID)
+	}
+	if len(details.RawContent) < 100 {
+		t.Errorf("Expected RawContent to be substantive, got length %d", len(details.RawContent))
+	}
+}