hireme/pkg/crawler/apple/apple.go

package apple

import (
	"context"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strings"
	"time"

	"git.pengzhan.dev/haopengzhan/hireme/internal/crawler"
	"git.pengzhan.dev/haopengzhan/hireme/internal/logger"
)

type AppleCrawler struct {
	client *http.Client
}

func NewAppleCrawler() *AppleCrawler {
	return &AppleCrawler{
		client: &http.Client{
			Timeout: 30 * time.Second,
		},
	}
}

func (a *AppleCrawler) CanHandle(platform string) bool {
	p := strings.ToLower(platform)
	return p == "apple" || p == "apple_jobs" || strings.Contains(p, "jobs.apple.com")
}

func (a *AppleCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
	seen := make(map[string]bool)
	var discovered []crawler.DiscoveredJob

	// Regex to match <a ... href="/en-us/details/200677768...">
	reID := regexp.MustCompile(`href="(/[^"]*/details/(\d+)[^"]*)"`)

	for page := 1; page <= 10; page++ {
		pageURL := targetURL
		if strings.Contains(pageURL, "?") {
			pageURL = fmt.Sprintf("%s&page=%d", targetURL, page)
		} else {
			pageURL = fmt.Sprintf("%s?page=%d", targetURL, page)
		}

		htmlContent, err := a.fetchURL(ctx, pageURL)
		if err != nil {
			if page == 1 {
				return nil, err
			}
			break
		}

		matches := reID.FindAllStringSubmatch(htmlContent, -1)
		if len(matches) == 0 {
			// No more jobs found on this page
			break
		}

		newJobsFound := false
		for _, m := range matches {
			if len(m) >= 3 {
				href := m[1]
				jobID := m[2]
				
				applyURL := href
				if strings.HasPrefix(applyURL, "/") {
					applyURL = "https://jobs.apple.com" + applyURL
				}
				
				if !seen[applyURL] {
					seen[applyURL] = true
					newJobsFound = true
					discovered = append(discovered, crawler.DiscoveredJob{
						CanonicalID: fmt.Sprintf("apple:%s", jobID),
						ApplyURL:    applyURL,
					})
				}
			}
		}
		
		if !newJobsFound {
			break
		}
	}

	return discovered, nil
}

func (a *AppleCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
	htmlContent, err := a.fetchURL(ctx, applyURL)
	if err != nil {
		return nil, err
	}

	jobID := parseIDFromURL(applyURL)
	if jobID == "" {
		jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16]
	}

	// Extract window.__staticRouterHydrationData
	re := regexp.MustCompile(`window\.__staticRouterHydrationData\s*=\s*JSON\.parse\("(.*?)"\);`)
	match := re.FindStringSubmatch(htmlContent)
	if len(match) < 2 {
		return nil, fmt.Errorf("could not find JSON payload in Apple job page")
	}

	quotedStr := "\"" + match[1] + "\""
	var rawJson string
	if err := json.Unmarshal([]byte(quotedStr), &rawJson); err != nil {
		return nil, fmt.Errorf("failed to unmarshal JSON string literal: %w", err)
	}

	var payload map[string]interface{}
	if err := json.Unmarshal([]byte(rawJson), &payload); err != nil {
		return nil, fmt.Errorf("failed to unmarshal JSON payload: %w", err)
	}

	var jobDetails map[string]interface{}
	loaderData, ok := payload["loaderData"].(map[string]interface{})
	if ok {
		if jdWrapper, ok := loaderData["jobDetails"].(map[string]interface{}); ok {
			if jd, ok := jdWrapper["jobsData"].(map[string]interface{}); ok {
				jobDetails = jd
			}
		}
	}

	if jobDetails == nil {
		return nil, fmt.Errorf("could not find jobsData in the parsed JSON")
	}

	title := getString(jobDetails, "postingTitle", "Unknown Apple Position")
	company := "Apple"
	
	// Try to get location
	var locationParts []string
	if locs, ok := jobDetails["locations"].([]interface{}); ok {
		for _, loc := range locs {
			if locMap, ok := loc.(map[string]interface{}); ok {
				if name, ok := locMap["name"].(string); ok && name != "" {
					locationParts = append(locationParts, name)
				}
			}
		}
	}
	location := "Apple Location"
	if len(locationParts) > 0 {
		location = strings.Join(locationParts, "; ")
	}

	// Construct raw markdown text
	var sb strings.Builder
	
	if summary := getString(jobDetails, "jobSummary", ""); summary != "" {
		sb.WriteString("## Summary\n\n")
		sb.WriteString(cleanHTML(summary))
		sb.WriteString("\n\n")
	}
	if desc := getString(jobDetails, "description", ""); desc != "" {
		sb.WriteString("## Description\n\n")
		sb.WriteString(cleanHTML(desc))
		sb.WriteString("\n\n")
	}
	if minQual := getString(jobDetails, "minimumQualifications", ""); minQual != "" {
		sb.WriteString("## Minimum Qualifications\n\n")
		sb.WriteString(cleanHTML(minQual))
		sb.WriteString("\n\n")
	}
	if prefQual := getString(jobDetails, "preferredQualifications", ""); prefQual != "" {
		sb.WriteString("## Preferred Qualifications\n\n")
		sb.WriteString(cleanHTML(prefQual))
		sb.WriteString("\n\n")
	}

	return &crawler.ExtractedJob{
		CanonicalID: fmt.Sprintf("apple:%s", jobID),
		Title:       title,
		Company:     company,
		Location:    location,
		ApplyURL:    applyURL,
		RawContent:  sb.String(),
	}, nil
}

func (a *AppleCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) {
	req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
	if err != nil {
		return "", fmt.Errorf("failed to create HTTP request for %s: %w", urlStr, err)
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")

	resp, err := a.client.Do(req)
	if err != nil {
		return "", fmt.Errorf("HTTP fetch failed for %s: %w", urlStr, err)
	}
	defer func() { _ = resp.Body.Close() }()

	if resp.StatusCode != http.StatusOK {
		logger.Log.Warnw("Apple crawler HTTP error", "status", resp.StatusCode, "url", urlStr)
		return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr)
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", fmt.Errorf("failed to read response body for %s: %w", urlStr, err)
	}

	return string(body), nil
}

func parseIDFromURL(u string) string {
	re := regexp.MustCompile(`/details/(\d+)`)
	m := re.FindStringSubmatch(u)
	if len(m) >= 2 {
		return m[1]
	}
	return ""
}

func getString(m map[string]interface{}, key, fallback string) string {
	if val, ok := m[key]; ok {
		if s, ok := val.(string); ok && s != "" {
			return s
		}
	}
	return fallback
}

func cleanHTML(s string) string {
	s = strings.ReplaceAll(s, "<b>", "**")
	s = strings.ReplaceAll(s, "</b>", "**")
	s = strings.ReplaceAll(s, "<strong>", "**")
	s = strings.ReplaceAll(s, "</strong>", "**")
	s = strings.ReplaceAll(s, "<br>", "\n")
	s = strings.ReplaceAll(s, "<br/>", "\n")
	s = strings.ReplaceAll(s, "<li>", "- ")
	s = strings.ReplaceAll(s, "</li>", "\n")
	s = strings.ReplaceAll(s, "<ul>", "\n")
	s = strings.ReplaceAll(s, "</ul>", "\n")
	s = strings.ReplaceAll(s, "<p>", "\n")
	s = strings.ReplaceAll(s, "</p>", "\n")
	return strings.TrimSpace(s)
}