feat: add Apple jobs crawler and implement duplicate-prevention logic for target loading
This commit is contained in:
@@ -0,0 +1,250 @@
|
||||
package apple
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hireme/hireme/internal/crawler"
|
||||
"github.com/hireme/hireme/internal/logger"
|
||||
)
|
||||
|
||||
type AppleCrawler struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewAppleCrawler() *AppleCrawler {
|
||||
return &AppleCrawler{
|
||||
client: &http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (a *AppleCrawler) CanHandle(platform string) bool {
|
||||
p := strings.ToLower(platform)
|
||||
return p == "apple" || p == "apple_jobs" || strings.Contains(p, "jobs.apple.com")
|
||||
}
|
||||
|
||||
func (a *AppleCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
|
||||
seen := make(map[string]bool)
|
||||
var discovered []crawler.DiscoveredJob
|
||||
|
||||
// Regex to match <a ... href="/en-us/details/200677768...">
|
||||
reID := regexp.MustCompile(`href="(/[^"]*/details/(\d+)[^"]*)"`)
|
||||
|
||||
for page := 1; page <= 10; page++ {
|
||||
pageURL := targetURL
|
||||
if strings.Contains(pageURL, "?") {
|
||||
pageURL = fmt.Sprintf("%s&page=%d", targetURL, page)
|
||||
} else {
|
||||
pageURL = fmt.Sprintf("%s?page=%d", targetURL, page)
|
||||
}
|
||||
|
||||
htmlContent, err := a.fetchURL(ctx, pageURL)
|
||||
if err != nil {
|
||||
if page == 1 {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
matches := reID.FindAllStringSubmatch(htmlContent, -1)
|
||||
if len(matches) == 0 {
|
||||
// No more jobs found on this page
|
||||
break
|
||||
}
|
||||
|
||||
newJobsFound := false
|
||||
for _, m := range matches {
|
||||
if len(m) >= 3 {
|
||||
href := m[1]
|
||||
jobID := m[2]
|
||||
|
||||
applyURL := href
|
||||
if strings.HasPrefix(applyURL, "/") {
|
||||
applyURL = "https://jobs.apple.com" + applyURL
|
||||
}
|
||||
|
||||
if !seen[applyURL] {
|
||||
seen[applyURL] = true
|
||||
newJobsFound = true
|
||||
discovered = append(discovered, crawler.DiscoveredJob{
|
||||
CanonicalID: fmt.Sprintf("apple:%s", jobID),
|
||||
ApplyURL: applyURL,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !newJobsFound {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return discovered, nil
|
||||
}
|
||||
|
||||
func (a *AppleCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
|
||||
htmlContent, err := a.fetchURL(ctx, applyURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
jobID := parseIDFromURL(applyURL)
|
||||
if jobID == "" {
|
||||
jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16]
|
||||
}
|
||||
|
||||
// Extract window.__staticRouterHydrationData
|
||||
re := regexp.MustCompile(`window\.__staticRouterHydrationData\s*=\s*JSON\.parse\("(.*?)"\);`)
|
||||
match := re.FindStringSubmatch(htmlContent)
|
||||
if len(match) < 2 {
|
||||
return nil, fmt.Errorf("could not find JSON payload in Apple job page")
|
||||
}
|
||||
|
||||
quotedStr := "\"" + match[1] + "\""
|
||||
var rawJson string
|
||||
if err := json.Unmarshal([]byte(quotedStr), &rawJson); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal JSON string literal: %w", err)
|
||||
}
|
||||
|
||||
var payload map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(rawJson), &payload); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal JSON payload: %w", err)
|
||||
}
|
||||
|
||||
var jobDetails map[string]interface{}
|
||||
loaderData, ok := payload["loaderData"].(map[string]interface{})
|
||||
if ok {
|
||||
if jdWrapper, ok := loaderData["jobDetails"].(map[string]interface{}); ok {
|
||||
if jd, ok := jdWrapper["jobsData"].(map[string]interface{}); ok {
|
||||
jobDetails = jd
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if jobDetails == nil {
|
||||
return nil, fmt.Errorf("could not find jobsData in the parsed JSON")
|
||||
}
|
||||
|
||||
title := getString(jobDetails, "postingTitle", "Unknown Apple Position")
|
||||
company := "Apple"
|
||||
|
||||
// Try to get location
|
||||
var locationParts []string
|
||||
if locs, ok := jobDetails["locations"].([]interface{}); ok {
|
||||
for _, loc := range locs {
|
||||
if locMap, ok := loc.(map[string]interface{}); ok {
|
||||
if name, ok := locMap["name"].(string); ok && name != "" {
|
||||
locationParts = append(locationParts, name)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
location := "Apple Location"
|
||||
if len(locationParts) > 0 {
|
||||
location = strings.Join(locationParts, "; ")
|
||||
}
|
||||
|
||||
// Construct raw markdown text
|
||||
var sb strings.Builder
|
||||
|
||||
if summary := getString(jobDetails, "jobSummary", ""); summary != "" {
|
||||
sb.WriteString("## Summary\n\n")
|
||||
sb.WriteString(cleanHTML(summary))
|
||||
sb.WriteString("\n\n")
|
||||
}
|
||||
if desc := getString(jobDetails, "description", ""); desc != "" {
|
||||
sb.WriteString("## Description\n\n")
|
||||
sb.WriteString(cleanHTML(desc))
|
||||
sb.WriteString("\n\n")
|
||||
}
|
||||
if minQual := getString(jobDetails, "minimumQualifications", ""); minQual != "" {
|
||||
sb.WriteString("## Minimum Qualifications\n\n")
|
||||
sb.WriteString(cleanHTML(minQual))
|
||||
sb.WriteString("\n\n")
|
||||
}
|
||||
if prefQual := getString(jobDetails, "preferredQualifications", ""); prefQual != "" {
|
||||
sb.WriteString("## Preferred Qualifications\n\n")
|
||||
sb.WriteString(cleanHTML(prefQual))
|
||||
sb.WriteString("\n\n")
|
||||
}
|
||||
|
||||
return &crawler.ExtractedJob{
|
||||
CanonicalID: fmt.Sprintf("apple:%s", jobID),
|
||||
Title: title,
|
||||
Company: company,
|
||||
Location: location,
|
||||
ApplyURL: applyURL,
|
||||
RawContent: sb.String(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *AppleCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create HTTP request for %s: %w", urlStr, err)
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
|
||||
resp, err := a.client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("HTTP fetch failed for %s: %w", urlStr, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
logger.Log.Warnw("Apple crawler HTTP error", "status", resp.StatusCode, "url", urlStr)
|
||||
return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read response body for %s: %w", urlStr, err)
|
||||
}
|
||||
|
||||
return string(body), nil
|
||||
}
|
||||
|
||||
func parseIDFromURL(u string) string {
|
||||
re := regexp.MustCompile(`/details/(\d+)`)
|
||||
m := re.FindStringSubmatch(u)
|
||||
if len(m) >= 2 {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func getString(m map[string]interface{}, key, fallback string) string {
|
||||
if val, ok := m[key]; ok {
|
||||
if s, ok := val.(string); ok && s != "" {
|
||||
return s
|
||||
}
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func cleanHTML(s string) string {
|
||||
s = strings.ReplaceAll(s, "<b>", "**")
|
||||
s = strings.ReplaceAll(s, "</b>", "**")
|
||||
s = strings.ReplaceAll(s, "<strong>", "**")
|
||||
s = strings.ReplaceAll(s, "</strong>", "**")
|
||||
s = strings.ReplaceAll(s, "<br>", "\n")
|
||||
s = strings.ReplaceAll(s, "<br/>", "\n")
|
||||
s = strings.ReplaceAll(s, "<li>", "- ")
|
||||
s = strings.ReplaceAll(s, "</li>", "\n")
|
||||
s = strings.ReplaceAll(s, "<ul>", "\n")
|
||||
s = strings.ReplaceAll(s, "</ul>", "\n")
|
||||
s = strings.ReplaceAll(s, "<p>", "\n")
|
||||
s = strings.ReplaceAll(s, "</p>", "\n")
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
package apple
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAppleCrawler_EndToEnd(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping end-to-end network test in short mode")
|
||||
}
|
||||
|
||||
crawler := NewAppleCrawler()
|
||||
|
||||
// Use the default test URL
|
||||
targetURL := "https://jobs.apple.com/en-us/search?location=seattle-SEA&key=Software%2520Engineer"
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
// 1. Test DiscoverJobs
|
||||
t.Logf("Discovering jobs from: %s", targetURL)
|
||||
jobs, err := crawler.DiscoverJobs(ctx, targetURL)
|
||||
if err != nil {
|
||||
t.Fatalf("DiscoverJobs failed: %v", err)
|
||||
}
|
||||
|
||||
if len(jobs) == 0 {
|
||||
t.Fatalf("DiscoverJobs returned 0 jobs for URL: %s", targetURL)
|
||||
}
|
||||
t.Logf("Successfully discovered %d jobs", len(jobs))
|
||||
|
||||
// 2. Pick the first job and test FetchJobDetail
|
||||
testJob := jobs[0]
|
||||
t.Logf("Fetching details for job: %s", testJob.ApplyURL)
|
||||
|
||||
details, err := crawler.FetchJobDetail(ctx, testJob.ApplyURL)
|
||||
if err != nil {
|
||||
t.Fatalf("FetchJobDetail failed: %v", err)
|
||||
}
|
||||
|
||||
if details == nil {
|
||||
t.Fatal("FetchJobDetail returned nil ExtractedJob")
|
||||
}
|
||||
|
||||
// 3. Verify extracted details
|
||||
t.Logf("Extracted Title: %s", details.Title)
|
||||
t.Logf("Extracted Company: %s", details.Company)
|
||||
t.Logf("Extracted Location: %s", details.Location)
|
||||
t.Logf("Raw Content Length: %d", len(details.RawContent))
|
||||
|
||||
if details.Title == "" || details.Title == "Unknown Apple Position" {
|
||||
t.Errorf("Expected valid job title, got %q", details.Title)
|
||||
}
|
||||
if details.Company != "Apple" {
|
||||
t.Errorf("Expected company 'Apple', got %q", details.Company)
|
||||
}
|
||||
if !strings.HasPrefix(details.CanonicalID, "apple:") {
|
||||
t.Errorf("Expected CanonicalID to start with 'apple:', got %q", details.CanonicalID)
|
||||
}
|
||||
if len(details.RawContent) < 100 {
|
||||
t.Errorf("Expected RawContent to be substantive, got length %d", len(details.RawContent))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user