feat: implement multiple new job crawlers and add corresponding test suites

2026-08-25 00:12:52 -07:00
parent 0de312edbd
commit 55d4f4a0d9
26 changed files with 4045 additions and 15 deletions
@@ -0,0 +1,275 @@
+package microsoft
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/hireme/hireme/internal/crawler"
+	"github.com/hireme/hireme/internal/logger"
+)
+
+type MicrosoftCrawler struct {
+	client *http.Client
+}
+
+func NewMicrosoftCrawler() *MicrosoftCrawler {
+	return NewMicrosoftCrawlerWithClient(&http.Client{
+		Timeout: 30 * time.Second,
+	})
+}
+
+func NewMicrosoftCrawlerWithClient(client *http.Client) *MicrosoftCrawler {
+	return &MicrosoftCrawler{
+		client: client,
+	}
+}
+
+func (c *MicrosoftCrawler) CanHandle(platform string) bool {
+	p := strings.ToLower(platform)
+	return p == "microsoft" || p == "azure" || strings.Contains(p, "careers.microsoft.com")
+}
+
+type msSearchResponse struct {
+	OperationResult struct {
+		Result struct {
+			TotalJobs int        `json:"totalJobs"`
+			Jobs      []msJobItem `json:"jobs"`
+		} `json:"result"`
+	} `json:"operationResult"`
+}
+
+type msJobDetailResponse struct {
+	OperationResult struct {
+		Result msJobItem `json:"result"`
+	} `json:"operationResult"`
+}
+
+type msJobItem struct {
+	JobID          string   `json:"jobId"`
+	Title          string   `json:"title"`
+	Description    string   `json:"description"`
+	Qualifications string   `json:"qualifications"`
+	Responsibilities string `json:"responsibilities"`
+	PostingDate    string   `json:"postingDate"`
+	Properties     struct {
+		Locations          []string `json:"locations"`
+		PrimaryLocation    string   `json:"primaryLocation"`
+		WorkSiteFlexibility string  `json:"workSiteFlexibility"`
+		Profession         string   `json:"profession"`
+		RoleType           string   `json:"roleType"`
+		Discipline         string   `json:"discipline"`
+	} `json:"properties"`
+}
+
+func (c *MicrosoftCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
+	seen := make(map[string]bool)
+	var discovered []crawler.DiscoveredJob
+
+	searchAPI := targetURL
+	if !strings.Contains(searchAPI, "gcsservices.careers.microsoft.com") {
+		u, err := url.Parse(targetURL)
+		if err == nil && u.Host != "" {
+			q := u.Query()
+			searchAPI = fmt.Sprintf("https://gcsservices.careers.microsoft.com/search/api/v1/search?%s", q.Encode())
+		} else {
+			searchAPI = fmt.Sprintf("https://gcsservices.careers.microsoft.com/search/api/v1/search?q=%s&lc=Redmond%%2C+Washington%%2C+United+States&l=en_us", url.QueryEscape(targetURL))
+		}
+	}
+
+	pageSize := 50
+	for page := 1; page <= 5; page++ {
+		reqURL := searchAPI
+		if strings.Contains(reqURL, "?") {
+			reqURL = fmt.Sprintf("%s&pg=%d&pgSz=%d", searchAPI, page, pageSize)
+		} else {
+			reqURL = fmt.Sprintf("%s?pg=%d&pgSz=%d", searchAPI, page, pageSize)
+		}
+
+		body, err := c.fetchURL(ctx, reqURL)
+		if err != nil {
+			if page == 1 {
+				return nil, err
+			}
+			break
+		}
+
+		var resp msSearchResponse
+		if err := json.Unmarshal([]byte(body), &resp); err != nil {
+			if page == 1 {
+				return nil, fmt.Errorf("failed to parse Microsoft search response: %w", err)
+			}
+			break
+		}
+
+		jobs := resp.OperationResult.Result.Jobs
+		if len(jobs) == 0 {
+			break
+		}
+
+		newFound := false
+		for _, job := range jobs {
+			if job.JobID == "" {
+				continue
+			}
+
+			applyURL := fmt.Sprintf("https://jobs.careers.microsoft.com/global/en/share/%s", job.JobID)
+			if !seen[applyURL] {
+				seen[applyURL] = true
+				newFound = true
+				discovered = append(discovered, crawler.DiscoveredJob{
+					CanonicalID: fmt.Sprintf("microsoft:%s", job.JobID),
+					ApplyURL:    applyURL,
+				})
+			}
+		}
+
+		if !newFound || len(jobs) < pageSize {
+			break
+		}
+	}
+
+	logger.Log.Infow("Microsoft discovery finished", "found_count", len(discovered))
+	return discovered, nil
+}
+
+func (c *MicrosoftCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
+	jobID := parseMicrosoftJobID(applyURL)
+	if jobID == "" {
+		jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16]
+	}
+
+	detailURL := fmt.Sprintf("https://gcsservices.careers.microsoft.com/search/api/v1/job/%s?lang=en_us", jobID)
+	body, err := c.fetchURL(ctx, detailURL)
+	if err == nil && json.Valid([]byte(body)) {
+		var detailResp msJobDetailResponse
+		if err := json.Unmarshal([]byte(body), &detailResp); err == nil && detailResp.OperationResult.Result.Title != "" {
+			j := detailResp.OperationResult.Result
+
+			var sb strings.Builder
+			if j.Description != "" {
+				sb.WriteString("## Overview\n\n")
+				sb.WriteString(cleanHTML(j.Description))
+				sb.WriteString("\n\n")
+			}
+			if j.Responsibilities != "" {
+				sb.WriteString("## Responsibilities\n\n")
+				sb.WriteString(cleanHTML(j.Responsibilities))
+				sb.WriteString("\n\n")
+			}
+			if j.Qualifications != "" {
+				sb.WriteString("## Qualifications\n\n")
+				sb.WriteString(cleanHTML(j.Qualifications))
+				sb.WriteString("\n\n")
+			}
+
+			loc := j.Properties.PrimaryLocation
+			if loc == "" && len(j.Properties.Locations) > 0 {
+				loc = strings.Join(j.Properties.Locations, "; ")
+			}
+			if loc == "" {
+				loc = "Redmond, WA, USA"
+			}
+
+			return &crawler.ExtractedJob{
+				CanonicalID: fmt.Sprintf("microsoft:%s", jobID),
+				Title:       j.Title,
+				Company:     "Microsoft",
+				Location:    loc,
+				ApplyURL:    applyURL,
+				RawContent:  sb.String(),
+			}, nil
+		}
+	}
+
+	// Fallback to HTML
+	htmlBody, err := c.fetchURL(ctx, applyURL)
+	if err != nil {
+		return nil, err
+	}
+
+	title := extractRegex(htmlBody, `<h1[^>]*>(.*?)</h1>`)
+	if title == "" {
+		title = "Software Engineer - Microsoft"
+	}
+
+	return &crawler.ExtractedJob{
+		CanonicalID: fmt.Sprintf("microsoft:%s", jobID),
+		Title:       cleanHTML(title),
+		Company:     "Microsoft",
+		Location:    "Redmond, WA, USA",
+		ApplyURL:    applyURL,
+		RawContent:  cleanHTML(htmlBody),
+	}, nil
+}
+
+func (c *MicrosoftCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to create request: %w", err)
+	}
+
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+	req.Header.Set("Accept", "application/json, text/html, */*")
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("HTTP GET failed: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read response: %w", err)
+	}
+
+	return string(body), nil
+}
+
+func parseMicrosoftJobID(u string) string {
+	re := regexp.MustCompile(`/(?:share|job)/([0-9A-Za-z]+)`)
+	m := re.FindStringSubmatch(u)
+	if len(m) >= 2 {
+		return m[1]
+	}
+	return ""
+}
+
+func extractRegex(s, pattern string) string {
+	re := regexp.MustCompile(pattern)
+	m := re.FindStringSubmatch(s)
+	if len(m) >= 2 {
+		return m[1]
+	}
+	return ""
+}
+
+func cleanHTML(s string) string {
+	s = strings.ReplaceAll(s, "<b>", "**")
+	s = strings.ReplaceAll(s, "</b>", "**")
+	s = strings.ReplaceAll(s, "<strong>", "**")
+	s = strings.ReplaceAll(s, "</strong>", "**")
+	s = strings.ReplaceAll(s, "<br>", "\n")
+	s = strings.ReplaceAll(s, "<br/>", "\n")
+	s = strings.ReplaceAll(s, "<br />", "\n")
+	s = strings.ReplaceAll(s, "<li>", "- ")
+	s = strings.ReplaceAll(s, "</li>", "\n")
+	s = strings.ReplaceAll(s, "<ul>", "\n")
+	s = strings.ReplaceAll(s, "</ul>", "\n")
+	s = strings.ReplaceAll(s, "<p>", "\n")
+	s = strings.ReplaceAll(s, "</p>", "\n")
+	reTags := regexp.MustCompile(`<[^>]*>`)
+	s = reTags.ReplaceAllString(s, "")
+	return strings.TrimSpace(s)
+}
@@ -0,0 +1,121 @@
+package microsoft
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/hireme/hireme/internal/crawler/crawlertest"
+)
+
+type roundTripFunc func(req *http.Request) *http.Response
+
+func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req), nil
+}
+
+func TestMicrosoftCrawler_DiscoverAndFetch(t *testing.T) {
+	mockClient := &http.Client{
+		Transport: roundTripFunc(func(r *http.Request) *http.Response {
+			if strings.Contains(r.URL.Path, "/job/1789001") {
+				body := `{
+					"operationResult": {
+						"result": {
+							"jobId": "1789001",
+							"title": "Software Engineer II - Azure Core",
+							"description": "<p>Azure Core is building the backbone of cloud compute.</p>",
+							"responsibilities": "<li>Design and implement scalable distributed services.</li>",
+							"qualifications": "<li>Bachelor degree and 2+ years of software development experience.</li>",
+							"properties": {
+								"primaryLocation": "Redmond, WA, United States"
+							}
+						}
+					}
+				}`
+				return &http.Response{
+					StatusCode: http.StatusOK,
+					Body:       io.NopCloser(bytes.NewBufferString(body)),
+					Header:     make(http.Header),
+				}
+			}
+
+			if strings.HasSuffix(r.URL.Path, "/search") || strings.Contains(r.URL.RawQuery, "pgSz") {
+				body := `{
+					"operationResult": {
+						"result": {
+							"totalJobs": 1,
+							"jobs": [
+								{
+									"jobId": "1789001",
+									"title": "Software Engineer II - Azure Core",
+									"properties": {
+										"primaryLocation": "Redmond, WA, United States",
+										"discipline": "Software Engineering"
+									}
+								}
+							]
+						}
+					}
+				}`
+				return &http.Response{
+					StatusCode: http.StatusOK,
+					Body:       io.NopCloser(bytes.NewBufferString(body)),
+					Header:     make(http.Header),
+				}
+			}
+
+			return &http.Response{
+				StatusCode: http.StatusNotFound,
+				Body:       io.NopCloser(bytes.NewBufferString("not found")),
+				Header:     make(http.Header),
+			}
+		}),
+	}
+
+	c := NewMicrosoftCrawlerWithClient(mockClient)
+	if !c.CanHandle("microsoft") || !c.CanHandle("azure") {
+		t.Errorf("CanHandle should support microsoft and azure")
+	}
+
+	discovered, err := c.DiscoverJobs(context.Background(), "https://gcsservices.careers.microsoft.com/search/api/v1/search?q=Software+Engineer")
+	if err != nil {
+		t.Fatalf("DiscoverJobs failed: %v", err)
+	}
+
+	if len(discovered) != 1 {
+		t.Fatalf("expected 1 discovered job, got %d", len(discovered))
+	}
+
+	if discovered[0].CanonicalID != "microsoft:1789001" {
+		t.Errorf("unexpected canonical ID: %s", discovered[0].CanonicalID)
+	}
+
+	extracted, err := c.FetchJobDetail(context.Background(), "https://jobs.careers.microsoft.com/global/en/share/1789001")
+	if err != nil {
+		t.Fatalf("FetchJobDetail failed: %v", err)
+	}
+
+	if extracted.Title != "Software Engineer II - Azure Core" {
+		t.Errorf("unexpected title: %s", extracted.Title)
+	}
+	if extracted.Company != "Microsoft" {
+		t.Errorf("unexpected company: %s", extracted.Company)
+	}
+	if extracted.Location != "Redmond, WA, United States" {
+		t.Errorf("unexpected location: %s", extracted.Location)
+	}
+}
+
+func TestMicrosoftCrawler_LiveIntegration(t *testing.T) {
+	crawlertest.RunLiveIntegration(t, crawlertest.LiveTestConfig{
+		PlatformName:            "Microsoft",
+		Crawler:                 NewMicrosoftCrawler(),
+		TargetURL:               "https://gcsservices.careers.microsoft.com/search/api/v1/search?q=Software+Engineer&lc=Redmond%2C+Washington%2C+United+States&l=en_us",
+		ExpectedCanonicalPrefix: "microsoft:",
+		ExpectedCompany:         "Microsoft",
+	})
+}
+