276 lines
7.4 KiB
Go
276 lines
7.4 KiB
Go
package microsoft
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.pengzhan.dev/haopengzhan/hireme/internal/crawler"
|
|
"git.pengzhan.dev/haopengzhan/hireme/internal/logger"
|
|
)
|
|
|
|
type MicrosoftCrawler struct {
|
|
client *http.Client
|
|
}
|
|
|
|
func NewMicrosoftCrawler() *MicrosoftCrawler {
|
|
return NewMicrosoftCrawlerWithClient(&http.Client{
|
|
Timeout: 30 * time.Second,
|
|
})
|
|
}
|
|
|
|
func NewMicrosoftCrawlerWithClient(client *http.Client) *MicrosoftCrawler {
|
|
return &MicrosoftCrawler{
|
|
client: client,
|
|
}
|
|
}
|
|
|
|
func (c *MicrosoftCrawler) CanHandle(platform string) bool {
|
|
p := strings.ToLower(platform)
|
|
return p == "microsoft" || p == "azure" || strings.Contains(p, "careers.microsoft.com")
|
|
}
|
|
|
|
type msSearchResponse struct {
|
|
OperationResult struct {
|
|
Result struct {
|
|
TotalJobs int `json:"totalJobs"`
|
|
Jobs []msJobItem `json:"jobs"`
|
|
} `json:"result"`
|
|
} `json:"operationResult"`
|
|
}
|
|
|
|
type msJobDetailResponse struct {
|
|
OperationResult struct {
|
|
Result msJobItem `json:"result"`
|
|
} `json:"operationResult"`
|
|
}
|
|
|
|
type msJobItem struct {
|
|
JobID string `json:"jobId"`
|
|
Title string `json:"title"`
|
|
Description string `json:"description"`
|
|
Qualifications string `json:"qualifications"`
|
|
Responsibilities string `json:"responsibilities"`
|
|
PostingDate string `json:"postingDate"`
|
|
Properties struct {
|
|
Locations []string `json:"locations"`
|
|
PrimaryLocation string `json:"primaryLocation"`
|
|
WorkSiteFlexibility string `json:"workSiteFlexibility"`
|
|
Profession string `json:"profession"`
|
|
RoleType string `json:"roleType"`
|
|
Discipline string `json:"discipline"`
|
|
} `json:"properties"`
|
|
}
|
|
|
|
func (c *MicrosoftCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
|
|
seen := make(map[string]bool)
|
|
var discovered []crawler.DiscoveredJob
|
|
|
|
searchAPI := targetURL
|
|
if !strings.Contains(searchAPI, "gcsservices.careers.microsoft.com") {
|
|
u, err := url.Parse(targetURL)
|
|
if err == nil && u.Host != "" {
|
|
q := u.Query()
|
|
searchAPI = fmt.Sprintf("https://gcsservices.careers.microsoft.com/search/api/v1/search?%s", q.Encode())
|
|
} else {
|
|
searchAPI = fmt.Sprintf("https://gcsservices.careers.microsoft.com/search/api/v1/search?q=%s&lc=Redmond%%2C+Washington%%2C+United+States&l=en_us", url.QueryEscape(targetURL))
|
|
}
|
|
}
|
|
|
|
pageSize := 50
|
|
for page := 1; page <= 5; page++ {
|
|
reqURL := searchAPI
|
|
if strings.Contains(reqURL, "?") {
|
|
reqURL = fmt.Sprintf("%s&pg=%d&pgSz=%d", searchAPI, page, pageSize)
|
|
} else {
|
|
reqURL = fmt.Sprintf("%s?pg=%d&pgSz=%d", searchAPI, page, pageSize)
|
|
}
|
|
|
|
body, err := c.fetchURL(ctx, reqURL)
|
|
if err != nil {
|
|
if page == 1 {
|
|
return nil, err
|
|
}
|
|
break
|
|
}
|
|
|
|
var resp msSearchResponse
|
|
if err := json.Unmarshal([]byte(body), &resp); err != nil {
|
|
if page == 1 {
|
|
return nil, fmt.Errorf("failed to parse Microsoft search response: %w", err)
|
|
}
|
|
break
|
|
}
|
|
|
|
jobs := resp.OperationResult.Result.Jobs
|
|
if len(jobs) == 0 {
|
|
break
|
|
}
|
|
|
|
newFound := false
|
|
for _, job := range jobs {
|
|
if job.JobID == "" {
|
|
continue
|
|
}
|
|
|
|
applyURL := fmt.Sprintf("https://jobs.careers.microsoft.com/global/en/share/%s", job.JobID)
|
|
if !seen[applyURL] {
|
|
seen[applyURL] = true
|
|
newFound = true
|
|
discovered = append(discovered, crawler.DiscoveredJob{
|
|
CanonicalID: fmt.Sprintf("microsoft:%s", job.JobID),
|
|
ApplyURL: applyURL,
|
|
})
|
|
}
|
|
}
|
|
|
|
if !newFound || len(jobs) < pageSize {
|
|
break
|
|
}
|
|
}
|
|
|
|
logger.Log.Infow("Microsoft discovery finished", "found_count", len(discovered))
|
|
return discovered, nil
|
|
}
|
|
|
|
func (c *MicrosoftCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
|
|
jobID := parseMicrosoftJobID(applyURL)
|
|
if jobID == "" {
|
|
jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16]
|
|
}
|
|
|
|
detailURL := fmt.Sprintf("https://gcsservices.careers.microsoft.com/search/api/v1/job/%s?lang=en_us", jobID)
|
|
body, err := c.fetchURL(ctx, detailURL)
|
|
if err == nil && json.Valid([]byte(body)) {
|
|
var detailResp msJobDetailResponse
|
|
if err := json.Unmarshal([]byte(body), &detailResp); err == nil && detailResp.OperationResult.Result.Title != "" {
|
|
j := detailResp.OperationResult.Result
|
|
|
|
var sb strings.Builder
|
|
if j.Description != "" {
|
|
sb.WriteString("## Overview\n\n")
|
|
sb.WriteString(cleanHTML(j.Description))
|
|
sb.WriteString("\n\n")
|
|
}
|
|
if j.Responsibilities != "" {
|
|
sb.WriteString("## Responsibilities\n\n")
|
|
sb.WriteString(cleanHTML(j.Responsibilities))
|
|
sb.WriteString("\n\n")
|
|
}
|
|
if j.Qualifications != "" {
|
|
sb.WriteString("## Qualifications\n\n")
|
|
sb.WriteString(cleanHTML(j.Qualifications))
|
|
sb.WriteString("\n\n")
|
|
}
|
|
|
|
loc := j.Properties.PrimaryLocation
|
|
if loc == "" && len(j.Properties.Locations) > 0 {
|
|
loc = strings.Join(j.Properties.Locations, "; ")
|
|
}
|
|
if loc == "" {
|
|
loc = "Redmond, WA, USA"
|
|
}
|
|
|
|
return &crawler.ExtractedJob{
|
|
CanonicalID: fmt.Sprintf("microsoft:%s", jobID),
|
|
Title: j.Title,
|
|
Company: "Microsoft",
|
|
Location: loc,
|
|
ApplyURL: applyURL,
|
|
RawContent: sb.String(),
|
|
}, nil
|
|
}
|
|
}
|
|
|
|
// Fallback to HTML
|
|
htmlBody, err := c.fetchURL(ctx, applyURL)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
title := extractRegex(htmlBody, `<h1[^>]*>(.*?)</h1>`)
|
|
if title == "" {
|
|
title = "Software Engineer - Microsoft"
|
|
}
|
|
|
|
return &crawler.ExtractedJob{
|
|
CanonicalID: fmt.Sprintf("microsoft:%s", jobID),
|
|
Title: cleanHTML(title),
|
|
Company: "Microsoft",
|
|
Location: "Redmond, WA, USA",
|
|
ApplyURL: applyURL,
|
|
RawContent: cleanHTML(htmlBody),
|
|
}, nil
|
|
}
|
|
|
|
func (c *MicrosoftCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) {
|
|
req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to create request: %w", err)
|
|
}
|
|
|
|
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
|
req.Header.Set("Accept", "application/json, text/html, */*")
|
|
|
|
resp, err := c.client.Do(req)
|
|
if err != nil {
|
|
return "", fmt.Errorf("HTTP GET failed: %w", err)
|
|
}
|
|
defer func() { _ = resp.Body.Close() }()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr)
|
|
}
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return "", fmt.Errorf("failed to read response: %w", err)
|
|
}
|
|
|
|
return string(body), nil
|
|
}
|
|
|
|
func parseMicrosoftJobID(u string) string {
|
|
re := regexp.MustCompile(`/(?:share|job)/([0-9A-Za-z]+)`)
|
|
m := re.FindStringSubmatch(u)
|
|
if len(m) >= 2 {
|
|
return m[1]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func extractRegex(s, pattern string) string {
|
|
re := regexp.MustCompile(pattern)
|
|
m := re.FindStringSubmatch(s)
|
|
if len(m) >= 2 {
|
|
return m[1]
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func cleanHTML(s string) string {
|
|
s = strings.ReplaceAll(s, "<b>", "**")
|
|
s = strings.ReplaceAll(s, "</b>", "**")
|
|
s = strings.ReplaceAll(s, "<strong>", "**")
|
|
s = strings.ReplaceAll(s, "</strong>", "**")
|
|
s = strings.ReplaceAll(s, "<br>", "\n")
|
|
s = strings.ReplaceAll(s, "<br/>", "\n")
|
|
s = strings.ReplaceAll(s, "<br />", "\n")
|
|
s = strings.ReplaceAll(s, "<li>", "- ")
|
|
s = strings.ReplaceAll(s, "</li>", "\n")
|
|
s = strings.ReplaceAll(s, "<ul>", "\n")
|
|
s = strings.ReplaceAll(s, "</ul>", "\n")
|
|
s = strings.ReplaceAll(s, "<p>", "\n")
|
|
s = strings.ReplaceAll(s, "</p>", "\n")
|
|
reTags := regexp.MustCompile(`<[^>]*>`)
|
|
s = reTags.ReplaceAllString(s, "")
|
|
return strings.TrimSpace(s)
|
|
}
|