package microsoft import ( "context" "crypto/sha256" "encoding/json" "fmt" "io" "net/http" "net/url" "regexp" "strings" "time" "github.com/hireme/hireme/internal/crawler" "github.com/hireme/hireme/internal/logger" ) type MicrosoftCrawler struct { client *http.Client } func NewMicrosoftCrawler() *MicrosoftCrawler { return NewMicrosoftCrawlerWithClient(&http.Client{ Timeout: 30 * time.Second, }) } func NewMicrosoftCrawlerWithClient(client *http.Client) *MicrosoftCrawler { return &MicrosoftCrawler{ client: client, } } func (c *MicrosoftCrawler) CanHandle(platform string) bool { p := strings.ToLower(platform) return p == "microsoft" || p == "azure" || strings.Contains(p, "careers.microsoft.com") } type msSearchResponse struct { OperationResult struct { Result struct { TotalJobs int `json:"totalJobs"` Jobs []msJobItem `json:"jobs"` } `json:"result"` } `json:"operationResult"` } type msJobDetailResponse struct { OperationResult struct { Result msJobItem `json:"result"` } `json:"operationResult"` } type msJobItem struct { JobID string `json:"jobId"` Title string `json:"title"` Description string `json:"description"` Qualifications string `json:"qualifications"` Responsibilities string `json:"responsibilities"` PostingDate string `json:"postingDate"` Properties struct { Locations []string `json:"locations"` PrimaryLocation string `json:"primaryLocation"` WorkSiteFlexibility string `json:"workSiteFlexibility"` Profession string `json:"profession"` RoleType string `json:"roleType"` Discipline string `json:"discipline"` } `json:"properties"` } func (c *MicrosoftCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { seen := make(map[string]bool) var discovered []crawler.DiscoveredJob searchAPI := targetURL if !strings.Contains(searchAPI, "gcsservices.careers.microsoft.com") { u, err := url.Parse(targetURL) if err == nil && u.Host != "" { q := u.Query() searchAPI = fmt.Sprintf("https://gcsservices.careers.microsoft.com/search/api/v1/search?%s", q.Encode()) } else { searchAPI = fmt.Sprintf("https://gcsservices.careers.microsoft.com/search/api/v1/search?q=%s&lc=Redmond%%2C+Washington%%2C+United+States&l=en_us", url.QueryEscape(targetURL)) } } pageSize := 50 for page := 1; page <= 5; page++ { reqURL := searchAPI if strings.Contains(reqURL, "?") { reqURL = fmt.Sprintf("%s&pg=%d&pgSz=%d", searchAPI, page, pageSize) } else { reqURL = fmt.Sprintf("%s?pg=%d&pgSz=%d", searchAPI, page, pageSize) } body, err := c.fetchURL(ctx, reqURL) if err != nil { if page == 1 { return nil, err } break } var resp msSearchResponse if err := json.Unmarshal([]byte(body), &resp); err != nil { if page == 1 { return nil, fmt.Errorf("failed to parse Microsoft search response: %w", err) } break } jobs := resp.OperationResult.Result.Jobs if len(jobs) == 0 { break } newFound := false for _, job := range jobs { if job.JobID == "" { continue } applyURL := fmt.Sprintf("https://jobs.careers.microsoft.com/global/en/share/%s", job.JobID) if !seen[applyURL] { seen[applyURL] = true newFound = true discovered = append(discovered, crawler.DiscoveredJob{ CanonicalID: fmt.Sprintf("microsoft:%s", job.JobID), ApplyURL: applyURL, }) } } if !newFound || len(jobs) < pageSize { break } } logger.Log.Infow("Microsoft discovery finished", "found_count", len(discovered)) return discovered, nil } func (c *MicrosoftCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { jobID := parseMicrosoftJobID(applyURL) if jobID == "" { jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16] } detailURL := fmt.Sprintf("https://gcsservices.careers.microsoft.com/search/api/v1/job/%s?lang=en_us", jobID) body, err := c.fetchURL(ctx, detailURL) if err == nil && json.Valid([]byte(body)) { var detailResp msJobDetailResponse if err := json.Unmarshal([]byte(body), &detailResp); err == nil && detailResp.OperationResult.Result.Title != "" { j := detailResp.OperationResult.Result var sb strings.Builder if j.Description != "" { sb.WriteString("## Overview\n\n") sb.WriteString(cleanHTML(j.Description)) sb.WriteString("\n\n") } if j.Responsibilities != "" { sb.WriteString("## Responsibilities\n\n") sb.WriteString(cleanHTML(j.Responsibilities)) sb.WriteString("\n\n") } if j.Qualifications != "" { sb.WriteString("## Qualifications\n\n") sb.WriteString(cleanHTML(j.Qualifications)) sb.WriteString("\n\n") } loc := j.Properties.PrimaryLocation if loc == "" && len(j.Properties.Locations) > 0 { loc = strings.Join(j.Properties.Locations, "; ") } if loc == "" { loc = "Redmond, WA, USA" } return &crawler.ExtractedJob{ CanonicalID: fmt.Sprintf("microsoft:%s", jobID), Title: j.Title, Company: "Microsoft", Location: loc, ApplyURL: applyURL, RawContent: sb.String(), }, nil } } // Fallback to HTML htmlBody, err := c.fetchURL(ctx, applyURL) if err != nil { return nil, err } title := extractRegex(htmlBody, `
", "\n") s = strings.ReplaceAll(s, "
", "\n") reTags := regexp.MustCompile(`<[^>]*>`) s = reTags.ReplaceAllString(s, "") return strings.TrimSpace(s) }