package linkedin import ( "context" "crypto/sha256" "fmt" "io" "net/http" "net/url" "regexp" "strings" "time" "git.pengzhan.dev/haopengzhan/hireme/internal/crawler" "git.pengzhan.dev/haopengzhan/hireme/internal/logger" ) type LinkedInCrawler struct { client *http.Client } func NewLinkedInCrawler() *LinkedInCrawler { return NewLinkedInCrawlerWithClient(&http.Client{ Timeout: 30 * time.Second, }) } func NewLinkedInCrawlerWithClient(client *http.Client) *LinkedInCrawler { return &LinkedInCrawler{ client: client, } } func (c *LinkedInCrawler) CanHandle(platform string) bool { p := strings.ToLower(platform) return p == "linkedin" || strings.Contains(p, "linkedin.com/jobs") } func (c *LinkedInCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { seen := make(map[string]bool) var discovered []crawler.DiscoveredJob searchAPI := targetURL if !strings.Contains(searchAPI, "jobs-guest/jobs/api/seeMoreJobPostings/search") { u, err := url.Parse(targetURL) if err == nil && u.Host != "" { q := u.Query() searchAPI = fmt.Sprintf("https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search?%s", q.Encode()) } else { searchAPI = fmt.Sprintf("https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search?keywords=%s&location=Seattle%%2C+Washington%%2C+United+States", url.QueryEscape(targetURL)) } } pageSize := 25 reID := regexp.MustCompile(`(?:urn:li:jobPosting:(\d+)|/jobs/view/(?:[a-z0-9\-]+-)?(\d+))`) for page := 0; page < 4; page++ { start := page * pageSize reqURL := searchAPI if strings.Contains(reqURL, "?") { reqURL = fmt.Sprintf("%s&start=%d", searchAPI, start) } else { reqURL = fmt.Sprintf("%s?start=%d", searchAPI, start) } body, err := c.fetchURL(ctx, reqURL) if err != nil { if page == 0 { return nil, err } break } matches := reID.FindAllStringSubmatch(body, -1) if len(matches) == 0 { break } newFound := false for _, m := range matches { jobID := m[1] if jobID == "" && len(m) >= 3 { jobID = m[2] } if jobID == "" { continue } applyURL := fmt.Sprintf("https://www.linkedin.com/jobs/view/%s", jobID) if !seen[applyURL] { seen[applyURL] = true newFound = true discovered = append(discovered, crawler.DiscoveredJob{ CanonicalID: fmt.Sprintf("linkedin:%s", jobID), ApplyURL: applyURL, }) } } if !newFound || len(matches) < pageSize { break } } logger.Log.Infow("LinkedIn discovery finished", "found_count", len(discovered)) return discovered, nil } func (c *LinkedInCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { jobID := parseLinkedInJobID(applyURL) if jobID == "" { jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16] } // LinkedIn guest API endpoint for job details: guestDetailURL := fmt.Sprintf("https://www.linkedin.com/jobs-guest/jobs/api/jobPosting/%s", jobID) body, err := c.fetchURL(ctx, guestDetailURL) if err != nil { body, err = c.fetchURL(ctx, applyURL) if err != nil { return nil, err } } title := extractRegex(body, `]*class="[^"]*top-card-layout__title[^"]*"[^>]*>(.*?)`) if title == "" { title = extractRegex(body, `]*>(.*?)`) } if title == "" { title = "Software Engineer" } company := extractRegex(body, `]*class="[^"]*topcard__org-name-link[^"]*"[^>]*>(.*?)`) if company == "" { company = extractRegex(body, `]*class="[^"]*topcard__flavor[^"]*"[^>]*>(.*?)`) } if company == "" { company = "LinkedIn" } location := extractRegex(body, `]*class="[^"]*topcard__flavor--bullet[^"]*"[^>]*>(.*?)`) if location == "" { location = "Seattle, WA, USA" } desc := extractRegex(body, `]*class="[^"]*show-more-less-html__markup[^"]*"[^>]*>([\s\S]*?)`) if desc == "" { desc = body } return &crawler.ExtractedJob{ CanonicalID: fmt.Sprintf("linkedin:%s", jobID), Title: cleanHTML(title), Company: cleanHTML(company), Location: cleanHTML(location), ApplyURL: applyURL, RawContent: cleanHTML(desc), }, nil } func (c *LinkedInCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) { req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) if err != nil { return "", fmt.Errorf("failed to create request: %w", err) } req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") req.Header.Set("Accept-Language", "en-US,en;q=0.9") resp, err := c.client.Do(req) if err != nil { return "", fmt.Errorf("HTTP GET failed: %w", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr) } body, err := io.ReadAll(resp.Body) if err != nil { return "", fmt.Errorf("failed to read response: %w", err) } return string(body), nil } func parseLinkedInJobID(u string) string { re := regexp.MustCompile(`/jobs/view/(?:[a-z0-9\-]+-)?(\d+)`) m := re.FindStringSubmatch(u) if len(m) >= 2 { return m[1] } re2 := regexp.MustCompile(`/jobPosting/(\d+)`) m2 := re2.FindStringSubmatch(u) if len(m2) >= 2 { return m2[1] } return "" } func extractRegex(s, pattern string) string { re := regexp.MustCompile(pattern) m := re.FindStringSubmatch(s) if len(m) >= 2 { return m[1] } return "" } func cleanHTML(s string) string { s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "
", "\n") s = strings.ReplaceAll(s, "
", "\n") s = strings.ReplaceAll(s, "
", "\n") s = strings.ReplaceAll(s, "
  • ", "- ") s = strings.ReplaceAll(s, "
  • ", "\n") s = strings.ReplaceAll(s, "
      ", "\n") s = strings.ReplaceAll(s, "
    ", "\n") s = strings.ReplaceAll(s, "

    ", "\n") s = strings.ReplaceAll(s, "

    ", "\n") reTags := regexp.MustCompile(`<[^>]*>`) s = reTags.ReplaceAllString(s, "") return strings.TrimSpace(s) }