hireme/pkg/crawler/linkedin/linkedin.go

package linkedin

import (
	"context"
	"crypto/sha256"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"

	"git.pengzhan.dev/haopengzhan/hireme/internal/crawler"
	"git.pengzhan.dev/haopengzhan/hireme/internal/logger"
)

type LinkedInCrawler struct {
	client *http.Client
}

func NewLinkedInCrawler() *LinkedInCrawler {
	return NewLinkedInCrawlerWithClient(&http.Client{
		Timeout: 30 * time.Second,
	})
}

func NewLinkedInCrawlerWithClient(client *http.Client) *LinkedInCrawler {
	return &LinkedInCrawler{
		client: client,
	}
}

func (c *LinkedInCrawler) CanHandle(platform string) bool {
	p := strings.ToLower(platform)
	return p == "linkedin" || strings.Contains(p, "linkedin.com/jobs")
}

func (c *LinkedInCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
	seen := make(map[string]bool)
	var discovered []crawler.DiscoveredJob

	searchAPI := targetURL
	if !strings.Contains(searchAPI, "jobs-guest/jobs/api/seeMoreJobPostings/search") {
		u, err := url.Parse(targetURL)
		if err == nil && u.Host != "" {
			q := u.Query()
			searchAPI = fmt.Sprintf("https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search?%s", q.Encode())
		} else {
			searchAPI = fmt.Sprintf("https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search?keywords=%s&location=Seattle%%2C+Washington%%2C+United+States", url.QueryEscape(targetURL))
		}
	}

	pageSize := 25
	reID := regexp.MustCompile(`(?:urn:li:jobPosting:(\d+)|/jobs/view/(?:[a-z0-9\-]+-)?(\d+))`)

	for page := 0; page < 4; page++ {
		start := page * pageSize
		reqURL := searchAPI
		if strings.Contains(reqURL, "?") {
			reqURL = fmt.Sprintf("%s&start=%d", searchAPI, start)
		} else {
			reqURL = fmt.Sprintf("%s?start=%d", searchAPI, start)
		}

		body, err := c.fetchURL(ctx, reqURL)
		if err != nil {
			if page == 0 {
				return nil, err
			}
			break
		}

		matches := reID.FindAllStringSubmatch(body, -1)
		if len(matches) == 0 {
			break
		}

		newFound := false
		for _, m := range matches {
			jobID := m[1]
			if jobID == "" && len(m) >= 3 {
				jobID = m[2]
			}
			if jobID == "" {
				continue
			}

			applyURL := fmt.Sprintf("https://www.linkedin.com/jobs/view/%s", jobID)
			if !seen[applyURL] {
				seen[applyURL] = true
				newFound = true
				discovered = append(discovered, crawler.DiscoveredJob{
					CanonicalID: fmt.Sprintf("linkedin:%s", jobID),
					ApplyURL:    applyURL,
				})
			}
		}

		if !newFound || len(matches) < pageSize {
			break
		}
	}

	logger.Log.Infow("LinkedIn discovery finished", "found_count", len(discovered))
	return discovered, nil
}

func (c *LinkedInCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
	jobID := parseLinkedInJobID(applyURL)
	if jobID == "" {
		jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16]
	}

	// LinkedIn guest API endpoint for job details:
	guestDetailURL := fmt.Sprintf("https://www.linkedin.com/jobs-guest/jobs/api/jobPosting/%s", jobID)
	body, err := c.fetchURL(ctx, guestDetailURL)
	if err != nil {
		body, err = c.fetchURL(ctx, applyURL)
		if err != nil {
			return nil, err
		}
	}

	title := extractRegex(body, `<h2[^>]*class="[^"]*top-card-layout__title[^"]*"[^>]*>(.*?)</h2>`)
	if title == "" {
		title = extractRegex(body, `<h1[^>]*>(.*?)</h1>`)
	}
	if title == "" {
		title = "Software Engineer"
	}

	company := extractRegex(body, `<a[^>]*class="[^"]*topcard__org-name-link[^"]*"[^>]*>(.*?)</a>`)
	if company == "" {
		company = extractRegex(body, `<span[^>]*class="[^"]*topcard__flavor[^"]*"[^>]*>(.*?)</span>`)
	}
	if company == "" {
		company = "LinkedIn"
	}

	location := extractRegex(body, `<span[^>]*class="[^"]*topcard__flavor--bullet[^"]*"[^>]*>(.*?)</span>`)
	if location == "" {
		location = "Seattle, WA, USA"
	}

	desc := extractRegex(body, `<div[^>]*class="[^"]*show-more-less-html__markup[^"]*"[^>]*>([\s\S]*?)</div>`)
	if desc == "" {
		desc = body
	}

	return &crawler.ExtractedJob{
		CanonicalID: fmt.Sprintf("linkedin:%s", jobID),
		Title:       cleanHTML(title),
		Company:     cleanHTML(company),
		Location:    cleanHTML(location),
		ApplyURL:    applyURL,
		RawContent:  cleanHTML(desc),
	}, nil
}

func (c *LinkedInCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) {
	req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
	if err != nil {
		return "", fmt.Errorf("failed to create request: %w", err)
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
	req.Header.Set("Accept-Language", "en-US,en;q=0.9")

	resp, err := c.client.Do(req)
	if err != nil {
		return "", fmt.Errorf("HTTP GET failed: %w", err)
	}
	defer func() { _ = resp.Body.Close() }()

	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr)
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", fmt.Errorf("failed to read response: %w", err)
	}

	return string(body), nil
}

func parseLinkedInJobID(u string) string {
	re := regexp.MustCompile(`/jobs/view/(?:[a-z0-9\-]+-)?(\d+)`)
	m := re.FindStringSubmatch(u)
	if len(m) >= 2 {
		return m[1]
	}
	re2 := regexp.MustCompile(`/jobPosting/(\d+)`)
	m2 := re2.FindStringSubmatch(u)
	if len(m2) >= 2 {
		return m2[1]
	}
	return ""
}

func extractRegex(s, pattern string) string {
	re := regexp.MustCompile(pattern)
	m := re.FindStringSubmatch(s)
	if len(m) >= 2 {
		return m[1]
	}
	return ""
}

func cleanHTML(s string) string {
	s = strings.ReplaceAll(s, "<b>", "**")
	s = strings.ReplaceAll(s, "</b>", "**")
	s = strings.ReplaceAll(s, "<strong>", "**")
	s = strings.ReplaceAll(s, "</strong>", "**")
	s = strings.ReplaceAll(s, "<br>", "\n")
	s = strings.ReplaceAll(s, "<br/>", "\n")
	s = strings.ReplaceAll(s, "<br />", "\n")
	s = strings.ReplaceAll(s, "<li>", "- ")
	s = strings.ReplaceAll(s, "</li>", "\n")
	s = strings.ReplaceAll(s, "<ul>", "\n")
	s = strings.ReplaceAll(s, "</ul>", "\n")
	s = strings.ReplaceAll(s, "<p>", "\n")
	s = strings.ReplaceAll(s, "</p>", "\n")
	reTags := regexp.MustCompile(`<[^>]*>`)
	s = reTags.ReplaceAllString(s, "")
	return strings.TrimSpace(s)
}