feat: implement multiple new job crawlers and add corresponding test suites
This commit is contained in:
@@ -0,0 +1,229 @@
|
||||
package linkedin
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hireme/hireme/internal/crawler"
|
||||
"github.com/hireme/hireme/internal/logger"
|
||||
)
|
||||
|
||||
type LinkedInCrawler struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewLinkedInCrawler() *LinkedInCrawler {
|
||||
return NewLinkedInCrawlerWithClient(&http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
}
|
||||
|
||||
func NewLinkedInCrawlerWithClient(client *http.Client) *LinkedInCrawler {
|
||||
return &LinkedInCrawler{
|
||||
client: client,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *LinkedInCrawler) CanHandle(platform string) bool {
|
||||
p := strings.ToLower(platform)
|
||||
return p == "linkedin" || strings.Contains(p, "linkedin.com/jobs")
|
||||
}
|
||||
|
||||
func (c *LinkedInCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
|
||||
seen := make(map[string]bool)
|
||||
var discovered []crawler.DiscoveredJob
|
||||
|
||||
searchAPI := targetURL
|
||||
if !strings.Contains(searchAPI, "jobs-guest/jobs/api/seeMoreJobPostings/search") {
|
||||
u, err := url.Parse(targetURL)
|
||||
if err == nil && u.Host != "" {
|
||||
q := u.Query()
|
||||
searchAPI = fmt.Sprintf("https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search?%s", q.Encode())
|
||||
} else {
|
||||
searchAPI = fmt.Sprintf("https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search?keywords=%s&location=Seattle%%2C+Washington%%2C+United+States", url.QueryEscape(targetURL))
|
||||
}
|
||||
}
|
||||
|
||||
pageSize := 25
|
||||
reID := regexp.MustCompile(`(?:urn:li:jobPosting:(\d+)|/jobs/view/(?:[a-z0-9\-]+-)?(\d+))`)
|
||||
|
||||
for page := 0; page < 4; page++ {
|
||||
start := page * pageSize
|
||||
reqURL := searchAPI
|
||||
if strings.Contains(reqURL, "?") {
|
||||
reqURL = fmt.Sprintf("%s&start=%d", searchAPI, start)
|
||||
} else {
|
||||
reqURL = fmt.Sprintf("%s?start=%d", searchAPI, start)
|
||||
}
|
||||
|
||||
body, err := c.fetchURL(ctx, reqURL)
|
||||
if err != nil {
|
||||
if page == 0 {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
matches := reID.FindAllStringSubmatch(body, -1)
|
||||
if len(matches) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
newFound := false
|
||||
for _, m := range matches {
|
||||
jobID := m[1]
|
||||
if jobID == "" && len(m) >= 3 {
|
||||
jobID = m[2]
|
||||
}
|
||||
if jobID == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
applyURL := fmt.Sprintf("https://www.linkedin.com/jobs/view/%s", jobID)
|
||||
if !seen[applyURL] {
|
||||
seen[applyURL] = true
|
||||
newFound = true
|
||||
discovered = append(discovered, crawler.DiscoveredJob{
|
||||
CanonicalID: fmt.Sprintf("linkedin:%s", jobID),
|
||||
ApplyURL: applyURL,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if !newFound || len(matches) < pageSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
logger.Log.Infow("LinkedIn discovery finished", "found_count", len(discovered))
|
||||
return discovered, nil
|
||||
}
|
||||
|
||||
func (c *LinkedInCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
|
||||
jobID := parseLinkedInJobID(applyURL)
|
||||
if jobID == "" {
|
||||
jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16]
|
||||
}
|
||||
|
||||
// LinkedIn guest API endpoint for job details:
|
||||
guestDetailURL := fmt.Sprintf("https://www.linkedin.com/jobs-guest/jobs/api/jobPosting/%s", jobID)
|
||||
body, err := c.fetchURL(ctx, guestDetailURL)
|
||||
if err != nil {
|
||||
body, err = c.fetchURL(ctx, applyURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
title := extractRegex(body, `<h2[^>]*class="[^"]*top-card-layout__title[^"]*"[^>]*>(.*?)</h2>`)
|
||||
if title == "" {
|
||||
title = extractRegex(body, `<h1[^>]*>(.*?)</h1>`)
|
||||
}
|
||||
if title == "" {
|
||||
title = "Software Engineer"
|
||||
}
|
||||
|
||||
company := extractRegex(body, `<a[^>]*class="[^"]*topcard__org-name-link[^"]*"[^>]*>(.*?)</a>`)
|
||||
if company == "" {
|
||||
company = extractRegex(body, `<span[^>]*class="[^"]*topcard__flavor[^"]*"[^>]*>(.*?)</span>`)
|
||||
}
|
||||
if company == "" {
|
||||
company = "LinkedIn"
|
||||
}
|
||||
|
||||
location := extractRegex(body, `<span[^>]*class="[^"]*topcard__flavor--bullet[^"]*"[^>]*>(.*?)</span>`)
|
||||
if location == "" {
|
||||
location = "Seattle, WA, USA"
|
||||
}
|
||||
|
||||
desc := extractRegex(body, `<div[^>]*class="[^"]*show-more-less-html__markup[^"]*"[^>]*>([\s\S]*?)</div>`)
|
||||
if desc == "" {
|
||||
desc = body
|
||||
}
|
||||
|
||||
return &crawler.ExtractedJob{
|
||||
CanonicalID: fmt.Sprintf("linkedin:%s", jobID),
|
||||
Title: cleanHTML(title),
|
||||
Company: cleanHTML(company),
|
||||
Location: cleanHTML(location),
|
||||
ApplyURL: applyURL,
|
||||
RawContent: cleanHTML(desc),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (c *LinkedInCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
||||
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
|
||||
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("HTTP GET failed: %w", err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to read response: %w", err)
|
||||
}
|
||||
|
||||
return string(body), nil
|
||||
}
|
||||
|
||||
func parseLinkedInJobID(u string) string {
|
||||
re := regexp.MustCompile(`/jobs/view/(?:[a-z0-9\-]+-)?(\d+)`)
|
||||
m := re.FindStringSubmatch(u)
|
||||
if len(m) >= 2 {
|
||||
return m[1]
|
||||
}
|
||||
re2 := regexp.MustCompile(`/jobPosting/(\d+)`)
|
||||
m2 := re2.FindStringSubmatch(u)
|
||||
if len(m2) >= 2 {
|
||||
return m2[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func extractRegex(s, pattern string) string {
|
||||
re := regexp.MustCompile(pattern)
|
||||
m := re.FindStringSubmatch(s)
|
||||
if len(m) >= 2 {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func cleanHTML(s string) string {
|
||||
s = strings.ReplaceAll(s, "<b>", "**")
|
||||
s = strings.ReplaceAll(s, "</b>", "**")
|
||||
s = strings.ReplaceAll(s, "<strong>", "**")
|
||||
s = strings.ReplaceAll(s, "</strong>", "**")
|
||||
s = strings.ReplaceAll(s, "<br>", "\n")
|
||||
s = strings.ReplaceAll(s, "<br/>", "\n")
|
||||
s = strings.ReplaceAll(s, "<br />", "\n")
|
||||
s = strings.ReplaceAll(s, "<li>", "- ")
|
||||
s = strings.ReplaceAll(s, "</li>", "\n")
|
||||
s = strings.ReplaceAll(s, "<ul>", "\n")
|
||||
s = strings.ReplaceAll(s, "</ul>", "\n")
|
||||
s = strings.ReplaceAll(s, "<p>", "\n")
|
||||
s = strings.ReplaceAll(s, "</p>", "\n")
|
||||
reTags := regexp.MustCompile(`<[^>]*>`)
|
||||
s = reTags.ReplaceAllString(s, "")
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
Reference in New Issue
Block a user