143 lines
4.0 KiB
Go
143 lines
4.0 KiB
Go
package meta
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/chromedp/cdproto/cdp"
|
|
"github.com/chromedp/chromedp"
|
|
"git.pengzhan.dev/haopengzhan/hireme/internal/crawler"
|
|
"git.pengzhan.dev/haopengzhan/hireme/internal/logger"
|
|
)
|
|
|
|
type MetaCrawler struct{}
|
|
|
|
func NewMetaCrawler() *MetaCrawler {
|
|
return &MetaCrawler{}
|
|
}
|
|
|
|
func (c *MetaCrawler) CanHandle(platform string) bool {
|
|
return strings.ToLower(platform) == "meta" || strings.ToLower(platform) == "facebook"
|
|
}
|
|
|
|
func (c *MetaCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
|
|
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
|
chromedp.Flag("headless", true),
|
|
chromedp.Flag("no-sandbox", true),
|
|
chromedp.Flag("disable-gpu", true),
|
|
chromedp.Flag("disable-dev-shm-usage", true),
|
|
)
|
|
|
|
allocCtx, cancelAlloc := chromedp.NewExecAllocator(ctx, opts...)
|
|
defer cancelAlloc()
|
|
|
|
chromeCtx, cancelCtx := chromedp.NewContext(allocCtx)
|
|
defer cancelCtx()
|
|
|
|
chromeCtx, cancelTimeout := context.WithTimeout(chromeCtx, 30*time.Second)
|
|
defer cancelTimeout()
|
|
|
|
var nodes []*cdp.Node
|
|
logger.Log.Infow("Navigating to Meta careers search", "url", targetURL)
|
|
|
|
err := chromedp.Run(chromeCtx,
|
|
chromedp.Navigate(targetURL),
|
|
chromedp.Sleep(5*time.Second), // Wait for React and GraphQL to load
|
|
chromedp.Nodes(`a`, &nodes, chromedp.ByQueryAll),
|
|
)
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("chromedp navigation failed: %w", err)
|
|
}
|
|
|
|
var results []crawler.DiscoveredJob
|
|
seen := make(map[string]bool)
|
|
|
|
for _, n := range nodes {
|
|
href := n.AttributeValue("href")
|
|
// Meta uses /v2/jobs/..., /jobs/..., or /profile/job_details/...
|
|
if (strings.Contains(href, "/v2/jobs/") || strings.Contains(href, "/jobs/") || strings.Contains(href, "/profile/job_details/")) && !strings.Contains(href, "jobsearch") {
|
|
fullURL := href
|
|
if strings.HasPrefix(href, "/") {
|
|
fullURL = "https://www.metacareers.com" + href
|
|
}
|
|
|
|
// Extract Canonical ID from URL
|
|
parts := strings.Split(strings.Trim(href, "/"), "/")
|
|
canonicalID := "unknown"
|
|
for i, p := range parts {
|
|
if (p == "jobs" || p == "job_details") && i+1 < len(parts) {
|
|
canonicalID = parts[i+1]
|
|
break
|
|
}
|
|
}
|
|
|
|
if !seen[fullURL] {
|
|
seen[fullURL] = true
|
|
results = append(results, crawler.DiscoveredJob{
|
|
ApplyURL: fullURL,
|
|
CanonicalID: "meta_" + canonicalID,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
logger.Log.Infow("Meta discovery finished", "found_count", len(results))
|
|
return results, nil
|
|
}
|
|
|
|
func (c *MetaCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
|
|
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
|
chromedp.Flag("headless", true),
|
|
chromedp.Flag("no-sandbox", true),
|
|
chromedp.Flag("disable-gpu", true),
|
|
chromedp.Flag("disable-dev-shm-usage", true),
|
|
)
|
|
|
|
allocCtx, cancelAlloc := chromedp.NewExecAllocator(ctx, opts...)
|
|
defer cancelAlloc()
|
|
|
|
chromeCtx, cancelCtx := chromedp.NewContext(allocCtx)
|
|
defer cancelCtx()
|
|
|
|
chromeCtx, cancelTimeout := context.WithTimeout(chromeCtx, 30*time.Second)
|
|
defer cancelTimeout()
|
|
|
|
logger.Log.Infow("Fetching Meta job detail", "url", applyURL)
|
|
|
|
var titleText string
|
|
var bodyText string
|
|
|
|
err := chromedp.Run(chromeCtx,
|
|
chromedp.Navigate(applyURL),
|
|
chromedp.Sleep(5*time.Second),
|
|
chromedp.Evaluate(`document.querySelector("h1") ? document.querySelector("h1").innerText : "Unknown Meta Title"`, &titleText),
|
|
chromedp.Evaluate(`document.body.innerText`, &bodyText),
|
|
)
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("chromedp navigation failed: %w", err)
|
|
}
|
|
|
|
// Extract ID
|
|
parts := strings.Split(strings.Trim(applyURL, "/"), "/")
|
|
canonicalID := "unknown"
|
|
for i, p := range parts {
|
|
if (p == "jobs" || p == "job_details") && i+1 < len(parts) {
|
|
canonicalID = parts[i+1]
|
|
break
|
|
}
|
|
}
|
|
|
|
return &crawler.ExtractedJob{
|
|
CanonicalID: "meta_" + canonicalID,
|
|
Title: titleText,
|
|
Company: "Meta",
|
|
Location: "Check description", // Location extraction is complex via text, leave it or parse later
|
|
ApplyURL: applyURL,
|
|
RawContent: bodyText,
|
|
}, nil
|
|
}
|