feat: implement Meta crawler using chromedp and register it in worker pool
This commit is contained in:
@@ -7,3 +7,7 @@ targets:
|
||||
target_url: "https://jobs.apple.com/en-us/search?location=seattle-SEA&key=Software%2520Engineer&team=apps-and-frameworks-SFTWR-AF+cloud-and-infrastructure-SFTWR-CLD+core-operating-systems-SFTWR-COS+devops-and-site-reliability-SFTWR-DSR+engineering-project-management-SFTWR-EPM+information-systems-and-technology-SFTWR-ISTECH+machine-learning-and-ai-SFTWR-MCHLN+security-and-privacy-SFTWR-SEC+software-quality-automation-and-tools-SFTWR-SQAT+wireless-software-SFTWR-WSFT+machine-learning-infrastructure-MLAI-MLI+deep-learning-and-reinforcement-learning-MLAI-DLRL+natural-language-processing-and-speech-technologies-MLAI-NLP+computer-vision-MLAI-CV+applied-research-MLAI-ARr"
|
||||
platform: "apple"
|
||||
is_active: true
|
||||
- name: "Meta Software Engineer (Seattle)"
|
||||
target_url: "https://www.metacareers.com/jobsearch/?offices[0]=Seattle%2C%20WA&roles[0]=Full%20time%20employment&teams[0]=Software%20Engineering"
|
||||
platform: "meta"
|
||||
is_active: true
|
||||
@@ -4,6 +4,8 @@ go 1.26
|
||||
|
||||
require (
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2
|
||||
github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f
|
||||
github.com/chromedp/chromedp v0.16.0
|
||||
github.com/mattn/go-sqlite3 v1.14.42
|
||||
github.com/spf13/cobra v1.9.1
|
||||
github.com/spf13/pflag v1.0.6
|
||||
@@ -14,6 +16,12 @@ require (
|
||||
|
||||
require (
|
||||
github.com/JohannesKaufmann/dom v0.3.1 // indirect
|
||||
github.com/chromedp/sysutil v1.1.0 // indirect
|
||||
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect
|
||||
github.com/gobwas/httphead v0.1.0 // indirect
|
||||
github.com/gobwas/pool v0.2.1 // indirect
|
||||
github.com/gobwas/ws v1.4.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
go.uber.org/multierr v1.10.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
)
|
||||
|
||||
@@ -2,24 +2,59 @@ github.com/JohannesKaufmann/dom v0.3.1 h1:J16l9JAHWgkFPR3VIPbQ1gvS0cWab6laK1q7PF
|
||||
github.com/JohannesKaufmann/dom v0.3.1/go.mod h1:BZPkf8ZeYrBgABjwJn9iiKt8aiCtkxpHkevms+Yp2DE=
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 h1:XFJZFWESIWlUEHHjzBuv8RvrtCWnSGlimEX17ysSDb8=
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2/go.mod h1:BHWO8lJzttJLqwuV8Rb1B3OG2OSzLbssZDI1FRg2eAA=
|
||||
github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f h1:0Z1zcSLEmnj2c2CmJYBqewtS6pxhB39bNWUSEUAWjgk=
|
||||
github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f/go.mod h1:RwFsSODCtFExll+GhHM6R92SARHR3Z3oipaxLHj46C0=
|
||||
github.com/chromedp/chromedp v0.16.0 h1:rOO4deOm4CbZgBCa8mD9g2rDyIoNs0BkgvNrlbp5ouk=
|
||||
github.com/chromedp/chromedp v0.16.0/go.mod h1:rbuGKFT1vMcFcFqKfPIO1GpX/N+2s8onm2qMxZLbU5U=
|
||||
github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM=
|
||||
github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg=
|
||||
github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg=
|
||||
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
|
||||
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
|
||||
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
|
||||
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
|
||||
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
|
||||
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo=
|
||||
github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs=
|
||||
github.com/mattn/go-sqlite3 v1.14.42 h1:MigqEP4ZmHw3aIdIT7T+9TLa90Z6smwcthx+Azv4Cgo=
|
||||
github.com/mattn/go-sqlite3 v1.14.42/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw=
|
||||
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc=
|
||||
github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI=
|
||||
github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw=
|
||||
github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
|
||||
github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
|
||||
github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
|
||||
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
|
||||
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE=
|
||||
github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
|
||||
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
|
||||
go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
|
||||
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"github.com/hireme/hireme/internal/logger"
|
||||
"github.com/hireme/hireme/pkg/crawler/apple"
|
||||
"github.com/hireme/hireme/pkg/crawler/google"
|
||||
"github.com/hireme/hireme/pkg/crawler/meta"
|
||||
)
|
||||
|
||||
type WorkerPool struct {
|
||||
@@ -296,5 +297,8 @@ func (p *WorkerPool) getCrawlerForPlatform(platform string) crawler.SiteCrawler
|
||||
if strings.Contains(plat, "apple") {
|
||||
return apple.NewAppleCrawler()
|
||||
}
|
||||
if strings.Contains(plat, "meta") || strings.Contains(plat, "facebook") {
|
||||
return meta.NewMetaCrawler()
|
||||
}
|
||||
return crawler.NewGenericCrawler()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
package meta
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/chromedp/cdproto/cdp"
|
||||
"github.com/chromedp/chromedp"
|
||||
"github.com/hireme/hireme/internal/crawler"
|
||||
"github.com/hireme/hireme/internal/logger"
|
||||
)
|
||||
|
||||
type MetaCrawler struct{}
|
||||
|
||||
func NewMetaCrawler() *MetaCrawler {
|
||||
return &MetaCrawler{}
|
||||
}
|
||||
|
||||
func (c *MetaCrawler) CanHandle(platform string) bool {
|
||||
return strings.ToLower(platform) == "meta" || strings.ToLower(platform) == "facebook"
|
||||
}
|
||||
|
||||
func (c *MetaCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
|
||||
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
||||
chromedp.Flag("headless", true),
|
||||
chromedp.Flag("no-sandbox", true),
|
||||
chromedp.Flag("disable-gpu", true),
|
||||
chromedp.Flag("disable-dev-shm-usage", true),
|
||||
)
|
||||
|
||||
allocCtx, cancelAlloc := chromedp.NewExecAllocator(ctx, opts...)
|
||||
defer cancelAlloc()
|
||||
|
||||
chromeCtx, cancelCtx := chromedp.NewContext(allocCtx)
|
||||
defer cancelCtx()
|
||||
|
||||
chromeCtx, cancelTimeout := context.WithTimeout(chromeCtx, 30*time.Second)
|
||||
defer cancelTimeout()
|
||||
|
||||
var nodes []*cdp.Node
|
||||
logger.Log.Infow("Navigating to Meta careers search", "url", targetURL)
|
||||
|
||||
err := chromedp.Run(chromeCtx,
|
||||
chromedp.Navigate(targetURL),
|
||||
chromedp.Sleep(5*time.Second), // Wait for React and GraphQL to load
|
||||
chromedp.Nodes(`a`, &nodes, chromedp.ByQueryAll),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("chromedp navigation failed: %w", err)
|
||||
}
|
||||
|
||||
var results []crawler.DiscoveredJob
|
||||
seen := make(map[string]bool)
|
||||
|
||||
for _, n := range nodes {
|
||||
href := n.AttributeValue("href")
|
||||
// Meta uses /v2/jobs/..., /jobs/..., or /profile/job_details/...
|
||||
if (strings.Contains(href, "/v2/jobs/") || strings.Contains(href, "/jobs/") || strings.Contains(href, "/profile/job_details/")) && !strings.Contains(href, "jobsearch") {
|
||||
fullURL := href
|
||||
if strings.HasPrefix(href, "/") {
|
||||
fullURL = "https://www.metacareers.com" + href
|
||||
}
|
||||
|
||||
// Extract Canonical ID from URL
|
||||
parts := strings.Split(strings.Trim(href, "/"), "/")
|
||||
canonicalID := "unknown"
|
||||
for i, p := range parts {
|
||||
if (p == "jobs" || p == "job_details") && i+1 < len(parts) {
|
||||
canonicalID = parts[i+1]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !seen[fullURL] {
|
||||
seen[fullURL] = true
|
||||
results = append(results, crawler.DiscoveredJob{
|
||||
ApplyURL: fullURL,
|
||||
CanonicalID: "meta_" + canonicalID,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
logger.Log.Infow("Meta discovery finished", "found_count", len(results))
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (c *MetaCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
|
||||
opts := append(chromedp.DefaultExecAllocatorOptions[:],
|
||||
chromedp.Flag("headless", true),
|
||||
chromedp.Flag("no-sandbox", true),
|
||||
chromedp.Flag("disable-gpu", true),
|
||||
chromedp.Flag("disable-dev-shm-usage", true),
|
||||
)
|
||||
|
||||
allocCtx, cancelAlloc := chromedp.NewExecAllocator(ctx, opts...)
|
||||
defer cancelAlloc()
|
||||
|
||||
chromeCtx, cancelCtx := chromedp.NewContext(allocCtx)
|
||||
defer cancelCtx()
|
||||
|
||||
chromeCtx, cancelTimeout := context.WithTimeout(chromeCtx, 30*time.Second)
|
||||
defer cancelTimeout()
|
||||
|
||||
logger.Log.Infow("Fetching Meta job detail", "url", applyURL)
|
||||
|
||||
var titleText string
|
||||
var bodyText string
|
||||
|
||||
err := chromedp.Run(chromeCtx,
|
||||
chromedp.Navigate(applyURL),
|
||||
chromedp.Sleep(5*time.Second),
|
||||
chromedp.Evaluate(`document.querySelector("h1") ? document.querySelector("h1").innerText : "Unknown Meta Title"`, &titleText),
|
||||
chromedp.Evaluate(`document.body.innerText`, &bodyText),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("chromedp navigation failed: %w", err)
|
||||
}
|
||||
|
||||
// Extract ID
|
||||
parts := strings.Split(strings.Trim(applyURL, "/"), "/")
|
||||
canonicalID := "unknown"
|
||||
for i, p := range parts {
|
||||
if (p == "jobs" || p == "job_details") && i+1 < len(parts) {
|
||||
canonicalID = parts[i+1]
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return &crawler.ExtractedJob{
|
||||
CanonicalID: "meta_" + canonicalID,
|
||||
Title: titleText,
|
||||
Company: "Meta",
|
||||
Location: "Check description", // Location extraction is complex via text, leave it or parse later
|
||||
ApplyURL: applyURL,
|
||||
RawContent: bodyText,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package meta
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMetaCrawler_EndToEnd(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("Skipping long-running Meta crawler network test in short mode")
|
||||
}
|
||||
|
||||
c := NewMetaCrawler()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
testURL := "https://www.metacareers.com/jobsearch/?offices[0]=Seattle%2C%20WA&roles[0]=Full%20time%20employment"
|
||||
|
||||
t.Logf("Discovering jobs from: %s", testURL)
|
||||
discovered, err := c.DiscoverJobs(ctx, testURL)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to discover jobs: %v", err)
|
||||
}
|
||||
|
||||
if len(discovered) == 0 {
|
||||
t.Fatalf("Expected to find at least 1 job, found 0")
|
||||
}
|
||||
|
||||
t.Logf("Successfully discovered %d jobs", len(discovered))
|
||||
|
||||
// Fetch details for the first discovered job
|
||||
firstJob := discovered[0]
|
||||
t.Logf("Fetching details for job: %s", firstJob.ApplyURL)
|
||||
|
||||
detail, err := c.FetchJobDetail(ctx, firstJob.ApplyURL)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to fetch job details: %v", err)
|
||||
}
|
||||
|
||||
if detail.Title == "" || detail.Title == "Unknown Meta Title" {
|
||||
t.Errorf("Expected a valid job title, got %s", detail.Title)
|
||||
}
|
||||
|
||||
if len(detail.RawContent) < 100 {
|
||||
t.Errorf("Expected RawContent to contain the job description, but it was too short (%d bytes)", len(detail.RawContent))
|
||||
}
|
||||
|
||||
if detail.CanonicalID == "unknown" || detail.CanonicalID == "" {
|
||||
t.Errorf("Failed to extract canonical ID from %s", firstJob.ApplyURL)
|
||||
}
|
||||
|
||||
t.Logf("Extracted Title: %s", detail.Title)
|
||||
t.Logf("Extracted CanonicalID: %s", detail.CanonicalID)
|
||||
t.Logf("Raw Content Length: %d", len(detail.RawContent))
|
||||
}
|
||||
Reference in New Issue
Block a user