diff --git a/data/config/targets.yaml b/data/config/targets.yaml index 2e97241..01a3aaa 100644 --- a/data/config/targets.yaml +++ b/data/config/targets.yaml @@ -6,4 +6,8 @@ targets: - name: "Apple Software Engineer (Seattle)" target_url: "https://jobs.apple.com/en-us/search?location=seattle-SEA&key=Software%2520Engineer&team=apps-and-frameworks-SFTWR-AF+cloud-and-infrastructure-SFTWR-CLD+core-operating-systems-SFTWR-COS+devops-and-site-reliability-SFTWR-DSR+engineering-project-management-SFTWR-EPM+information-systems-and-technology-SFTWR-ISTECH+machine-learning-and-ai-SFTWR-MCHLN+security-and-privacy-SFTWR-SEC+software-quality-automation-and-tools-SFTWR-SQAT+wireless-software-SFTWR-WSFT+machine-learning-infrastructure-MLAI-MLI+deep-learning-and-reinforcement-learning-MLAI-DLRL+natural-language-processing-and-speech-technologies-MLAI-NLP+computer-vision-MLAI-CV+applied-research-MLAI-ARr" platform: "apple" + is_active: true + - name: "Meta Software Engineer (Seattle)" + target_url: "https://www.metacareers.com/jobsearch/?offices[0]=Seattle%2C%20WA&roles[0]=Full%20time%20employment&teams[0]=Software%20Engineering" + platform: "meta" is_active: true \ No newline at end of file diff --git a/go.mod b/go.mod index a95fd04..abf4773 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.26 require ( github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 + github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f + github.com/chromedp/chromedp v0.16.0 github.com/mattn/go-sqlite3 v1.14.42 github.com/spf13/cobra v1.9.1 github.com/spf13/pflag v1.0.6 @@ -14,6 +16,12 @@ require ( require ( github.com/JohannesKaufmann/dom v0.3.1 // indirect + github.com/chromedp/sysutil v1.1.0 // indirect + github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/gobwas/ws v1.4.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect go.uber.org/multierr v1.10.0 // indirect + golang.org/x/sys v0.47.0 // indirect ) diff --git a/go.sum b/go.sum index 5debd4c..e665133 100644 --- a/go.sum +++ b/go.sum @@ -2,24 +2,59 @@ github.com/JohannesKaufmann/dom v0.3.1 h1:J16l9JAHWgkFPR3VIPbQ1gvS0cWab6laK1q7PF github.com/JohannesKaufmann/dom v0.3.1/go.mod h1:BZPkf8ZeYrBgABjwJn9iiKt8aiCtkxpHkevms+Yp2DE= github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 h1:XFJZFWESIWlUEHHjzBuv8RvrtCWnSGlimEX17ysSDb8= github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2/go.mod h1:BHWO8lJzttJLqwuV8Rb1B3OG2OSzLbssZDI1FRg2eAA= +github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f h1:0Z1zcSLEmnj2c2CmJYBqewtS6pxhB39bNWUSEUAWjgk= +github.com/chromedp/cdproto v0.0.0-20260714215040-dc233986426f/go.mod h1:RwFsSODCtFExll+GhHM6R92SARHR3Z3oipaxLHj46C0= +github.com/chromedp/chromedp v0.16.0 h1:rOO4deOm4CbZgBCa8mD9g2rDyIoNs0BkgvNrlbp5ouk= +github.com/chromedp/chromedp v0.16.0/go.mod h1:rbuGKFT1vMcFcFqKfPIO1GpX/N+2s8onm2qMxZLbU5U= +github.com/chromedp/sysutil v1.1.0 h1:PUFNv5EcprjqXZD9nJb9b/c9ibAbxiYo4exNWZyipwM= +github.com/chromedp/sysutil v1.1.0/go.mod h1:WiThHUdltqCNKGc4gaU50XgYjwjYIhKWoHGPTUfWTJ8= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68 h1:KZaTBSyshWX3MP5jukJcNSuXDQTO+rNpt0J564dX/eg= +github.com/go-json-experiment/json v0.0.0-20260623181947-01eb4420fa68/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80 h1:6Yzfa6GP0rIo/kULo2bwGEkFvCePZ3qHDDTC3/J9Swo= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= github.com/mattn/go-sqlite3 v1.14.42 h1:MigqEP4ZmHw3aIdIT7T+9TLa90Z6smwcthx+Azv4Cgo= github.com/mattn/go-sqlite3 v1.14.42/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sebdah/goldie/v2 v2.8.0 h1:dZb9wR8q5++oplmEiJT+U/5KyotVD+HNGCAc5gNr8rc= +github.com/sebdah/goldie/v2 v2.8.0/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= +github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= -golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= -golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/worker/pool.go b/internal/worker/pool.go index ce0b7d5..efa4047 100644 --- a/internal/worker/pool.go +++ b/internal/worker/pool.go @@ -17,6 +17,7 @@ import ( "github.com/hireme/hireme/internal/logger" "github.com/hireme/hireme/pkg/crawler/apple" "github.com/hireme/hireme/pkg/crawler/google" + "github.com/hireme/hireme/pkg/crawler/meta" ) type WorkerPool struct { @@ -296,5 +297,8 @@ func (p *WorkerPool) getCrawlerForPlatform(platform string) crawler.SiteCrawler if strings.Contains(plat, "apple") { return apple.NewAppleCrawler() } + if strings.Contains(plat, "meta") || strings.Contains(plat, "facebook") { + return meta.NewMetaCrawler() + } return crawler.NewGenericCrawler() } diff --git a/pkg/crawler/meta/meta.go b/pkg/crawler/meta/meta.go new file mode 100644 index 0000000..54f9f37 --- /dev/null +++ b/pkg/crawler/meta/meta.go @@ -0,0 +1,142 @@ +package meta + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/chromedp/cdproto/cdp" + "github.com/chromedp/chromedp" + "github.com/hireme/hireme/internal/crawler" + "github.com/hireme/hireme/internal/logger" +) + +type MetaCrawler struct{} + +func NewMetaCrawler() *MetaCrawler { + return &MetaCrawler{} +} + +func (c *MetaCrawler) CanHandle(platform string) bool { + return strings.ToLower(platform) == "meta" || strings.ToLower(platform) == "facebook" +} + +func (c *MetaCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.Flag("headless", true), + chromedp.Flag("no-sandbox", true), + chromedp.Flag("disable-gpu", true), + chromedp.Flag("disable-dev-shm-usage", true), + ) + + allocCtx, cancelAlloc := chromedp.NewExecAllocator(ctx, opts...) + defer cancelAlloc() + + chromeCtx, cancelCtx := chromedp.NewContext(allocCtx) + defer cancelCtx() + + chromeCtx, cancelTimeout := context.WithTimeout(chromeCtx, 30*time.Second) + defer cancelTimeout() + + var nodes []*cdp.Node + logger.Log.Infow("Navigating to Meta careers search", "url", targetURL) + + err := chromedp.Run(chromeCtx, + chromedp.Navigate(targetURL), + chromedp.Sleep(5*time.Second), // Wait for React and GraphQL to load + chromedp.Nodes(`a`, &nodes, chromedp.ByQueryAll), + ) + + if err != nil { + return nil, fmt.Errorf("chromedp navigation failed: %w", err) + } + + var results []crawler.DiscoveredJob + seen := make(map[string]bool) + + for _, n := range nodes { + href := n.AttributeValue("href") + // Meta uses /v2/jobs/..., /jobs/..., or /profile/job_details/... + if (strings.Contains(href, "/v2/jobs/") || strings.Contains(href, "/jobs/") || strings.Contains(href, "/profile/job_details/")) && !strings.Contains(href, "jobsearch") { + fullURL := href + if strings.HasPrefix(href, "/") { + fullURL = "https://www.metacareers.com" + href + } + + // Extract Canonical ID from URL + parts := strings.Split(strings.Trim(href, "/"), "/") + canonicalID := "unknown" + for i, p := range parts { + if (p == "jobs" || p == "job_details") && i+1 < len(parts) { + canonicalID = parts[i+1] + break + } + } + + if !seen[fullURL] { + seen[fullURL] = true + results = append(results, crawler.DiscoveredJob{ + ApplyURL: fullURL, + CanonicalID: "meta_" + canonicalID, + }) + } + } + } + + logger.Log.Infow("Meta discovery finished", "found_count", len(results)) + return results, nil +} + +func (c *MetaCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { + opts := append(chromedp.DefaultExecAllocatorOptions[:], + chromedp.Flag("headless", true), + chromedp.Flag("no-sandbox", true), + chromedp.Flag("disable-gpu", true), + chromedp.Flag("disable-dev-shm-usage", true), + ) + + allocCtx, cancelAlloc := chromedp.NewExecAllocator(ctx, opts...) + defer cancelAlloc() + + chromeCtx, cancelCtx := chromedp.NewContext(allocCtx) + defer cancelCtx() + + chromeCtx, cancelTimeout := context.WithTimeout(chromeCtx, 30*time.Second) + defer cancelTimeout() + + logger.Log.Infow("Fetching Meta job detail", "url", applyURL) + + var titleText string + var bodyText string + + err := chromedp.Run(chromeCtx, + chromedp.Navigate(applyURL), + chromedp.Sleep(5*time.Second), + chromedp.Evaluate(`document.querySelector("h1") ? document.querySelector("h1").innerText : "Unknown Meta Title"`, &titleText), + chromedp.Evaluate(`document.body.innerText`, &bodyText), + ) + + if err != nil { + return nil, fmt.Errorf("chromedp navigation failed: %w", err) + } + + // Extract ID + parts := strings.Split(strings.Trim(applyURL, "/"), "/") + canonicalID := "unknown" + for i, p := range parts { + if (p == "jobs" || p == "job_details") && i+1 < len(parts) { + canonicalID = parts[i+1] + break + } + } + + return &crawler.ExtractedJob{ + CanonicalID: "meta_" + canonicalID, + Title: titleText, + Company: "Meta", + Location: "Check description", // Location extraction is complex via text, leave it or parse later + ApplyURL: applyURL, + RawContent: bodyText, + }, nil +} diff --git a/pkg/crawler/meta/meta_test.go b/pkg/crawler/meta/meta_test.go new file mode 100644 index 0000000..be7de9c --- /dev/null +++ b/pkg/crawler/meta/meta_test.go @@ -0,0 +1,56 @@ +package meta + +import ( + "context" + "testing" + "time" +) + +func TestMetaCrawler_EndToEnd(t *testing.T) { + if testing.Short() { + t.Skip("Skipping long-running Meta crawler network test in short mode") + } + + c := NewMetaCrawler() + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + testURL := "https://www.metacareers.com/jobsearch/?offices[0]=Seattle%2C%20WA&roles[0]=Full%20time%20employment" + + t.Logf("Discovering jobs from: %s", testURL) + discovered, err := c.DiscoverJobs(ctx, testURL) + if err != nil { + t.Fatalf("Failed to discover jobs: %v", err) + } + + if len(discovered) == 0 { + t.Fatalf("Expected to find at least 1 job, found 0") + } + + t.Logf("Successfully discovered %d jobs", len(discovered)) + + // Fetch details for the first discovered job + firstJob := discovered[0] + t.Logf("Fetching details for job: %s", firstJob.ApplyURL) + + detail, err := c.FetchJobDetail(ctx, firstJob.ApplyURL) + if err != nil { + t.Fatalf("Failed to fetch job details: %v", err) + } + + if detail.Title == "" || detail.Title == "Unknown Meta Title" { + t.Errorf("Expected a valid job title, got %s", detail.Title) + } + + if len(detail.RawContent) < 100 { + t.Errorf("Expected RawContent to contain the job description, but it was too short (%d bytes)", len(detail.RawContent)) + } + + if detail.CanonicalID == "unknown" || detail.CanonicalID == "" { + t.Errorf("Failed to extract canonical ID from %s", firstJob.ApplyURL) + } + + t.Logf("Extracted Title: %s", detail.Title) + t.Logf("Extracted CanonicalID: %s", detail.CanonicalID) + t.Logf("Raw Content Length: %d", len(detail.RawContent)) +}