141 lines
3.2 KiB
Go
141 lines
3.2 KiB
Go
package crawler
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"golang.org/x/net/html"
|
|
)
|
|
|
|
type GenericCrawler struct {
|
|
client *http.Client
|
|
}
|
|
|
|
func NewGenericCrawler() *GenericCrawler {
|
|
return &GenericCrawler{
|
|
client: &http.Client{
|
|
Timeout: 15 * time.Second,
|
|
},
|
|
}
|
|
}
|
|
|
|
func (c *GenericCrawler) CanHandle(platform string) bool {
|
|
p := strings.ToLower(platform)
|
|
return p == "generic" || p == "greenhouse" || p == "lever" || p == "linkedin"
|
|
}
|
|
|
|
func (c *GenericCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]DiscoveredJob, error) {
|
|
cid := GenerateCanonicalID(extractCompanyFromURL(targetURL), "Generic Job", targetURL)
|
|
return []DiscoveredJob{
|
|
{
|
|
CanonicalID: cid,
|
|
ApplyURL: targetURL,
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
func (c *GenericCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*ExtractedJob, error) {
|
|
req, err := http.NewRequestWithContext(ctx, "GET", applyURL, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create request: %w", err)
|
|
}
|
|
|
|
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
|
|
|
resp, err := c.client.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("HTTP request failed: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return nil, fmt.Errorf("HTTP status error %d for URL %s", resp.StatusCode, applyURL)
|
|
}
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read response body: %w", err)
|
|
}
|
|
|
|
rawText := ExtractTextFromHTML(string(body))
|
|
title := ExtractTitleFromHTML(string(body))
|
|
if title == "" {
|
|
title = "Job Opportunity"
|
|
}
|
|
|
|
company := extractCompanyFromURL(applyURL)
|
|
cid := GenerateCanonicalID(company, title, applyURL)
|
|
|
|
return &ExtractedJob{
|
|
CanonicalID: cid,
|
|
Title: title,
|
|
Company: company,
|
|
Location: "Remote / Listed Location",
|
|
ApplyURL: applyURL,
|
|
RawContent: rawText,
|
|
}, nil
|
|
}
|
|
|
|
func ExtractTitleFromHTML(htmlContent string) string {
|
|
doc, err := html.Parse(strings.NewReader(htmlContent))
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
var title string
|
|
var f func(*html.Node)
|
|
f = func(n *html.Node) {
|
|
if n.Type == html.ElementNode && n.Data == "title" && n.FirstChild != nil {
|
|
title = strings.TrimSpace(n.FirstChild.Data)
|
|
return
|
|
}
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
f(c)
|
|
}
|
|
}
|
|
f(doc)
|
|
return title
|
|
}
|
|
|
|
func ExtractTextFromHTML(htmlContent string) string {
|
|
doc, err := html.Parse(strings.NewReader(htmlContent))
|
|
if err != nil {
|
|
return htmlContent
|
|
}
|
|
var b strings.Builder
|
|
var f func(*html.Node)
|
|
f = func(n *html.Node) {
|
|
if n.Type == html.ElementNode && (n.Data == "script" || n.Data == "style") {
|
|
return
|
|
}
|
|
if n.Type == html.TextNode {
|
|
txt := strings.TrimSpace(n.Data)
|
|
if txt != "" {
|
|
b.WriteString(txt)
|
|
b.WriteString(" ")
|
|
}
|
|
}
|
|
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
|
f(c)
|
|
}
|
|
}
|
|
f(doc)
|
|
return b.String()
|
|
}
|
|
|
|
func extractCompanyFromURL(u string) string {
|
|
if strings.Contains(u, "google") {
|
|
return "Google"
|
|
} else if strings.Contains(u, "linkedin") {
|
|
return "LinkedIn Network"
|
|
} else if strings.Contains(u, "greenhouse") {
|
|
return "Greenhouse Client"
|
|
} else if strings.Contains(u, "lever") {
|
|
return "Lever Client"
|
|
}
|
|
return "Tech Corp"
|
|
}
|