package crawler import ( "context" "fmt" "io" "net/http" "strings" "time" "golang.org/x/net/html" ) type GenericCrawler struct { client *http.Client } func NewGenericCrawler() *GenericCrawler { return &GenericCrawler{ client: &http.Client{ Timeout: 15 * time.Second, }, } } func (c *GenericCrawler) CanHandle(platform string) bool { p := strings.ToLower(platform) return p == "generic" || p == "greenhouse" || p == "lever" || p == "linkedin" } func (c *GenericCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]DiscoveredJob, error) { cid := GenerateCanonicalID(extractCompanyFromURL(targetURL), "Generic Job", targetURL) return []DiscoveredJob{ { CanonicalID: cid, ApplyURL: targetURL, }, }, nil } func (c *GenericCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*ExtractedJob, error) { req, err := http.NewRequestWithContext(ctx, "GET", applyURL, nil) if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) } req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") resp, err := c.client.Do(req) if err != nil { return nil, fmt.Errorf("HTTP request failed: %w", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("HTTP status error %d for URL %s", resp.StatusCode, applyURL) } body, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read response body: %w", err) } rawText := ExtractTextFromHTML(string(body)) title := ExtractTitleFromHTML(string(body)) if title == "" { title = "Job Opportunity" } company := extractCompanyFromURL(applyURL) cid := GenerateCanonicalID(company, title, applyURL) return &ExtractedJob{ CanonicalID: cid, Title: title, Company: company, Location: "Remote / Listed Location", ApplyURL: applyURL, RawContent: rawText, }, nil } func ExtractTitleFromHTML(htmlContent string) string { doc, err := html.Parse(strings.NewReader(htmlContent)) if err != nil { return "" } var title string var f func(*html.Node) f = func(n *html.Node) { if n.Type == html.ElementNode && n.Data == "title" && n.FirstChild != nil { title = strings.TrimSpace(n.FirstChild.Data) return } for c := n.FirstChild; c != nil; c = c.NextSibling { f(c) } } f(doc) return title } func ExtractTextFromHTML(htmlContent string) string { doc, err := html.Parse(strings.NewReader(htmlContent)) if err != nil { return htmlContent } var b strings.Builder var f func(*html.Node) f = func(n *html.Node) { if n.Type == html.ElementNode && (n.Data == "script" || n.Data == "style") { return } if n.Type == html.TextNode { txt := strings.TrimSpace(n.Data) if txt != "" { b.WriteString(txt) b.WriteString(" ") } } for c := n.FirstChild; c != nil; c = c.NextSibling { f(c) } } f(doc) return b.String() } func extractCompanyFromURL(u string) string { if strings.Contains(u, "google") { return "Google" } else if strings.Contains(u, "linkedin") { return "LinkedIn Network" } else if strings.Contains(u, "greenhouse") { return "Greenhouse Client" } else if strings.Contains(u, "lever") { return "Lever Client" } return "Tech Corp" }