feat: initialize project structure with SQLite database, crawler framework, and LLM-based JD analysis integration

internal/crawler/generic.go

@@ -0,0 +1,140 @@
+package crawler
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"golang.org/x/net/html"
+)
+
+type GenericCrawler struct {
+	client *http.Client
+}
+
+func NewGenericCrawler() *GenericCrawler {
+	return &GenericCrawler{
+		client: &http.Client{
+			Timeout: 15 * time.Second,
+		},
+	}
+}
+
+func (c *GenericCrawler) CanHandle(platform string) bool {
+	p := strings.ToLower(platform)
+	return p == "generic" || p == "greenhouse" || p == "lever" || p == "linkedin"
+}
+
+func (c *GenericCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]DiscoveredJob, error) {
+	cid := GenerateCanonicalID(extractCompanyFromURL(targetURL), "Generic Job", targetURL)
+	return []DiscoveredJob{
+		{
+			CanonicalID: cid,
+			ApplyURL:    targetURL,
+		},
+	}, nil
+}
+
+func (c *GenericCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*ExtractedJob, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", applyURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
+
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("HTTP request failed: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("HTTP status error %d for URL %s", resp.StatusCode, applyURL)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	rawText := ExtractTextFromHTML(string(body))
+	title := ExtractTitleFromHTML(string(body))
+	if title == "" {
+		title = "Job Opportunity"
+	}
+
+	company := extractCompanyFromURL(applyURL)
+	cid := GenerateCanonicalID(company, title, applyURL)
+
+	return &ExtractedJob{
+		CanonicalID: cid,
+		Title:       title,
+		Company:     company,
+		Location:    "Remote / Listed Location",
+		ApplyURL:    applyURL,
+		RawContent:  rawText,
+	}, nil
+}
+
+func ExtractTitleFromHTML(htmlContent string) string {
+	doc, err := html.Parse(strings.NewReader(htmlContent))
+	if err != nil {
+		return ""
+	}
+	var title string
+	var f func(*html.Node)
+	f = func(n *html.Node) {
+		if n.Type == html.ElementNode && n.Data == "title" && n.FirstChild != nil {
+			title = strings.TrimSpace(n.FirstChild.Data)
+			return
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			f(c)
+		}
+	}
+	f(doc)
+	return title
+}
+
+func ExtractTextFromHTML(htmlContent string) string {
+	doc, err := html.Parse(strings.NewReader(htmlContent))
+	if err != nil {
+		return htmlContent
+	}
+	var b strings.Builder
+	var f func(*html.Node)
+	f = func(n *html.Node) {
+		if n.Type == html.ElementNode && (n.Data == "script" || n.Data == "style") {
+			return
+		}
+		if n.Type == html.TextNode {
+			txt := strings.TrimSpace(n.Data)
+			if txt != "" {
+				b.WriteString(txt)
+				b.WriteString(" ")
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			f(c)
+		}
+	}
+	f(doc)
+	return b.String()
+}
+
+func extractCompanyFromURL(u string) string {
+	if strings.Contains(u, "google") {
+		return "Google"
+	} else if strings.Contains(u, "linkedin") {
+		return "LinkedIn Network"
+	} else if strings.Contains(u, "greenhouse") {
+		return "Greenhouse Client"
+	} else if strings.Contains(u, "lever") {
+		return "Lever Client"
+	}
+	return "Tech Corp"
+}