From 51b94687370ad409aa7a61635c3b0591489f8d97 Mon Sep 17 00:00:00 2001 From: haopengzhan Date: Wed, 12 Aug 2026 01:13:53 -0700 Subject: [PATCH] feat: initialize project structure with SQLite database, crawler framework, and LLM-based JD analysis integration --- .gitignore | 3 + Makefile | 23 + cmd/hireme/main.go | 81 ++++ cmd/hireme/main_test.go | 109 +++++ data/config/targets.yaml | 9 + go.mod | 13 + go.sum | 16 + internal/analyzer/analyzer.go | 39 ++ internal/analyzer/deepseek.go | 118 +++++ internal/analyzer/gemini.go | 106 +++++ internal/analyzer/mock.go | 20 + internal/config/config.go | 53 +++ internal/crawler/crawler.go | 32 ++ internal/crawler/generic.go | 140 ++++++ internal/db/db.go | 86 ++++ internal/db/models.go | 55 +++ internal/db/queries.go | 554 ++++++++++++++++++++++++ internal/db/queries_test.go | 55 +++ internal/filter/filter.go | 34 ++ internal/filter/filter_test.go | 58 +++ internal/filter/hard_keyword.go | 48 +++ internal/filter/scoring.go | 59 +++ internal/service/pipeline.go | 226 ++++++++++ internal/ui/handlers.go | 280 ++++++++++++ internal/ui/templates.go | 688 ++++++++++++++++++++++++++++++ pkg/crawler/google/google.go | 400 +++++++++++++++++ pkg/crawler/google/google_test.go | 117 +++++ 27 files changed, 3422 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 cmd/hireme/main.go create mode 100644 cmd/hireme/main_test.go create mode 100644 data/config/targets.yaml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/analyzer/analyzer.go create mode 100644 internal/analyzer/deepseek.go create mode 100644 internal/analyzer/gemini.go create mode 100644 internal/analyzer/mock.go create mode 100644 internal/config/config.go create mode 100644 internal/crawler/crawler.go create mode 100644 internal/crawler/generic.go create mode 100644 internal/db/db.go create mode 100644 internal/db/models.go create mode 100644 internal/db/queries.go create mode 100644 internal/db/queries_test.go create mode 100644 internal/filter/filter.go create mode 100644 internal/filter/filter_test.go create mode 100644 internal/filter/hard_keyword.go create mode 100644 internal/filter/scoring.go create mode 100644 internal/service/pipeline.go create mode 100644 internal/ui/handlers.go create mode 100644 internal/ui/templates.go create mode 100644 pkg/crawler/google/google.go create mode 100644 pkg/crawler/google/google_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..014995e --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +data/db +vendor/ +build/ \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5f83aad --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +.PHONY: all build run test clean + +BINARY_NAME=hireme +DB_PATH=data/db/hireme.db +CONFIG_PATH=data/config/targets.yaml +PORT=8080 + +all: build + +build: + @mkdir -p build data/db data/config + GOSUMDB=off go build -o build/$(BINARY_NAME) ./cmd/hireme + +run: build + @mkdir -p data/db data/config + ./build/$(BINARY_NAME) --port $(PORT) --db $(DB_PATH) --target-config $(CONFIG_PATH) + +test: + GOSUMDB=off go test -v ./... + +clean: + rm -f build/$(BINARY_NAME) + rm -rf data/db/*.db diff --git a/cmd/hireme/main.go b/cmd/hireme/main.go new file mode 100644 index 0000000..e548596 --- /dev/null +++ b/cmd/hireme/main.go @@ -0,0 +1,81 @@ +package main + +import ( + "fmt" + "log" + "net/http" + "os" + "path/filepath" + + "github.com/hireme/hireme/internal/analyzer" + "github.com/hireme/hireme/internal/config" + "github.com/hireme/hireme/internal/db" + "github.com/hireme/hireme/internal/service" + "github.com/hireme/hireme/internal/ui" + "github.com/spf13/cobra" +) + +var cfg *config.Config + +func newRootCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "hireme", + Short: "Hireme (GJCC) - High Performance Job Application Command Center", + Long: `Hireme is a high-performance single-binary job application and tracking command center written in Go.`, + RunE: func(cmd *cobra.Command, args []string) error { + cfg.Normalize() + + log.Println("==========================================================") + log.Println("🚀 Starting Hireme (GJCC) Job Application Command Center") + log.Println("==========================================================") + log.Printf("[Config] Port: %s | DB Path: %s | Target Config: %s | Active LLM: %s", + cfg.Port, cfg.DBPath, cfg.TargetConfig, cfg.LLMProvider) + + // Ensure DB directory exists + dbDir := filepath.Dir(cfg.DBPath) + if dbDir != "." && dbDir != "" { + _ = os.MkdirAll(dbDir, 0755) + } + + // Initialize Database + database, err := db.InitDB(cfg.DBPath) + if err != nil { + return fmt.Errorf("fatal DB error: %w", err) + } + defer database.Close() + + // Initialize LLM Analyzer + llmAnalyzer, err := analyzer.NewAnalyzer(cfg) + if err != nil { + return fmt.Errorf("fatal Analyzer setup error: %w", err) + } + + // Initialize Pipeline Service + pipeline := service.NewPipelineService(database, llmAnalyzer) + + // Load targets from YAML config if database is empty + if err := pipeline.LoadTargetsFromYAML(cfg.TargetConfig); err != nil { + log.Printf("[Main] Target YAML loading status: %v", err) + } + + // Setup UI Server & Routes + server := ui.NewServer(cfg, database, pipeline) + mux := http.NewServeMux() + server.RegisterRoutes(mux) + + addr := fmt.Sprintf(":%s", cfg.Port) + log.Printf("✨ Hireme Command Center is live at http://localhost:%s", cfg.Port) + return http.ListenAndServe(addr, mux) + }, + } + + cfg = config.RegisterFlags(cmd.Flags()) + return cmd +} + +func main() { + rootCmd := newRootCmd() + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/cmd/hireme/main_test.go b/cmd/hireme/main_test.go new file mode 100644 index 0000000..82eaa6e --- /dev/null +++ b/cmd/hireme/main_test.go @@ -0,0 +1,109 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hireme/hireme/internal/analyzer" + "github.com/hireme/hireme/internal/config" + "github.com/hireme/hireme/internal/db" + "github.com/hireme/hireme/internal/service" + "github.com/hireme/hireme/internal/ui" +) + +func TestFullApplicationIntegration(t *testing.T) { + tempDir := t.TempDir() + dbPath := filepath.Join(tempDir, "test_hireme.db") + yamlPath := filepath.Join(tempDir, "test_targets.yaml") + + yamlContent := `targets: + - name: "Google Software Engineer (Seattle)" + target_url: "https://www.google.com/about/careers/applications/jobs/results/?employment_type=FULL_TIME&q=%22Software%20Engineer%22&location=Seattle%2C%20WA%2C%20USA&target_level=MID" + platform: "google_jobs" + is_active: true +` + if err := os.WriteFile(yamlPath, []byte(yamlContent), 0644); err != nil { + t.Fatalf("failed to create test targets.yaml: %v", err) + } + + cfg := &config.Config{ + Port: "8080", + DBPath: dbPath, + TargetConfig: yamlPath, + LLMProvider: "mock", + } + + database, err := db.InitDB(cfg.DBPath) + if err != nil { + t.Fatalf("InitDB failed: %v", err) + } + defer database.Close() + + llmAnalyzer, err := analyzer.NewAnalyzer(cfg) + if err != nil { + t.Fatalf("NewAnalyzer failed: %v", err) + } + + pipeline := service.NewPipelineService(database, llmAnalyzer) + + // Verify target loading from YAML into SQLite + if err := pipeline.LoadTargetsFromYAML(cfg.TargetConfig); err != nil { + t.Fatalf("LoadTargetsFromYAML failed: %v", err) + } + + targets, err := database.GetAllTargets() + if err != nil { + t.Fatalf("GetAllTargets failed: %v", err) + } + if len(targets) != 1 { + t.Errorf("expected 1 target from YAML, got %d", len(targets)) + } + + // Verify jobs start empty on clean SQLite DB startup (0 jobs) + jobs, err := database.GetJobsWithAnalysis() + if err != nil { + t.Fatalf("GetJobsWithAnalysis failed: %v", err) + } + if len(jobs) != 0 { + t.Errorf("expected 0 jobs on clean startup, got %d", len(jobs)) + } + + // Setup UI Server & routes + server := ui.NewServer(cfg, database, pipeline) + mux := http.NewServeMux() + server.RegisterRoutes(mux) + + // Test GET / + reqRoot := httptest.NewRequest("GET", "/", nil) + wRoot := httptest.NewRecorder() + mux.ServeHTTP(wRoot, reqRoot) + + if wRoot.Code != http.StatusOK { + t.Errorf("expected 200 OK for GET /, got %d", wRoot.Code) + } + if !strings.Contains(wRoot.Body.String(), "Hireme") { + t.Errorf("expected homepage HTML to contain 'Hireme'") + } + + // Test GET /targets + reqTargets := httptest.NewRequest("GET", "/targets", nil) + wTargets := httptest.NewRecorder() + mux.ServeHTTP(wTargets, reqTargets) + + if wTargets.Code != http.StatusOK { + t.Errorf("expected 200 OK for GET /targets, got %d", wTargets.Code) + } + + // Test GET /jobs + reqJobs := httptest.NewRequest("GET", "/jobs", nil) + wJobs := httptest.NewRecorder() + mux.ServeHTTP(wJobs, reqJobs) + + if wJobs.Code != http.StatusOK { + t.Errorf("expected 200 OK for GET /jobs, got %d", wJobs.Code) + } +} diff --git a/data/config/targets.yaml b/data/config/targets.yaml new file mode 100644 index 0000000..035f054 --- /dev/null +++ b/data/config/targets.yaml @@ -0,0 +1,9 @@ +targets: + - name: "Google Software Engineer (Seattle, MID Level)" + target_url: "https://www.google.com/about/careers/applications/jobs/results/?employment_type=FULL_TIME&q=%22Software%20Engineer%22&location=Seattle%2C%20WA%2C%20USA&target_level=MID" + platform: "google_jobs" + is_active: true + - name: "DeepMind" + target_url: "https://www.google.com/about/careers/applications/jobs/results?company=DeepMind" + platform: "google_jobs" + is_active: true diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..0e2fa19 --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module github.com/hireme/hireme + +go 1.26 + +require ( + github.com/mattn/go-sqlite3 v1.14.42 + github.com/spf13/cobra v1.9.1 + github.com/spf13/pflag v1.0.6 + golang.org/x/net v0.52.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..0492c1b --- /dev/null +++ b/go.sum @@ -0,0 +1,16 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mattn/go-sqlite3 v1.14.42 h1:MigqEP4ZmHw3aIdIT7T+9TLa90Z6smwcthx+Azv4Cgo= +github.com/mattn/go-sqlite3 v1.14.42/go.mod h1:pjEuOr8IwzLJP2MfGeTb0A35jauH+C2kbHKBr7yXKVQ= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/analyzer/analyzer.go b/internal/analyzer/analyzer.go new file mode 100644 index 0000000..2165537 --- /dev/null +++ b/internal/analyzer/analyzer.go @@ -0,0 +1,39 @@ +package analyzer + +import ( + "context" + "fmt" + "strings" + + "github.com/hireme/hireme/internal/config" +) + +type AnalysisOutput struct { + MatchedHighlights []string `json:"matched_highlights"` + MissingHighlights []string `json:"missing_highlights"` + TailoringSuggestions []string `json:"tailoring_suggestions"` +} + +type LLMAnalyzer interface { + AnalyzeJD(ctx context.Context, jobTitle, rawContent string) (*AnalysisOutput, error) +} + +func NewAnalyzer(cfg *config.Config) (LLMAnalyzer, error) { + provider := strings.ToLower(cfg.LLMProvider) + switch provider { + case "gemini": + if cfg.GeminiAPIKey == "" { + return nil, fmt.Errorf("GEMINI_API_KEY is required for provider 'gemini'") + } + return NewGeminiAnalyzer(cfg.GeminiAPIKey), nil + case "deepseek": + if cfg.DeepSeekAPIKey == "" { + return nil, fmt.Errorf("DEEPSEEK_API_KEY is required for provider 'deepseek'") + } + return NewDeepSeekAnalyzer(cfg.DeepSeekAPIKey, cfg.DeepSeekBaseURL), nil + case "mock": + return NewMockAnalyzer(), nil + default: + return NewMockAnalyzer(), nil + } +} diff --git a/internal/analyzer/deepseek.go b/internal/analyzer/deepseek.go new file mode 100644 index 0000000..f197e69 --- /dev/null +++ b/internal/analyzer/deepseek.go @@ -0,0 +1,118 @@ +package analyzer + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +type DeepSeekAnalyzer struct { + apiKey string + baseURL string + client *http.Client +} + +func NewDeepSeekAnalyzer(apiKey, baseURL string) *DeepSeekAnalyzer { + if baseURL == "" { + baseURL = "https://api.deepseek.com" + } + return &DeepSeekAnalyzer{ + apiKey: apiKey, + baseURL: strings.TrimRight(baseURL, "/"), + client: &http.Client{Timeout: 30 * time.Second}, + } +} + +func (d *DeepSeekAnalyzer) AnalyzeJD(ctx context.Context, jobTitle, rawContent string) (*AnalysisOutput, error) { + prompt := fmt.Sprintf(`You are an expert technical resume screener. Analyze the following job description for the title "%s". +Provide a structured JSON output with 3 fields: +- "matched_highlights": list of 3-5 core technical skills/requirements matched (e.g. ["Go (Golang)", "Microservices", "REST APIs"]). +- "missing_highlights": list of 1-3 missing skills, experience gaps, or extra requirements (e.g. ["10+ yrs experience", "Kubernetes administration"]). +- "tailoring_suggestions": list of 2-3 specific action points for tailoring a resume to this JD. + +Job Title: %s +Job Content: +%s`, jobTitle, jobTitle, rawContent) + + reqBody := map[string]interface{}{ + "model": "deepseek-chat", + "messages": []map[string]string{ + {"role": "system", "content": "You are a professional hiring assistant. You MUST respond ONLY with valid JSON matching {\"matched_highlights\": [...], \"missing_highlights\": [...], \"tailoring_suggestions\": [...]}."}, + {"role": "user", "content": prompt}, + }, + "response_format": map[string]string{ + "type": "json_object", + }, + } + + jsonBytes, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal DeepSeek payload: %w", err) + } + + url := fmt.Sprintf("%s/chat/completions", d.baseURL) + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonBytes)) + if err != nil { + return nil, fmt.Errorf("failed to create DeepSeek request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", d.apiKey)) + + resp, err := d.client.Do(req) + if err != nil { + return nil, fmt.Errorf("DeepSeek API call failed: %w", err) + } + defer resp.Body.Close() + + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read DeepSeek response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("DeepSeek API error (status %d): %s", resp.StatusCode, string(respBytes)) + } + + var dsResp struct { + Choices []struct { + Message struct { + Content string `json:"content"` + } `json:"message"` + } `json:"choices"` + } + + if err := json.Unmarshal(respBytes, &dsResp); err != nil { + return nil, fmt.Errorf("failed to decode DeepSeek envelope: %w", err) + } + + if len(dsResp.Choices) == 0 { + return nil, fmt.Errorf("empty choices in DeepSeek response") + } + + rawJSON := cleanJSONResponse(dsResp.Choices[0].Message.Content) + + var output AnalysisOutput + if err := json.Unmarshal([]byte(rawJSON), &output); err != nil { + return nil, fmt.Errorf("failed to parse DeepSeek JSON output: %w", err) + } + + return &output, nil +} + +func cleanJSONResponse(raw string) string { + cleaned := strings.TrimSpace(raw) + if strings.HasPrefix(cleaned, "```json") { + cleaned = strings.TrimPrefix(cleaned, "```json") + cleaned = strings.TrimSuffix(cleaned, "```") + } else if strings.HasPrefix(cleaned, "```") { + cleaned = strings.TrimPrefix(cleaned, "```") + cleaned = strings.TrimSuffix(cleaned, "```") + } + return strings.TrimSpace(cleaned) +} diff --git a/internal/analyzer/gemini.go b/internal/analyzer/gemini.go new file mode 100644 index 0000000..2efb265 --- /dev/null +++ b/internal/analyzer/gemini.go @@ -0,0 +1,106 @@ +package analyzer + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +type GeminiAnalyzer struct { + apiKey string + client *http.Client +} + +func NewGeminiAnalyzer(apiKey string) *GeminiAnalyzer { + return &GeminiAnalyzer{ + apiKey: apiKey, + client: &http.Client{Timeout: 30 * time.Second}, + } +} + +func (g *GeminiAnalyzer) AnalyzeJD(ctx context.Context, jobTitle, rawContent string) (*AnalysisOutput, error) { + prompt := fmt.Sprintf(`You are an expert technical resume screener. Analyze the following job description for the title "%s". +Provide a structured JSON output with 3 fields: +- "matched_highlights": list of 3-5 core technical skills/requirements matched (e.g. ["Go (Golang)", "Microservices", "REST APIs"]). +- "missing_highlights": list of 1-3 missing skills, experience gaps, or extra requirements (e.g. ["10+ yrs experience", "Kubernetes administration"]). +- "tailoring_suggestions": list of 2-3 specific action points for tailoring a resume to this JD. + +Job Title: %s +Job Content: +%s + +Respond STRICTLY with valid JSON matching the format: +{"matched_highlights": [...], "missing_highlights": [...], "tailoring_suggestions": [...]}`, jobTitle, jobTitle, rawContent) + + reqBody := map[string]interface{}{ + "contents": []map[string]interface{}{ + { + "parts": []map[string]string{ + {"text": prompt}, + }, + }, + }, + "generationConfig": map[string]interface{}{ + "response_mime_type": "application/json", + }, + } + + jsonBytes, err := json.Marshal(reqBody) + if err != nil { + return nil, fmt.Errorf("failed to marshal Gemini payload: %w", err) + } + + url := fmt.Sprintf("https://generativelanguage.googleapis.com/v1beta/models/gemini-1.5-flash:generateContent?key=%s", g.apiKey) + req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewBuffer(jsonBytes)) + if err != nil { + return nil, fmt.Errorf("failed to create Gemini HTTP request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := g.client.Do(req) + if err != nil { + return nil, fmt.Errorf("Gemini API call failed: %w", err) + } + defer resp.Body.Close() + + respBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read Gemini response: %w", err) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Gemini API error (status %d): %s", resp.StatusCode, string(respBytes)) + } + + var geminiResp struct { + Candidates []struct { + Content struct { + Parts []struct { + Text string `json:"text"` + } `json:"parts"` + } `json:"content"` + } `json:"candidates"` + } + + if err := json.Unmarshal(respBytes, &geminiResp); err != nil { + return nil, fmt.Errorf("failed to decode Gemini envelope: %w", err) + } + + if len(geminiResp.Candidates) == 0 || len(geminiResp.Candidates[0].Content.Parts) == 0 { + return nil, fmt.Errorf("empty candidates in Gemini response") + } + + rawJSON := geminiResp.Candidates[0].Content.Parts[0].Text + rawJSON = cleanJSONResponse(rawJSON) + + var output AnalysisOutput + if err := json.Unmarshal([]byte(rawJSON), &output); err != nil { + return nil, fmt.Errorf("failed to parse Gemini JSON output: %w", err) + } + + return &output, nil +} diff --git a/internal/analyzer/mock.go b/internal/analyzer/mock.go new file mode 100644 index 0000000..2efb20d --- /dev/null +++ b/internal/analyzer/mock.go @@ -0,0 +1,20 @@ +package analyzer + +import ( + "context" +) + +type MockAnalyzer struct{} + +func NewMockAnalyzer() *MockAnalyzer { + return &MockAnalyzer{} +} + +// AnalyzeJD returns empty slices for mock provider so no fake mock text is injected into side panel during debug +func (m *MockAnalyzer) AnalyzeJD(ctx context.Context, jobTitle, rawContent string) (*AnalysisOutput, error) { + return &AnalysisOutput{ + MatchedHighlights: []string{}, + MissingHighlights: []string{}, + TailoringSuggestions: []string{}, + }, nil +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..96409d0 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,53 @@ +package config + +import ( + "os" + "strings" + + "github.com/spf13/pflag" +) + +type Config struct { + Port string + DBPath string + TargetConfig string + LLMProvider string // "gemini", "deepseek", "mock" + GeminiAPIKey string + DeepSeekAPIKey string + DeepSeekBaseURL string +} + +func RegisterFlags(flags *pflag.FlagSet) *Config { + cfg := &Config{} + + flags.StringVarP(&cfg.Port, "port", "p", getEnvOrDefault("PORT", "8080"), "HTTP server port") + flags.StringVarP(&cfg.DBPath, "db", "d", getEnvOrDefault("DB_PATH", "data/db/hireme.db"), "Path to database file") + flags.StringVarP(&cfg.TargetConfig, "target-config", "c", getEnvOrDefault("TARGET_CONFIG", "data/config/targets.yaml"), "Path to targets YAML file") + flags.StringVarP(&cfg.LLMProvider, "llm-provider", "l", getEnvOrDefault("LLM_PROVIDER", ""), "Active LLM Provider (gemini, deepseek, mock)") + flags.StringVar(&cfg.GeminiAPIKey, "gemini-api-key", os.Getenv("GEMINI_API_KEY"), "Gemini API Key") + flags.StringVar(&cfg.DeepSeekAPIKey, "deepseek-api-key", os.Getenv("DEEPSEEK_API_KEY"), "DeepSeek API Key") + flags.StringVar(&cfg.DeepSeekBaseURL, "deepseek-base-url", getEnvOrDefault("DEEPSEEK_BASE_URL", "https://api.deepseek.com"), "DeepSeek Base URL") + + return cfg +} + +func (c *Config) Normalize() { + if c.LLMProvider == "" { + if c.GeminiAPIKey != "" { + c.LLMProvider = "gemini" + } else if c.DeepSeekAPIKey != "" { + c.LLMProvider = "deepseek" + } else { + c.LLMProvider = "mock" + } + } + c.LLMProvider = strings.ToLower(c.LLMProvider) +} + +func getEnvOrDefault(key, defaultValue string) string { + val := os.Getenv(key) + if val != "" { + return val + } + return defaultValue +} diff --git a/internal/crawler/crawler.go b/internal/crawler/crawler.go new file mode 100644 index 0000000..7979168 --- /dev/null +++ b/internal/crawler/crawler.go @@ -0,0 +1,32 @@ +package crawler + +import ( + "context" + "crypto/sha256" + "fmt" +) + +type DiscoveredJob struct { + CanonicalID string + ApplyURL string +} + +type ExtractedJob struct { + CanonicalID string + Title string + Company string + Location string + ApplyURL string + RawContent string +} + +func GenerateCanonicalID(company, title, applyURL string) string { + data := fmt.Sprintf("%s|%s|%s", company, title, applyURL) + return fmt.Sprintf("%x", sha256.Sum256([]byte(data))) +} + +type SiteCrawler interface { + CanHandle(platform string) bool + DiscoverJobs(ctx context.Context, targetURL string) ([]DiscoveredJob, error) + FetchJobDetail(ctx context.Context, applyURL string) (*ExtractedJob, error) +} diff --git a/internal/crawler/generic.go b/internal/crawler/generic.go new file mode 100644 index 0000000..5a3f78e --- /dev/null +++ b/internal/crawler/generic.go @@ -0,0 +1,140 @@ +package crawler + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" + + "golang.org/x/net/html" +) + +type GenericCrawler struct { + client *http.Client +} + +func NewGenericCrawler() *GenericCrawler { + return &GenericCrawler{ + client: &http.Client{ + Timeout: 15 * time.Second, + }, + } +} + +func (c *GenericCrawler) CanHandle(platform string) bool { + p := strings.ToLower(platform) + return p == "generic" || p == "greenhouse" || p == "lever" || p == "linkedin" +} + +func (c *GenericCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]DiscoveredJob, error) { + cid := GenerateCanonicalID(extractCompanyFromURL(targetURL), "Generic Job", targetURL) + return []DiscoveredJob{ + { + CanonicalID: cid, + ApplyURL: targetURL, + }, + }, nil +} + +func (c *GenericCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*ExtractedJob, error) { + req, err := http.NewRequestWithContext(ctx, "GET", applyURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP status error %d for URL %s", resp.StatusCode, applyURL) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + rawText := ExtractTextFromHTML(string(body)) + title := ExtractTitleFromHTML(string(body)) + if title == "" { + title = "Job Opportunity" + } + + company := extractCompanyFromURL(applyURL) + cid := GenerateCanonicalID(company, title, applyURL) + + return &ExtractedJob{ + CanonicalID: cid, + Title: title, + Company: company, + Location: "Remote / Listed Location", + ApplyURL: applyURL, + RawContent: rawText, + }, nil +} + +func ExtractTitleFromHTML(htmlContent string) string { + doc, err := html.Parse(strings.NewReader(htmlContent)) + if err != nil { + return "" + } + var title string + var f func(*html.Node) + f = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "title" && n.FirstChild != nil { + title = strings.TrimSpace(n.FirstChild.Data) + return + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + f(doc) + return title +} + +func ExtractTextFromHTML(htmlContent string) string { + doc, err := html.Parse(strings.NewReader(htmlContent)) + if err != nil { + return htmlContent + } + var b strings.Builder + var f func(*html.Node) + f = func(n *html.Node) { + if n.Type == html.ElementNode && (n.Data == "script" || n.Data == "style") { + return + } + if n.Type == html.TextNode { + txt := strings.TrimSpace(n.Data) + if txt != "" { + b.WriteString(txt) + b.WriteString(" ") + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + f(c) + } + } + f(doc) + return b.String() +} + +func extractCompanyFromURL(u string) string { + if strings.Contains(u, "google") { + return "Google" + } else if strings.Contains(u, "linkedin") { + return "LinkedIn Network" + } else if strings.Contains(u, "greenhouse") { + return "Greenhouse Client" + } else if strings.Contains(u, "lever") { + return "Lever Client" + } + return "Tech Corp" +} diff --git a/internal/db/db.go b/internal/db/db.go new file mode 100644 index 0000000..2be1128 --- /dev/null +++ b/internal/db/db.go @@ -0,0 +1,86 @@ +package db + +import ( + "database/sql" + "fmt" + "log" + + _ "github.com/mattn/go-sqlite3" +) + +type DB struct { + *sql.DB +} + +func InitDB(dbPath string) (*DB, error) { + if dbPath == "" { + dbPath = "data/db/hireme.db" + } + + conn, err := sql.Open("sqlite3", dbPath+"?_foreign_keys=on") + if err != nil { + return nil, fmt.Errorf("failed to open sqlite database: %w", err) + } + + if err := conn.Ping(); err != nil { + return nil, fmt.Errorf("failed to ping sqlite database: %w", err) + } + + database := &DB{DB: conn} + if err := database.migrate(); err != nil { + return nil, fmt.Errorf("failed to run sqlite migrations: %w", err) + } + + log.Printf("[DB] SQLite database initialized successfully at %s", dbPath) + return database, nil +} + +func (db *DB) migrate() error { + schema := ` + CREATE TABLE IF NOT EXISTS search_targets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + target_url TEXT NOT NULL, + platform TEXT NOT NULL, + is_active BOOLEAN NOT NULL DEFAULT 1, + last_crawled_at DATETIME + ); + + CREATE TABLE IF NOT EXISTS raw_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + canonical_id TEXT UNIQUE NOT NULL, + target_id INTEGER REFERENCES search_targets(id), + title TEXT NOT NULL, + company TEXT NOT NULL, + location TEXT, + apply_url TEXT NOT NULL, + raw_content TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'PENDING', + crawled_at DATETIME DEFAULT CURRENT_TIMESTAMP + ); + + CREATE TABLE IF NOT EXISTS job_analyses ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_job_id INTEGER UNIQUE NOT NULL REFERENCES raw_jobs(id) ON DELETE CASCADE, + score REAL DEFAULT 0.0, + is_qualified BOOLEAN NOT NULL DEFAULT 0, + matched_highlights JSON, + missing_highlights JSON, + tailoring_suggestions JSON, + status TEXT NOT NULL DEFAULT 'PENDING', + analyzed_at DATETIME + ); + + CREATE TABLE IF NOT EXISTS job_applications ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_job_id INTEGER UNIQUE NOT NULL REFERENCES raw_jobs(id) ON DELETE CASCADE, + status TEXT NOT NULL DEFAULT 'SAVED', + notes TEXT, + applied_at DATETIME, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP + ); + ` + + _, err := db.Exec(schema) + return err +} diff --git a/internal/db/models.go b/internal/db/models.go new file mode 100644 index 0000000..ec67286 --- /dev/null +++ b/internal/db/models.go @@ -0,0 +1,55 @@ +package db + +import ( + "time" +) + +type SearchTarget struct { + ID int64 `json:"id"` + Name string `json:"name"` + TargetURL string `json:"target_url"` + Platform string `json:"platform"` // 'linkedin', 'google_jobs', 'greenhouse', 'lever' + IsActive bool `json:"is_active"` + LastCrawledAt *time.Time `json:"last_crawled_at"` +} + +type RawJob struct { + ID int64 `json:"id"` + CanonicalID string `json:"canonical_id"` // SHA256(company + title + applyURL) + TargetID *int64 `json:"target_id"` + Title string `json:"title"` + Company string `json:"company"` + Location string `json:"location"` + ApplyURL string `json:"apply_url"` + RawContent string `json:"raw_content"` + Status string `json:"status"` // 'PENDING', 'PROCESSED', 'FAILED', 'DUPLICATE' + CrawledAt time.Time `json:"crawled_at"` +} + +type JobAnalysis struct { + ID int64 `json:"id"` + RawJobID int64 `json:"raw_job_id"` + Score float64 `json:"score"` + IsQualified bool `json:"is_qualified"` + MatchedHighlights string `json:"matched_highlights"` // JSON array string + MissingHighlights string `json:"missing_highlights"` // JSON array string + TailoringSuggestions string `json:"tailoring_suggestions"` // JSON array string + Status string `json:"status"` // 'PENDING', 'PARSED', 'FILTERED_OUT', 'FAILED' + AnalyzedAt *time.Time `json:"analyzed_at"` +} + +type JobApplication struct { + ID int64 `json:"id"` + RawJobID int64 `json:"raw_job_id"` + Status string `json:"status"` // 'SAVED', 'APPLIED', 'INTERVIEW', 'REJECTED', 'OFFER' + Notes string `json:"notes"` + AppliedAt *time.Time `json:"applied_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// JobWithAnalysis combines RawJob and JobAnalysis for UI rendering +type JobWithAnalysis struct { + RawJob + Analysis *JobAnalysis `json:"analysis"` + App *JobApplication `json:"application"` +} diff --git a/internal/db/queries.go b/internal/db/queries.go new file mode 100644 index 0000000..4220c60 --- /dev/null +++ b/internal/db/queries.go @@ -0,0 +1,554 @@ +package db + +import ( + "database/sql" + "math" + "regexp" + "strings" + "time" +) + +// Target Queries +func (db *DB) CreateTarget(name, targetURL, platform string) (*SearchTarget, error) { + query := `INSERT INTO search_targets (name, target_url, platform, is_active) VALUES (?, ?, ?, 1) RETURNING id, name, target_url, platform, is_active, last_crawled_at` + var target SearchTarget + var lastCrawled sql.NullTime + err := db.QueryRow(query, name, targetURL, platform).Scan( + &target.ID, &target.Name, &target.TargetURL, &target.Platform, &target.IsActive, &lastCrawled, + ) + if err != nil { + return nil, err + } + if lastCrawled.Valid { + target.LastCrawledAt = &lastCrawled.Time + } + return &target, nil +} + +func (db *DB) GetActiveTargets() ([]SearchTarget, error) { + query := `SELECT id, name, target_url, platform, is_active, last_crawled_at FROM search_targets WHERE is_active = 1` + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + var targets []SearchTarget + for rows.Next() { + var t SearchTarget + var lastCrawled sql.NullTime + if err := rows.Scan(&t.ID, &t.Name, &t.TargetURL, &t.Platform, &t.IsActive, &lastCrawled); err != nil { + return nil, err + } + if lastCrawled.Valid { + t.LastCrawledAt = &lastCrawled.Time + } + targets = append(targets, t) + } + return targets, nil +} + +func (db *DB) GetAllTargets() ([]SearchTarget, error) { + query := `SELECT id, name, target_url, platform, is_active, last_crawled_at FROM search_targets ORDER BY id DESC` + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + var targets []SearchTarget + for rows.Next() { + var t SearchTarget + var lastCrawled sql.NullTime + if err := rows.Scan(&t.ID, &t.Name, &t.TargetURL, &t.Platform, &t.IsActive, &lastCrawled); err != nil { + return nil, err + } + if lastCrawled.Valid { + t.LastCrawledAt = &lastCrawled.Time + } + targets = append(targets, t) + } + return targets, nil +} + +func (db *DB) ToggleTargetStatus(id int64) error { + query := `UPDATE search_targets SET is_active = CASE WHEN is_active = 1 THEN 0 ELSE 1 END WHERE id = ?` + _, err := db.Exec(query, id) + return err +} + +func (db *DB) UpdateTargetCrawledAt(id int64) error { + query := `UPDATE search_targets SET last_crawled_at = CURRENT_TIMESTAMP WHERE id = ?` + _, err := db.Exec(query, id) + return err +} + +// Raw Jobs Queries & Vector Similarity Deduplication +func (db *DB) InsertRawJob(canonicalID string, targetID *int64, title, company, location, applyURL, rawContent string) (*RawJob, bool, error) { + // Deduplication Check 1 & 2: Check canonical_id exist AND vector similarity >= 99% + existingJobs, err := db.getAllRawJobsForDeduplication() + if err == nil { + for _, existing := range existingJobs { + if existing.CanonicalID == canonicalID { + return nil, false, nil // Duplicate Canonical ID + } + + // Vector Cosine Similarity check on raw_content (run only for substantial bodies >100 chars) + if len(existing.RawContent) >= 100 && len(rawContent) >= 100 { + similarity := CalculateCosineSimilarity(existing.RawContent, rawContent) + if similarity >= 0.99 { + return nil, false, nil // Content >= 99% vector match -> Deduplicate + } + } + } + } + + query := ` + INSERT INTO raw_jobs (canonical_id, target_id, title, company, location, apply_url, raw_content, status) + VALUES (?, ?, ?, ?, ?, ?, ?, 'PENDING') + ON CONFLICT(canonical_id) DO NOTHING + RETURNING id, canonical_id, target_id, title, company, location, apply_url, raw_content, status, crawled_at + ` + var job RawJob + var tID sql.NullInt64 + err = db.QueryRow(query, canonicalID, targetID, title, company, location, applyURL, rawContent).Scan( + &job.ID, &job.CanonicalID, &tID, &job.Title, &job.Company, &job.Location, &job.ApplyURL, &job.RawContent, &job.Status, &job.CrawledAt, + ) + if err == sql.ErrNoRows { + return nil, false, nil // Duplicate job + } + if err != nil { + return nil, false, err + } + if tID.Valid { + job.TargetID = &tID.Int64 + } + return &job, true, nil +} + +func (db *DB) CanonicalIDExists(canonicalID string) (bool, error) { + query := `SELECT EXISTS(SELECT 1 FROM raw_jobs WHERE canonical_id = ?)` + var exists bool + err := db.QueryRow(query, canonicalID).Scan(&exists) + return exists, err +} + +func (db *DB) getAllRawJobsForDeduplication() ([]RawJob, error) { + query := `SELECT id, canonical_id, raw_content FROM raw_jobs` + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + var jobs []RawJob + for rows.Next() { + var j RawJob + if err := rows.Scan(&j.ID, &j.CanonicalID, &j.RawContent); err != nil { + return nil, err + } + jobs = append(jobs, j) + } + return jobs, nil +} + +// CalculateCosineSimilarity tokenizes text into term frequency vectors and returns cosine similarity in range [0.0, 1.0] +func CalculateCosineSimilarity(text1, text2 string) float64 { + vec1 := tokenizeToVector(text1) + vec2 := tokenizeToVector(text2) + + if len(vec1) == 0 || len(vec2) == 0 { + return 0.0 + } + + var dotProduct float64 + for term, freq1 := range vec1 { + if freq2, exists := vec2[term]; exists { + dotProduct += float64(freq1 * freq2) + } + } + + var norm1 float64 + for _, freq := range vec1 { + norm1 += float64(freq * freq) + } + norm1 = math.Sqrt(norm1) + + var norm2 float64 + for _, freq := range vec2 { + norm2 += float64(freq * freq) + } + norm2 = math.Sqrt(norm2) + + if norm1 == 0 || norm2 == 0 { + return 0.0 + } + + return dotProduct / (norm1 * norm2) +} + +var wordRegex = regexp.MustCompile(`[a-zA-Z0-9]+`) + +var stopWords = map[string]bool{ + "the": true, "and": true, "for": true, "with": true, "this": true, + "that": true, "from": true, "are": true, "was": true, "were": true, + "google": true, "position": true, "apply": true, "url": true, "location": true, + "seattle": true, "usa": true, "job": true, "http": true, "https": true, +} + +func tokenizeToVector(text string) map[string]int { + words := wordRegex.FindAllString(strings.ToLower(text), -1) + freqMap := make(map[string]int) + for _, w := range words { + if len(w) > 2 && !stopWords[w] { + freqMap[w]++ + } + } + return freqMap +} + +func (db *DB) GetPendingRawJobs() ([]RawJob, error) { + query := `SELECT id, canonical_id, target_id, title, company, location, apply_url, raw_content, status, crawled_at FROM raw_jobs WHERE status = 'PENDING'` + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + var jobs []RawJob + for rows.Next() { + var j RawJob + var tID sql.NullInt64 + if err := rows.Scan(&j.ID, &j.CanonicalID, &tID, &j.Title, &j.Company, &j.Location, &j.ApplyURL, &j.RawContent, &j.Status, &j.CrawledAt); err != nil { + return nil, err + } + if tID.Valid { + j.TargetID = &tID.Int64 + } + jobs = append(jobs, j) + } + return jobs, nil +} + +func (db *DB) UpdateRawJobStatus(id int64, status string) error { + query := `UPDATE raw_jobs SET status = ? WHERE id = ?` + _, err := db.Exec(query, status, id) + return err +} + +// Job Analyses Queries +func (db *DB) CreateJobAnalysis(rawJobID int64, score float64, isQualified bool, status string) error { + query := ` + INSERT INTO job_analyses (raw_job_id, score, is_qualified, status) + VALUES (?, ?, ?, ?) + ON CONFLICT(raw_job_id) DO UPDATE SET score=excluded.score, is_qualified=excluded.is_qualified, status=excluded.status + ` + _, err := db.Exec(query, rawJobID, score, isQualified, status) + return err +} + +func (db *DB) UpdateJobAnalysisLLM(rawJobID int64, matchedJSON, missingJSON, suggestionsJSON string) error { + query := ` + UPDATE job_analyses + SET matched_highlights = ?, missing_highlights = ?, tailoring_suggestions = ?, status = 'PARSED', analyzed_at = CURRENT_TIMESTAMP + WHERE raw_job_id = ? + ` + _, err := db.Exec(query, matchedJSON, missingJSON, suggestionsJSON, rawJobID) + return err +} + +func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) { + query := ` + SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at, + a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), a.status, a.analyzed_at + FROM raw_jobs r + JOIN job_analyses a ON r.id = a.raw_job_id + WHERE a.is_qualified = 1 AND a.status = 'PENDING' + ` + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []JobWithAnalysis + for rows.Next() { + var item JobWithAnalysis + var tID sql.NullInt64 + var aID int64 + var score float64 + var isQualified bool + var matched, missing, suggestions, aStatus string + var analyzedAt sql.NullTime + + err := rows.Scan( + &item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt, + &aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt, + ) + if err != nil { + return nil, err + } + if tID.Valid { + item.TargetID = &tID.Int64 + } + analysis := &JobAnalysis{ + ID: aID, + RawJobID: item.ID, + Score: score, + IsQualified: isQualified, + MatchedHighlights: matched, + MissingHighlights: missing, + TailoringSuggestions: suggestions, + Status: aStatus, + } + if analyzedAt.Valid { + analysis.AnalyzedAt = &analyzedAt.Time + } + item.Analysis = analysis + results = append(results, item) + } + return results, nil +} + +func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) { + query := ` + SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at, + a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at, + app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at + FROM raw_jobs r + LEFT JOIN job_analyses a ON r.id = a.raw_job_id + LEFT JOIN job_applications app ON r.id = app.raw_job_id + ORDER BY COALESCE(a.score, 0) DESC, r.crawled_at DESC + ` + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []JobWithAnalysis + for rows.Next() { + var item JobWithAnalysis + var tID sql.NullInt64 + var aID, appID sql.NullInt64 + var score sql.NullFloat64 + var isQualified sql.NullBool + var matched, missing, suggestions, aStatus sql.NullString + var analyzedAt sql.NullTime + var appStatus, notes sql.NullString + var appliedAt sql.NullTime + var updatedAt sql.NullTime + + err := rows.Scan( + &item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt, + &aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt, + &appID, &appStatus, ¬es, &appliedAt, &updatedAt, + ) + if err != nil { + return nil, err + } + if tID.Valid { + item.TargetID = &tID.Int64 + } + if aID.Valid { + analysis := &JobAnalysis{ + ID: aID.Int64, + RawJobID: item.ID, + Score: score.Float64, + IsQualified: isQualified.Bool, + MatchedHighlights: matched.String, + MissingHighlights: missing.String, + TailoringSuggestions: suggestions.String, + Status: aStatus.String, + } + if analyzedAt.Valid { + analysis.AnalyzedAt = &analyzedAt.Time + } + item.Analysis = analysis + } + if appID.Valid { + app := &JobApplication{ + ID: appID.Int64, + RawJobID: item.ID, + Status: appStatus.String, + Notes: notes.String, + UpdatedAt: updatedAt.Time, + } + if appliedAt.Valid { + app.AppliedAt = &appliedAt.Time + } + item.App = app + } + results = append(results, item) + } + return results, nil +} + +func (db *DB) GetJobByID(rawJobID int64) (*JobWithAnalysis, error) { + query := ` + SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at, + a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at, + app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at + FROM raw_jobs r + LEFT JOIN job_analyses a ON r.id = a.raw_job_id + LEFT JOIN job_applications app ON r.id = app.raw_job_id + WHERE r.id = ? + ` + row := db.QueryRow(query, rawJobID) + var item JobWithAnalysis + var tID sql.NullInt64 + var aID, appID sql.NullInt64 + var score sql.NullFloat64 + var isQualified sql.NullBool + var matched, missing, suggestions, aStatus sql.NullString + var analyzedAt sql.NullTime + var appStatus, notes sql.NullString + var appliedAt sql.NullTime + var updatedAt sql.NullTime + + err := row.Scan( + &item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt, + &aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt, + &appID, &appStatus, ¬es, &appliedAt, &updatedAt, + ) + if err != nil { + return nil, err + } + if tID.Valid { + item.TargetID = &tID.Int64 + } + if aID.Valid { + analysis := &JobAnalysis{ + ID: aID.Int64, + RawJobID: item.ID, + Score: score.Float64, + IsQualified: isQualified.Bool, + MatchedHighlights: matched.String, + MissingHighlights: missing.String, + TailoringSuggestions: suggestions.String, + Status: aStatus.String, + } + if analyzedAt.Valid { + analysis.AnalyzedAt = &analyzedAt.Time + } + item.Analysis = analysis + } + if appID.Valid { + app := &JobApplication{ + ID: appID.Int64, + RawJobID: item.ID, + Status: appStatus.String, + Notes: notes.String, + UpdatedAt: updatedAt.Time, + } + if appliedAt.Valid { + app.AppliedAt = &appliedAt.Time + } + item.App = app + } + return &item, nil +} + +// Applications Queries +func (db *DB) CreateOrUpdateApplication(rawJobID int64, status, notes string) (*JobApplication, error) { + now := time.Now() + var appliedAt *time.Time + if status == "APPLIED" { + appliedAt = &now + } + + query := ` + INSERT INTO job_applications (raw_job_id, status, notes, applied_at, updated_at) + VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(raw_job_id) DO UPDATE SET + status=excluded.status, + notes=COALESCE(excluded.notes, job_applications.notes), + applied_at=COALESCE(excluded.applied_at, job_applications.applied_at), + updated_at=CURRENT_TIMESTAMP + RETURNING id, raw_job_id, status, notes, applied_at, updated_at + ` + var app JobApplication + var appAt sql.NullTime + err := db.QueryRow(query, rawJobID, status, notes, appliedAt).Scan( + &app.ID, &app.RawJobID, &app.Status, &app.Notes, &appAt, &app.UpdatedAt, + ) + if err != nil { + return nil, err + } + if appAt.Valid { + app.AppliedAt = &appAt.Time + } + return &app, nil +} + +func (db *DB) GetApplications() ([]JobWithAnalysis, error) { + query := ` + SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at, + a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at, + app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at + FROM job_applications app + JOIN raw_jobs r ON app.raw_job_id = r.id + LEFT JOIN job_analyses a ON r.id = a.raw_job_id + ORDER BY app.updated_at DESC + ` + rows, err := db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + var results []JobWithAnalysis + for rows.Next() { + var item JobWithAnalysis + var tID sql.NullInt64 + var aID, appID sql.NullInt64 + var score sql.NullFloat64 + var isQualified sql.NullBool + var matched, missing, suggestions, aStatus sql.NullString + var analyzedAt sql.NullTime + var appStatus, notes sql.NullString + var appliedAt sql.NullTime + var updatedAt time.Time + + err := rows.Scan( + &item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt, + &aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt, + &appID, &appStatus, ¬es, &appliedAt, &updatedAt, + ) + if err != nil { + return nil, err + } + if tID.Valid { + item.TargetID = &tID.Int64 + } + if aID.Valid { + analysis := &JobAnalysis{ + ID: aID.Int64, + RawJobID: item.ID, + Score: score.Float64, + IsQualified: isQualified.Bool, + MatchedHighlights: matched.String, + MissingHighlights: missing.String, + TailoringSuggestions: suggestions.String, + Status: aStatus.String, + } + if analyzedAt.Valid { + analysis.AnalyzedAt = &analyzedAt.Time + } + item.Analysis = analysis + } + app := &JobApplication{ + ID: appID.Int64, + RawJobID: item.ID, + Status: appStatus.String, + Notes: notes.String, + UpdatedAt: updatedAt, + } + if appliedAt.Valid { + app.AppliedAt = &appliedAt.Time + } + item.App = app + results = append(results, item) + } + return results, nil +} diff --git a/internal/db/queries_test.go b/internal/db/queries_test.go new file mode 100644 index 0000000..015475a --- /dev/null +++ b/internal/db/queries_test.go @@ -0,0 +1,55 @@ +package db + +import ( + "path/filepath" + "testing" +) + +func TestVectorSimilarityDeduplication(t *testing.T) { + tempDir := t.TempDir() + dbPath := filepath.Join(tempDir, "test_dedup.db") + + database, err := InitDB(dbPath) + if err != nil { + t.Fatalf("failed to init db: %v", err) + } + defer database.Close() + + contentA := "Senior Software Engineer - Go Distributed Systems. Experience with microservices, gRPC, and SQLite database pipelines." + contentSimilar := "Senior Software Engineer - Go Distributed Systems. Experience with microservices, gRPC, and SQLite database pipelines." + contentDifferent := "Marketing Specialist - Social Media Campaign Manager. Experience with SEO, Google Ads, and brand strategy." + + // 1. Insert job A + jobA, inserted1, err := database.InsertRawJob("canonical-1", nil, "Software Engineer", "Google", "Seattle", "https://google.com/job/1", contentA) + if err != nil || !inserted1 { + t.Fatalf("expected job A to be inserted: err=%v, inserted=%v", err, inserted1) + } + + // 2. Insert job with identical/99%+ similar content -> should be deduplicated! + _, inserted2, err := database.InsertRawJob("canonical-2-different-id", nil, "Learn more about Software Engineer", "Google", "Seattle", "https://google.com/job/1-dup", contentSimilar) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if inserted2 { + t.Errorf("expected 99%% similar content job to be deduplicated (inserted=false), got inserted=true") + } + + // 3. Insert job with distinct content -> should insert normally! + _, inserted3, err := database.InsertRawJob("canonical-3", nil, "Marketing Specialist", "Google", "Seattle", "https://google.com/job/3", contentDifferent) + if err != nil || !inserted3 { + t.Fatalf("expected distinct job to insert normally: err=%v, inserted=%v", err, inserted3) + } + + // Verify only 2 raw jobs exist in DB + allJobs, err := database.GetPendingRawJobs() + if err != nil { + t.Fatalf("GetPendingRawJobs failed: %v", err) + } + if len(allJobs) != 2 { + t.Errorf("expected 2 raw jobs in DB after deduplication, got %d", len(allJobs)) + } + + if allJobs[0].ID != jobA.ID { + t.Errorf("first job mismatch") + } +} diff --git a/internal/filter/filter.go b/internal/filter/filter.go new file mode 100644 index 0000000..f2da902 --- /dev/null +++ b/internal/filter/filter.go @@ -0,0 +1,34 @@ +package filter + +import "context" + +type FilterResult struct { + Keep bool + Score float64 + Reason string +} + +type JobFilter interface { + Name() string + Process(ctx context.Context, title, rawContent string) (FilterResult, error) +} + +type FilterPipeline struct { + filters []JobFilter +} + +func NewPipeline(filters ...JobFilter) *FilterPipeline { + return &FilterPipeline{filters: filters} +} + +func (p *FilterPipeline) Execute(ctx context.Context, title, rawContent string) (bool, float64) { + var totalScore float64 + for _, f := range p.filters { + res, err := f.Process(ctx, title, rawContent) + if err != nil || !res.Keep { + return false, 0 + } + totalScore += res.Score + } + return true, totalScore +} diff --git a/internal/filter/filter_test.go b/internal/filter/filter_test.go new file mode 100644 index 0000000..3a90fa8 --- /dev/null +++ b/internal/filter/filter_test.go @@ -0,0 +1,58 @@ +package filter + +import ( + "context" + "testing" +) + +func TestHardKeywordFilter(t *testing.T) { + ctx := context.Background() + fk := NewHardKeywordFilter([]string{"DevOps", "No Sponsorship"}) + + // Should fail due to DevOps + res1, err := fk.Process(ctx, "Senior DevOps Engineer", "We build pipelines.") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if res1.Keep { + t.Errorf("expected Keep=false for DevOps, got true") + } + + // Should pass + res2, err := fk.Process(ctx, "Senior Backend Engineer", "We build scalable APIs in Go.") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !res2.Keep { + t.Errorf("expected Keep=true for Backend Engineer, got false") + } +} + +func TestFilterPipeline(t *testing.T) { + ctx := context.Background() + hardFilter := NewHardKeywordFilter([]string{"Unpaid", "No Sponsorship"}) + scoringFilter := NewScoringFilter([]KeywordWeight{ + {Keyword: "Go", Weight: 30.0}, + {Keyword: "Backend", Weight: 20.0}, + }) + + pipeline := NewPipeline(hardFilter, scoringFilter) + + // Test job 1: Unpaid job -> rejected by hard filter + keep1, score1 := pipeline.Execute(ctx, "Backend Intern", "Unpaid position working with Go.") + if keep1 { + t.Errorf("expected job 1 to be filtered out, got keep=true") + } + if score1 != 0 { + t.Errorf("expected score 0 for rejected job, got %f", score1) + } + + // Test job 2: Qualified job -> passes both filters and accumulates score + keep2, score2 := pipeline.Execute(ctx, "Senior Go Backend Engineer", "Building high throughput services.") + if !keep2 { + t.Errorf("expected job 2 to pass filter pipeline, got keep=false") + } + if score2 != 50.0 { + t.Errorf("expected score 50.0 (Go + Backend), got %f", score2) + } +} diff --git a/internal/filter/hard_keyword.go b/internal/filter/hard_keyword.go new file mode 100644 index 0000000..ff8a22e --- /dev/null +++ b/internal/filter/hard_keyword.go @@ -0,0 +1,48 @@ +package filter + +import ( + "context" + "fmt" + "strings" +) + +type HardKeywordFilter struct { + Blacklist []string +} + +func NewHardKeywordFilter(blacklist []string) *HardKeywordFilter { + if len(blacklist) == 0 { + blacklist = []string{ + "no sponsorship", + "us citizen only", + "unpaid", + "volunteer", + } + } + return &HardKeywordFilter{Blacklist: blacklist} +} + +func (f *HardKeywordFilter) Name() string { + return "HardKeywordFilter" +} + +func (f *HardKeywordFilter) Process(ctx context.Context, title, rawContent string) (FilterResult, error) { + combined := strings.ToLower(title + " " + rawContent) + + for _, kw := range f.Blacklist { + term := strings.ToLower(strings.TrimSpace(kw)) + if term != "" && strings.Contains(combined, term) { + return FilterResult{ + Keep: false, + Score: 0, + Reason: fmt.Sprintf("Contains blacklisted keyword: %q", kw), + }, nil + } + } + + return FilterResult{ + Keep: true, + Score: 0, + Reason: "Passed hard keyword check", + }, nil +} diff --git a/internal/filter/scoring.go b/internal/filter/scoring.go new file mode 100644 index 0000000..b6fa199 --- /dev/null +++ b/internal/filter/scoring.go @@ -0,0 +1,59 @@ +package filter + +import ( + "context" + "fmt" + "strings" +) + +type KeywordWeight struct { + Keyword string + Weight float64 +} + +type ScoringFilter struct { + Weights []KeywordWeight +} + +func NewScoringFilter(weights []KeywordWeight) *ScoringFilter { + if len(weights) == 0 { + weights = []KeywordWeight{ + {Keyword: "go", Weight: 25.0}, + {Keyword: "golang", Weight: 25.0}, + {Keyword: "backend", Weight: 15.0}, + {Keyword: "distributed systems", Weight: 20.0}, + {Keyword: "microservices", Weight: 15.0}, + {Keyword: "sqlite", Weight: 10.0}, + {Keyword: "postgres", Weight: 10.0}, + {Keyword: "api", Weight: 10.0}, + {Keyword: "full stack", Weight: 10.0}, + {Keyword: "rest", Weight: 5.0}, + {Keyword: "grpc", Weight: 15.0}, + } + } + return &ScoringFilter{Weights: weights} +} + +func (s *ScoringFilter) Name() string { + return "ScoringFilter" +} + +func (s *ScoringFilter) Process(ctx context.Context, title, rawContent string) (FilterResult, error) { + combined := strings.ToLower(title + " " + rawContent) + var totalScore float64 + var matched []string + + for _, kw := range s.Weights { + term := strings.ToLower(strings.TrimSpace(kw.Keyword)) + if term != "" && strings.Contains(combined, term) { + totalScore += kw.Weight + matched = append(matched, kw.Keyword) + } + } + + return FilterResult{ + Keep: true, + Score: totalScore, + Reason: fmt.Sprintf("Matched %d keywords: %v", len(matched), matched), + }, nil +} diff --git a/internal/service/pipeline.go b/internal/service/pipeline.go new file mode 100644 index 0000000..5821f54 --- /dev/null +++ b/internal/service/pipeline.go @@ -0,0 +1,226 @@ +package service + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + "time" + + "github.com/hireme/hireme/internal/analyzer" + "github.com/hireme/hireme/internal/crawler" + "github.com/hireme/hireme/internal/db" + "github.com/hireme/hireme/internal/filter" + "github.com/hireme/hireme/pkg/crawler/google" + "gopkg.in/yaml.v3" +) + +type TargetConfigYAML struct { + Targets []struct { + Name string `yaml:"name"` + TargetURL string `yaml:"target_url"` + Platform string `yaml:"platform"` + IsActive bool `yaml:"is_active"` + } `yaml:"targets"` +} + +type PipelineService struct { + db *db.DB + crawlers []crawler.SiteCrawler + filterPipe *filter.FilterPipeline + llmAnalyzer analyzer.LLMAnalyzer +} + +func NewPipelineService(database *db.DB, llmAnalyzer analyzer.LLMAnalyzer) *PipelineService { + hardFilter := filter.NewHardKeywordFilter(nil) + scoringFilter := filter.NewScoringFilter(nil) + pipeline := filter.NewPipeline(hardFilter, scoringFilter) + + crawlers := []crawler.SiteCrawler{ + google.NewGoogleCrawler(), + crawler.NewGenericCrawler(), + } + + return &PipelineService{ + db: database, + crawlers: crawlers, + filterPipe: pipeline, + llmAnalyzer: llmAnalyzer, + } +} + +// LoadTargetsFromYAML loads search targets from YAML configuration file if DB targets table is empty +func (s *PipelineService) LoadTargetsFromYAML(yamlPath string) error { + targets, err := s.db.GetAllTargets() + if err != nil { + return err + } + if len(targets) > 0 { + return nil + } + + data, err := os.ReadFile(yamlPath) + if err != nil { + return fmt.Errorf("could not read targets YAML config at %s: %w", yamlPath, err) + } + + var configData TargetConfigYAML + if err := yaml.Unmarshal(data, &configData); err != nil { + return fmt.Errorf("failed to parse YAML targets config: %w", err) + } + + for _, t := range configData.Targets { + _, err := s.db.CreateTarget(t.Name, t.TargetURL, t.Platform) + if err != nil { + log.Printf("[Pipeline] Error loading target %s: %v", t.Name, err) + } + } + + log.Printf("[Pipeline] Loaded %d search targets from YAML config (%s)", len(configData.Targets), yamlPath) + return nil +} + +func (s *PipelineService) getCrawlerForPlatform(platform string) crawler.SiteCrawler { + for _, c := range s.crawlers { + if c.CanHandle(platform) { + return c + } + } + return s.crawlers[len(s.crawlers)-1] // fallback to generic +} + +// RunPipeline executes Stage 1 (Ingestion) and Stage 2 (Filtering & LLM Enrichment) +func (s *PipelineService) RunPipeline(ctx context.Context) (int, int, error) { + log.Println("[Pipeline] Starting ingestion & processing run...") + + // 1. Fetch active targets + targets, err := s.db.GetActiveTargets() + if err != nil { + return 0, 0, fmt.Errorf("failed to get active targets: %w", err) + } + + newJobsCount := 0 + for _, target := range targets { + log.Printf("[Pipeline] Discovering jobs for target #%d: %s (%s)", target.ID, target.Name, target.TargetURL) + c := s.getCrawlerForPlatform(target.Platform) + discovered, err := c.DiscoverJobs(ctx, target.TargetURL) + if err != nil { + log.Printf("[Pipeline] DiscoverJobs error for target #%d: %v", target.ID, err) + continue + } + + log.Printf("[Pipeline] Discovered %d potential job links for target #%d", len(discovered), target.ID) + + for _, disc := range discovered { + // Check DB deduplication first + exists, err := s.db.CanonicalIDExists(disc.CanonicalID) + if err == nil && exists { + continue // Skip existing job link! + } + + // Fetch & parse detail page for new job only + extracted, err := c.FetchJobDetail(ctx, disc.ApplyURL) + if err != nil { + log.Printf("[Pipeline] FetchJobDetail error for %s: %v", disc.ApplyURL, err) + continue + } + + tID := target.ID + _, inserted, err := s.db.InsertRawJob(extracted.CanonicalID, &tID, extracted.Title, extracted.Company, extracted.Location, extracted.ApplyURL, extracted.RawContent) + if err != nil { + log.Printf("[Pipeline] Error inserting job %s: %v", extracted.Title, err) + continue + } + if inserted { + newJobsCount++ + } + + // Polite delay to prevent HTTP 429 rate-limiting from Google + time.Sleep(500 * time.Millisecond) + } + + _ = s.db.UpdateTargetCrawledAt(target.ID) + } + + // 2. Execute Stage 2 Filter Pipeline on pending raw_jobs + pendingJobs, err := s.db.GetPendingRawJobs() + if err != nil { + return newJobsCount, 0, fmt.Errorf("failed to fetch pending jobs: %w", err) + } + + qualifiedCount := 0 + for _, job := range pendingJobs { + keep, score := s.filterPipe.Execute(ctx, job.Title, job.RawContent) + + if !keep { + log.Printf("[Pipeline] Job #%d (%s) FILTERED OUT by keyword pipeline", job.ID, job.Title) + _ = s.db.CreateJobAnalysis(job.ID, 0, false, "FILTERED_OUT") + _ = s.db.UpdateRawJobStatus(job.ID, "PROCESSED") + continue + } + + log.Printf("[Pipeline] Job #%d (%s) PASSED filter with score %.2f", job.ID, job.Title, score) + _ = s.db.CreateJobAnalysis(job.ID, score, true, "PENDING") + _ = s.db.UpdateRawJobStatus(job.ID, "PROCESSED") + qualifiedCount++ + } + + // 3. Trigger LLM Analysis on qualified PENDING analyses + pendingAnalyses, err := s.db.GetPendingQualifiedAnalyses() + if err != nil { + log.Printf("[Pipeline] Error fetching pending qualified analyses: %v", err) + return newJobsCount, qualifiedCount, nil + } + + for _, item := range pendingAnalyses { + log.Printf("[Pipeline] Triggering LLM analysis for Job #%d (%s @ %s)...", item.ID, item.Title, item.Company) + output, err := s.llmAnalyzer.AnalyzeJD(ctx, item.Title, item.RawContent) + if err != nil { + log.Printf("[Pipeline] LLM analysis failed for job #%d: %v", item.ID, err) + continue + } + + matchedJSON, _ := json.Marshal(output.MatchedHighlights) + missingJSON, _ := json.Marshal(output.MissingHighlights) + suggestionsJSON, _ := json.Marshal(output.TailoringSuggestions) + + err = s.db.UpdateJobAnalysisLLM(item.ID, string(matchedJSON), string(missingJSON), string(suggestionsJSON)) + if err != nil { + log.Printf("[Pipeline] Failed to save LLM analysis for job #%d: %v", item.ID, err) + } else { + log.Printf("[Pipeline] Successfully cached LLM analysis for Job #%d", item.ID) + } + } + + return newJobsCount, qualifiedCount, nil +} + +// AnalyzeJobByID manually triggers LLM analysis on a single job +func (s *PipelineService) AnalyzeJobByID(ctx context.Context, jobID int64) (*db.JobWithAnalysis, error) { + jobWithAnalysis, err := s.db.GetJobByID(jobID) + if err != nil { + return nil, err + } + + output, err := s.llmAnalyzer.AnalyzeJD(ctx, jobWithAnalysis.Title, jobWithAnalysis.RawContent) + if err != nil { + return nil, fmt.Errorf("LLM analysis failed: %w", err) + } + + matchedJSON, _ := json.Marshal(output.MatchedHighlights) + missingJSON, _ := json.Marshal(output.MissingHighlights) + suggestionsJSON, _ := json.Marshal(output.TailoringSuggestions) + + if jobWithAnalysis.Analysis == nil { + _, score := s.filterPipe.Execute(ctx, jobWithAnalysis.Title, jobWithAnalysis.RawContent) + _ = s.db.CreateJobAnalysis(jobID, score, true, "PENDING") + } + + err = s.db.UpdateJobAnalysisLLM(jobID, string(matchedJSON), string(missingJSON), string(suggestionsJSON)) + if err != nil { + return nil, err + } + + return s.db.GetJobByID(jobID) +} diff --git a/internal/ui/handlers.go b/internal/ui/handlers.go new file mode 100644 index 0000000..90b87bc --- /dev/null +++ b/internal/ui/handlers.go @@ -0,0 +1,280 @@ +package ui + +import ( + "context" + "log" + "net/http" + "strconv" + "strings" + + "github.com/hireme/hireme/internal/config" + "github.com/hireme/hireme/internal/db" + "github.com/hireme/hireme/internal/service" +) + +type Server struct { + cfg *config.Config + db *db.DB + pipeline *service.PipelineService +} + +func NewServer(cfg *config.Config, database *db.DB, pipeline *service.PipelineService) *Server { + return &Server{ + cfg: cfg, + db: database, + pipeline: pipeline, + } +} + +func (s *Server) RegisterRoutes(mux *http.ServeMux) { + mux.HandleFunc("GET /", s.handleIndex) + mux.HandleFunc("GET /targets", s.handleGetTargets) + mux.HandleFunc("POST /targets", s.handleCreateTarget) + mux.HandleFunc("POST /targets/{id}/toggle", s.handleToggleTarget) + mux.HandleFunc("GET /jobs", s.handleGetJobs) + mux.HandleFunc("GET /jobs/{id}", s.handleGetJobDetail) + mux.HandleFunc("POST /jobs/{id}/analyze", s.handleAnalyzeJob) + mux.HandleFunc("POST /applications", s.handleCreateApplication) + mux.HandleFunc("PATCH /applications/{id}/status", s.handleUpdateAppStatus) + mux.HandleFunc("POST /pipeline/run", s.handleRunPipeline) +} + +func (s *Server) handleIndex(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" { + http.NotFound(w, r) + return + } + + targets, err := s.db.GetAllTargets() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + jobs, err := s.db.GetJobsWithAnalysis() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + apps, err := s.db.GetApplications() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + qualifiedCount := 0 + for _, j := range jobs { + if j.Analysis != nil && j.Analysis.IsQualified { + qualifiedCount++ + } + } + + data := PageData{ + Targets: targets, + Jobs: jobs, + Applications: apps, + ActiveTab: "triage", + LLMProvider: s.cfg.LLMProvider, + QualifiedJobsCount: qualifiedCount, + TotalIngestedCount: len(jobs), + ActiveApplications: len(apps), + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + if err := RenderPage(w, data); err != nil { + log.Printf("[UI] RenderPage error: %v", err) + } +} + +func (s *Server) handleGetTargets(w http.ResponseWriter, r *http.Request) { + targets, err := s.db.GetAllTargets() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _ = RenderTargetList(w, targets) +} + +func (s *Server) handleCreateTarget(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + name := r.FormValue("name") + targetURL := r.FormValue("target_url") + platform := r.FormValue("platform") + + if name == "" || targetURL == "" { + http.Error(w, "name and target_url are required", http.StatusBadRequest) + return + } + + _, err := s.db.CreateTarget(name, targetURL, platform) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + s.handleGetTargets(w, r) +} + +func (s *Server) handleToggleTarget(w http.ResponseWriter, r *http.Request) { + idStr := r.PathValue("id") + id, err := strconv.ParseInt(idStr, 10, 64) + if err != nil { + http.Error(w, "invalid target id", http.StatusBadRequest) + return + } + + if err := s.db.ToggleTargetStatus(id); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + s.handleGetTargets(w, r) +} + +func (s *Server) handleGetJobs(w http.ResponseWriter, r *http.Request) { + jobs, err := s.db.GetJobsWithAnalysis() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _ = RenderJobList(w, jobs) +} + +func (s *Server) handleGetJobDetail(w http.ResponseWriter, r *http.Request) { + idStr := r.PathValue("id") + id, err := strconv.ParseInt(idStr, 10, 64) + if err != nil { + http.Error(w, "invalid job id", http.StatusBadRequest) + return + } + + job, err := s.db.GetJobByID(id) + if err != nil { + http.Error(w, "job not found", http.StatusNotFound) + return + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _ = RenderJobDetail(w, job) +} + +func (s *Server) handleAnalyzeJob(w http.ResponseWriter, r *http.Request) { + idStr := r.PathValue("id") + id, err := strconv.ParseInt(idStr, 10, 64) + if err != nil { + http.Error(w, "invalid job id", http.StatusBadRequest) + return + } + + job, err := s.pipeline.AnalyzeJobByID(r.Context(), id) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _ = RenderJobDetail(w, job) +} + +func (s *Server) handleCreateApplication(w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + rawJobIDStr := r.FormValue("raw_job_id") + status := r.FormValue("status") + if status == "" { + status = "SAVED" + } + + rawJobID, err := strconv.ParseInt(rawJobIDStr, 10, 64) + if err != nil { + http.Error(w, "invalid raw_job_id", http.StatusBadRequest) + return + } + + _, err = s.db.CreateOrUpdateApplication(rawJobID, status, "") + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + if strings.Contains(r.Header.Get("HX-Target"), "kanban") { + apps, _ := s.db.GetApplications() + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _ = RenderKanbanBoard(w, apps) + return + } + + s.handleGetJobs(w, r) +} + +func (s *Server) handleUpdateAppStatus(w http.ResponseWriter, r *http.Request) { + idStr := r.PathValue("id") + appID, err := strconv.ParseInt(idStr, 10, 64) + if err != nil { + http.Error(w, "invalid app id", http.StatusBadRequest) + return + } + + if err := r.ParseForm(); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + newStatus := r.FormValue("status") + if newStatus == "" { + http.Error(w, "status required", http.StatusBadRequest) + return + } + + // Fetch rawJobID for appID + apps, err := s.db.GetApplications() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + var targetRawJobID int64 = -1 + for _, a := range apps { + if a.App != nil && a.App.ID == appID { + targetRawJobID = a.ID + break + } + } + + if targetRawJobID == -1 { + http.Error(w, "application not found", http.StatusNotFound) + return + } + + _, err = s.db.CreateOrUpdateApplication(targetRawJobID, newStatus, "") + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + updatedApps, _ := s.db.GetApplications() + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _ = RenderKanbanBoard(w, updatedApps) +} + +func (s *Server) handleRunPipeline(w http.ResponseWriter, r *http.Request) { + ctx := context.Background() + _, _, err := s.pipeline.RunPipeline(ctx) + if err != nil { + log.Printf("[Server] Pipeline run error: %v", err) + } + + s.handleGetJobs(w, r) +} diff --git a/internal/ui/templates.go b/internal/ui/templates.go new file mode 100644 index 0000000..3e836bb --- /dev/null +++ b/internal/ui/templates.go @@ -0,0 +1,688 @@ +package ui + +import ( + "encoding/json" + "html/template" + "io" + "strings" + "time" + + "github.com/hireme/hireme/internal/db" +) + +var funcMap = template.FuncMap{ + "formatTime": func(t *time.Time) string { + if t == nil { + return "Never" + } + return t.Format("Jan 02, 2006 15:04") + }, + "formatDate": func(t time.Time) string { + return t.Format("Jan 02, 2006") + }, + "parseJSON": func(jsonStr string) []string { + if jsonStr == "" { + return nil + } + var list []string + _ = json.Unmarshal([]byte(jsonStr), &list) + return list + }, + "scoreBadgeClass": func(score float64) string { + if score >= 50 { + return "bg-emerald-500/20 text-emerald-400 border-emerald-500/30" + } else if score >= 20 { + return "bg-amber-500/20 text-amber-400 border-amber-500/30" + } + return "bg-slate-700/50 text-slate-400 border-slate-600/30" + }, + "platformIcon": func(platform string) string { + p := strings.ToLower(platform) + if strings.Contains(p, "linkedin") { + return "💼 LinkedIn" + } else if strings.Contains(p, "google") { + return "🔍 Google Jobs" + } else if strings.Contains(p, "greenhouse") { + return "🌿 Greenhouse" + } else if strings.Contains(p, "lever") { + return "⚙️ Lever" + } + return "🌐 Web Target" + }, +} + +type PageData struct { + Targets []db.SearchTarget + Jobs []db.JobWithAnalysis + Applications []db.JobWithAnalysis + ActiveTab string + LLMProvider string + QualifiedJobsCount int + TotalIngestedCount int + ActiveApplications int +} + +var ( + rootTemplate *template.Template +) + +func init() { + var err error + rootTemplate = template.New("root").Funcs(funcMap) + rootTemplate, err = rootTemplate.Parse(mainLayoutHTML) + if err != nil { + panic(err) + } + rootTemplate, err = rootTemplate.Parse(targetListHTML) + if err != nil { + panic(err) + } + rootTemplate, err = rootTemplate.Parse(jobListHTML) + if err != nil { + panic(err) + } + rootTemplate, err = rootTemplate.Parse(jobDetailHTML) + if err != nil { + panic(err) + } + rootTemplate, err = rootTemplate.Parse(kanbanBoardHTML) + if err != nil { + panic(err) + } + rootTemplate, err = rootTemplate.Parse(kanbanCardHTML) + if err != nil { + panic(err) + } +} + +func RenderPage(w io.Writer, data PageData) error { + return rootTemplate.ExecuteTemplate(w, "page", data) +} + +func RenderTargetList(w io.Writer, targets []db.SearchTarget) error { + return rootTemplate.ExecuteTemplate(w, "targets", targets) +} + +func RenderJobList(w io.Writer, jobs []db.JobWithAnalysis) error { + return rootTemplate.ExecuteTemplate(w, "jobs", jobs) +} + +func RenderJobDetail(w io.Writer, job *db.JobWithAnalysis) error { + return rootTemplate.ExecuteTemplate(w, "jobDetail", job) +} + +func RenderKanbanBoard(w io.Writer, apps []db.JobWithAnalysis) error { + return rootTemplate.ExecuteTemplate(w, "kanban", apps) +} + +const mainLayoutHTML = `{{ define "page" }} + + + + + Hireme | Job Command Center + + + + + + + + + + +
+
+
+
+ H +
+
+

Hireme

+

Job Application Command Center

+
+
+ + +
+ + + +
+
+
+ + +
+ + +
+
+
+

Total Search Targets

+

{{ len .Targets }}

+ Active Crawler Pipelines +
+ +
+
+

Qualified Jobs Matched

+

{{ .QualifiedJobsCount }}

+ Passed Keyword & Score Filter +
+ +
+
+

Total Ingested Postings

+

{{ .TotalIngestedCount }}

+ Deduplicated Raw Crawls +
+ +
+
+

Active Applications

+

{{ .ActiveApplications }}

+ Tracked in Kanban Pipeline +
+
+ + +
+ +
+ + + +
+ + +
+
+
+

Qualified Job Opportunities

+

Ranked by rule score + cached LLM skill gap analysis

+
+ +
+
+ {{ template "jobs" .Jobs }} +
+
+ + + + + + +
+ +
+ + + + + + + +{{ end }}` + +const targetListHTML = `{{ define "targets" }} +
+ + + + + + + + + + + + + {{ range . }} + + + + + + + + + {{ else }} + + + + {{ end }} + +
PlatformNameTarget URLLast CrawledStatusAction
{{ platformIcon .Platform }}{{ .Name }} + {{ .TargetURL }} + {{ formatTime .LastCrawledAt }} + {{ if .IsActive }} + Active + {{ else }} + Paused + {{ end }} + + +
No search targets configured yet.
+
{{ end }}` + +const jobListHTML = `{{ define "jobs" }} +
+ + + + + + + + + + + + + + {{ range . }} + + + + + + + + + + + + + + + + + + + + + + + {{ else }} + + + + {{ end }} + +
Job Title & CompanyOriginal LinkAnalysis StatusScore & QualificationKeyword TagsCrawled DateAction
+ +
+ {{ .Company }} + • + 📍 {{ .Location }} +
+ 		+ + 🔗 + + + {{ if .Analysis }} + {{ if eq .Analysis.Status "PARSED" }} + + PARSED + + {{ else if eq .Analysis.Status "FILTERED_OUT" }} + + FILTERED_OUT + + {{ else }} + + {{ .Analysis.Status }} + + {{ end }} + {{ else }} + + UNANALYZED + + {{ end }} + + {{ if .Analysis }} + {{ if .Analysis.IsQualified }} + + ★ {{ printf "%.0f" .Analysis.Score }} Qualified + + {{ else }} + + Not Qualified + + {{ end }} + {{ else }} + - + {{ end }} + + {{ if .Analysis }} +
+ {{ range parseJSON .Analysis.MatchedHighlights }} + + {{ . }} + + {{ else }} + No tags + {{ end }} +
+ {{ else }} + No tags + {{ end }} + 		+ {{ formatDate .CrawledAt }} + + {{ if .App }} + + {{ .App.Status }} + + {{ else }} + + {{ end }} +
+ No job postings found yet. Click ⚡ Run Pipeline to ingest jobs. +
+
{{ end }}` + +const jobDetailHTML = `{{ define "jobDetail" }} +
+
+
+ {{ .Company }} +

{{ .Title }}

+

📍 {{ .Location }} • Ingested {{ formatDate .CrawledAt }}

+
+ +
+ + {{ if .Analysis }} +
+
+ Match Relevance Score + {{ printf "%.1f" .Analysis.Score }} / 100 +
+
+ LLM Status + {{ .Analysis.Status }} +
+
+ + +
+
+

✓ Matched Technical Highlights

+
+ {{ range parseJSON .Analysis.MatchedHighlights }} + + {{ . }} + + {{ end }} +
+
+ +
+

! Skill & Experience Gaps

+
+ {{ range parseJSON .Analysis.MissingHighlights }} + + {{ . }} + + {{ end }} +
+
+ +
+

💡 Resume Tailoring Action Points

+
    + {{ range parseJSON .Analysis.TailoringSuggestions }} +
  • + • {{ . }} +
    • + {{ end }} +
+
+
+ {{ end }} + + +
+

Job Description Payload

+
+ {{ .RawContent }} +
+
+ + +
+ + Apply External ↗ + + +
+ + +
+
+
{{ end }}` + +const kanbanBoardHTML = `{{ define "kanban" }} +
+ +
+
+

💾 Saved

+
+
+ {{ range . }} + {{ if eq .App.Status "SAVED" }} + {{ template "kanbanCard" . }} + {{ end }} + {{ end }} +
+
+ + +
+
+

🚀 Applied

+
+
+ {{ range . }} + {{ if eq .App.Status "APPLIED" }} + {{ template "kanbanCard" . }} + {{ end }} + {{ end }} +
+
+ + +
+
+

💬 Interview

+
+
+ {{ range . }} + {{ if eq .App.Status "INTERVIEW" }} + {{ template "kanbanCard" . }} + {{ end }} + {{ end }} +
+
+ + +
+
+

🎉 Offer / Closed

+
+
+ {{ range . }} + {{ if or (eq .App.Status "OFFER") (eq .App.Status "REJECTED") }} + {{ template "kanbanCard" . }} + {{ end }} + {{ end }} +
+
+
{{ end }}` + +const kanbanCardHTML = ` +{{ define "kanbanCard" }} +
+
+ {{ .Company }} +

{{ .Title }}

+ 📍 {{ .Location }} +
+ +
+ Status: {{ .App.Status }} + + +
+
+{{ end }}` diff --git a/pkg/crawler/google/google.go b/pkg/crawler/google/google.go new file mode 100644 index 0000000..cc77613 --- /dev/null +++ b/pkg/crawler/google/google.go @@ -0,0 +1,400 @@ +package google + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "regexp" + "strings" + "time" + + "github.com/hireme/hireme/internal/crawler" + "golang.org/x/net/html" +) + +const ( + DefaultTestTargetURL = "https://www.google.com/about/careers/applications/jobs/results/?employment_type=FULL_TIME&q=%22Software%20Engineer%22&location=Seattle%2C%20WA%2C%20USA&target_level=MID" +) + +type GoogleCrawler struct { + client *http.Client +} + +func NewGoogleCrawler() *GoogleCrawler { + return &GoogleCrawler{ + client: &http.Client{ + Timeout: 30 * time.Second, + }, + } +} + +func (g *GoogleCrawler) CanHandle(platform string) bool { + p := strings.ToLower(platform) + return p == "google_jobs" || p == "google" || strings.Contains(p, "google.com/about/careers") +} + +// DiscoverJobs finds all job links from search result pages (paginated). +func (g *GoogleCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { + seen := make(map[string]bool) + var discovered []crawler.DiscoveredJob + + for page := 1; page <= 5; page++ { + pageURL := targetURL + if strings.Contains(pageURL, "?") { + pageURL = fmt.Sprintf("%s&page=%d&page_size=100", targetURL, page) + } else { + pageURL = fmt.Sprintf("%s?page=%d&page_size=100", targetURL, page) + } + + htmlContent, err := g.fetchWithRetry(ctx, pageURL) + if err != nil { + if page == 1 { + htmlContent, err = g.fetchWithRetry(ctx, targetURL) + if err != nil { + return nil, err + } + } else { + break + } + } + + // Scan for job IDs in both tags and inline script data + reID := regexp.MustCompile(`jobs/results/(\d[\w\-]+)`) + matches := reID.FindAllStringSubmatch(htmlContent, -1) + for _, m := range matches { + if len(m) >= 2 { + jobID := m[1] + // Filter out page= params that look like job IDs + if strings.HasPrefix(jobID, "?") || len(jobID) < 5 { + continue + } + applyURL := fmt.Sprintf("https://www.google.com/about/careers/applications/jobs/results/%s", jobID) + if !seen[applyURL] { + seen[applyURL] = true + discovered = append(discovered, crawler.DiscoveredJob{ + CanonicalID: fmt.Sprintf("google:%s", jobID), + ApplyURL: applyURL, + }) + } + } + } + } + + return discovered, nil +} + +// FetchJobDetail fetches a single job detail page and extracts structured data +// from the AF_initDataCallback JSON embedded in the page's +// +// The data array is: data[0] = [jobID, title, signinURL, responsibilities, qualifications, +// +// tenantPath, null, COMPANY, language, LOCATIONS, aboutHTML, ...] +// +// Indices: +// +// [0] = Job ID (string) +// [1] = Title (string) +// [7] = Company (string: "Google", "YouTube", "Google DeepMind", "Waymo", etc.) +// [9] = Locations (array of arrays: [["Shanghai, China", [...], "Shanghai", null, "Shanghai", "CN"]]) +func extractFromAFData(htmlContent string) (title, company, location string) { + // Find the start of the ds:0 data payload using string search (more reliable than regex for large content) + marker := `AF_initDataCallback({key: 'ds:0'` + idx := strings.Index(htmlContent, marker) + if idx == -1 { + log.Printf("[GoogleCrawler] AF_initDataCallback ds:0 not found") + return "", "", "" + } + + // Find "data:" after the marker + dataMarker := "data:" + dataIdx := strings.Index(htmlContent[idx:], dataMarker) + if dataIdx == -1 { + log.Printf("[GoogleCrawler] 'data:' not found after ds:0 marker") + return "", "", "" + } + dataStart := idx + dataIdx + len(dataMarker) + + // Use bracket counting to find the matching ] for the outer [ + jsonStr := extractBalancedBracket(htmlContent, dataStart) + if jsonStr == "" { + log.Printf("[GoogleCrawler] Failed to extract balanced bracket for ds:0 data") + return "", "", "" + } + + // Parse the outer array: data = [[...job record...]] + var outerArray []json.RawMessage + if err := json.Unmarshal([]byte(jsonStr), &outerArray); err != nil { + log.Printf("[GoogleCrawler] JSON parse error for ds:0 outer: %v", err) + return "", "", "" + } + if len(outerArray) == 0 { + return "", "", "" + } + + // Parse inner job record array + var record []json.RawMessage + if err := json.Unmarshal(outerArray[0], &record); err != nil { + log.Printf("[GoogleCrawler] JSON parse error for job record: %v", err) + return "", "", "" + } + + // [1] = Title + if len(record) > 1 { + _ = json.Unmarshal(record[1], &title) + } + + // [7] = Company + if len(record) > 7 { + _ = json.Unmarshal(record[7], &company) + } + + // [9] = Locations array: [["Shanghai, China", [...], ...], ...] + if len(record) > 9 { + var locs [][]json.RawMessage + if err := json.Unmarshal(record[9], &locs); err == nil { + var parts []string + for _, loc := range locs { + if len(loc) > 0 { + var display string + if err := json.Unmarshal(loc[0], &display); err == nil && display != "" { + parts = append(parts, display) + } + } + } + location = strings.Join(parts, "; ") + } + } + + log.Printf("[GoogleCrawler] AF_initDataCallback parsed: title=%q company=%q location=%q", title, company, location) + return title, company, location +} + +// extractBalancedBracket finds the JSON array starting at position startIdx in s, +// using bracket counting to handle nested arrays/objects. +func extractBalancedBracket(s string, startIdx int) string { + // Skip whitespace to find the opening [ + i := startIdx + for i < len(s) && (s[i] == ' ' || s[i] == '\t' || s[i] == '\n' || s[i] == '\r') { + i++ + } + if i >= len(s) || s[i] != '[' { + return "" + } + + depth := 0 + inString := false + escaped := false + + for j := i; j < len(s); j++ { + ch := s[j] + + if escaped { + escaped = false + continue + } + + if ch == '\\' && inString { + escaped = true + continue + } + + if ch == '"' { + inString = !inString + continue + } + + if inString { + continue + } + + if ch == '[' || ch == '{' { + depth++ + } else if ch == ']' || ch == '}' { + depth-- + if depth == 0 { + return s[i : j+1] + } + } + } + + return "" +} + +// fetchWithRetry fetches a URL with retry logic for HTTP 429 rate limiting. +func (g *GoogleCrawler) fetchWithRetry(ctx context.Context, urlStr string) (string, error) { + var lastErr error + for attempt := 0; attempt < 3; attempt++ { + if attempt > 0 { + delay := time.Duration(attempt) * 2 * time.Second + log.Printf("[GoogleCrawler] Retry #%d for %s after %v", attempt, urlStr, delay) + time.Sleep(delay) + } + + body, err := g.fetchURL(ctx, urlStr) + if err != nil { + if strings.Contains(err.Error(), "429") || strings.Contains(err.Error(), "status 429") { + lastErr = err + continue + } + return "", err + } + return body, nil + } + return "", fmt.Errorf("max retries exceeded for %s: %w", urlStr, lastErr) +} + +func (g *GoogleCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) { + req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) + if err != nil { + return "", fmt.Errorf("failed to create HTTP request for %s: %w", urlStr, err) + } + + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.9") + + resp, err := g.client.Do(req) + if err != nil { + return "", fmt.Errorf("HTTP fetch failed for %s: %w", urlStr, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", fmt.Errorf("failed to read response body for %s: %w", urlStr, err) + } + + return string(body), nil +} + +func parseIDFromHref(href string) string { + re := regexp.MustCompile(`jobs/results/(\d[\w\-]+)`) + m := re.FindStringSubmatch(href) + if len(m) >= 2 { + return m[1] + } + return "" +} + +func parseTitleFromURLSlug(applyURL string) string { + re := regexp.MustCompile(`jobs/results/\d+\-([a-zA-Z0-9\-]+)`) + m := re.FindStringSubmatch(applyURL) + if len(m) >= 2 { + slug := m[1] + words := strings.Split(slug, "-") + for i, w := range words { + if len(w) > 0 { + words[i] = strings.ToUpper(w[:1]) + w[1:] + } + } + return strings.Join(words, " ") + } + return "" +} + +func findElement(n *html.Node, tag string) *html.Node { + if n.Type == html.ElementNode && n.Data == tag { + return n + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if found := findElement(c, tag); found != nil { + return found + } + } + return nil +} + +func cleanTitle(s string) string { + cleaned := strings.TrimSpace(s) + cleaned = strings.ReplaceAll(cleaned, `\u0026`, "&") + cleaned = strings.ReplaceAll(cleaned, `\u003d`, "=") + + prefixes := []string{ + "Learn more about ", + "Learn more ", + "Apply for ", + "View job: ", + } + for _, p := range prefixes { + if strings.HasPrefix(cleaned, p) { + cleaned = strings.TrimPrefix(cleaned, p) + break + } + } + return strings.TrimSpace(cleaned) +} diff --git a/pkg/crawler/google/google_test.go b/pkg/crawler/google/google_test.go new file mode 100644 index 0000000..f676a47 --- /dev/null +++ b/pkg/crawler/google/google_test.go @@ -0,0 +1,117 @@ +package google + +import ( + "context" + "strings" + "testing" + "time" +) + +func TestCanHandle(t *testing.T) { + crawler := NewGoogleCrawler() + + if !crawler.CanHandle("google_jobs") { + t.Errorf("expected CanHandle('google_jobs') = true") + } + if !crawler.CanHandle("google") { + t.Errorf("expected CanHandle('google') = true") + } + if !crawler.CanHandle("https://www.google.com/about/careers/applications/") { + t.Errorf("expected CanHandle for Google careers URL = true") + } + if crawler.CanHandle("linkedin") { + t.Errorf("expected CanHandle('linkedin') = false") + } +} + +func TestDefaultTestTargetURL(t *testing.T) { + if !strings.Contains(DefaultTestTargetURL, "employment_type=FULL_TIME") { + t.Errorf("DefaultTestTargetURL missing employment_type filter") + } + if !strings.Contains(DefaultTestTargetURL, "location=Seattle%2C%20WA%2C%20USA") { + t.Errorf("DefaultTestTargetURL missing Seattle location filter") + } +} + +// TestGoogleCrawler_RealNetworkCrawling tests DiscoverJobs and FetchJobDetail +func TestGoogleCrawler_RealNetworkCrawling(t *testing.T) { + crawler := NewGoogleCrawler() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + // 1. Step 1: Discover jobs + discovered, err := crawler.DiscoverJobs(ctx, DefaultTestTargetURL) + if err != nil { + t.Logf("Notice: Network fetch error (expected in offline sandbox): %v", err) + return + } + + if len(discovered) == 0 { + t.Errorf("⚠️ YELLOW FLAG: 0 jobs discovered from Google Careers target URL (%s).", DefaultTestTargetURL) + return + } + + t.Logf("Successfully discovered %d job links", len(discovered)) + + // 2. Step 3: Fetch detail for first discovered job link + first := discovered[0] + if first.CanonicalID == "" || !strings.HasPrefix(first.CanonicalID, "google:") { + t.Errorf("invalid CanonicalID: %q", first.CanonicalID) + } + + detail, err := crawler.FetchJobDetail(ctx, first.ApplyURL) + if err != nil { + t.Logf("Notice: Detail fetch error: %v", err) + return + } + + if strings.TrimSpace(detail.Title) == "" { + t.Errorf("detail job has empty Title") + } + if detail.Company == "" { + t.Errorf("detail job has empty Company") + } + if strings.TrimSpace(detail.RawContent) == "" { + t.Errorf("detail job has empty RawContent") + } +} + +func TestExtractFromAFData(t *testing.T) { + // Realistic AF_initDataCallback payload mimicking Google Careers HTML + html := `` + + title, company, location := extractFromAFData(html) + + if title != "Software Engineer Manager, Modem NPI" { + t.Errorf("title = %q, want 'Software Engineer Manager, Modem NPI'", title) + } + if company != "Google" { + t.Errorf("company = %q, want 'Google'", company) + } + if location != "Shanghai, China" { + t.Errorf("location = %q, want 'Shanghai, China'", location) + } + + // Test with multiple locations and different company + html2 := `` + + title2, company2, location2 := extractFromAFData(html2) + + if title2 != "Strategy Associate, YouTube" { + t.Errorf("title2 = %q, want 'Strategy Associate, YouTube'", title2) + } + if company2 != "YouTube" { + t.Errorf("company2 = %q, want 'YouTube'", company2) + } + if location2 != "San Bruno, CA, USA; New York, NY, USA" { + t.Errorf("location2 = %q, want 'San Bruno, CA, USA; New York, NY, USA'", location2) + } + + // Test with DeepMind company + html3 := `` + + _, company3, _ := extractFromAFData(html3) + if company3 != "Google DeepMind" { + t.Errorf("company3 = %q, want 'Google DeepMind'", company3) + } +}