feat: externalize filter configuration and add global structured logging and job reprocessing functionality

Makefile

@@ -1,9 +1,10 @@
 .PHONY: all build run test clean
 
 BINARY_NAME=hireme
-DB_PATH=data/db/hireme.db
 CONFIG_PATH=data/config/targets.yaml
 PORT=8080
+ENV=dev
+DB_PATH=data/db/$(ENV)/hireme.db
 
 all: build
 
@@ -20,4 +21,4 @@ test:
 
 clean:
 	rm -f build/$(BINARY_NAME)
-	rm -rf data/db/*.db
+	rm -rf data/db/$(ENV)/*.db

cmd/hireme/main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"log"
 	"net/http"
 	"os"
 	"path/filepath"
@@ -10,6 +9,7 @@ import (
 	"github.com/hireme/hireme/internal/analyzer"
 	"github.com/hireme/hireme/internal/config"
 	"github.com/hireme/hireme/internal/db"
+	"github.com/hireme/hireme/internal/logger"
 	"github.com/hireme/hireme/internal/ui"
 	"github.com/hireme/hireme/internal/worker"
 	"github.com/spf13/cobra"
@@ -25,11 +25,13 @@ func newRootCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cfg.Normalize()
 
-			log.Println("==========================================================")
-			log.Println("🚀 Starting Hireme (GJCC) Job Application Command Center")
-			log.Println("==========================================================")
-			log.Printf("[Config] Port: %s | DB Path: %s | Target Config: %s | Active LLM: %s",
-				cfg.Port, cfg.DBPath, cfg.TargetConfig, cfg.LLMProvider)
+			logger.SetLevel(cfg.LogLevel)
+
+			logger.Log.Info("==========================================================")
+			logger.Log.Info("🚀 Starting Hireme (GJCC) Job Application Command Center")
+			logger.Log.Info("==========================================================")
+			logger.Log.Infof("[Config] Port: %s | DB Path: %s | Target Config: %s | Active LLM: %s | Log Level: %s",
+				cfg.Port, cfg.DBPath, cfg.TargetConfig, cfg.LLMProvider, cfg.LogLevel)
 
 			// Ensure DB directory exists
 			dbDir := filepath.Dir(cfg.DBPath)
@@ -46,7 +48,7 @@ func newRootCmd() *cobra.Command {
 
 			// Load targets from YAML config if database is empty
 			if err := database.LoadTargetsFromYAML(cfg.TargetConfig); err != nil {
-				log.Printf("[Main] Target YAML loading status: %v", err)
+				logger.Log.Errorf("[Main] Target YAML loading status: %v", err)
 			}
 
 			// Initialize LLM Analyzer
@@ -55,8 +57,14 @@ func newRootCmd() *cobra.Command {
 				return fmt.Errorf("fatal Analyzer setup error: %w", err)
 			}
 
-			// Initialize Worker Pool
-			pool := worker.NewWorkerPool(database, llmAnalyzer, 3)
+			// Load filter configuration
+			filterCfg, err := config.LoadFilterConfig(cfg.FilterConfig)
+			if err != nil {
+				logger.Log.Fatalf("Failed to load filter config: %v", err)
+			}
+
+			// 4. Initialize Worker Pool
+			pool := worker.NewWorkerPool(database, llmAnalyzer, 3, filterCfg)
 			pool.Start() // Auto start the worker pool
 
 			// Setup UI Server & Routes
@@ -65,7 +73,7 @@ func newRootCmd() *cobra.Command {
 			server.RegisterRoutes(mux)
 
 			addr := fmt.Sprintf(":%s", cfg.Port)
-			log.Printf("✨ Hireme Command Center is live at http://localhost:%s", cfg.Port)
+			logger.Log.Infof("✨ Hireme Command Center is live at http://localhost:%s", cfg.Port)
 			return http.ListenAndServe(addr, mux)
 		},
 	}

data/config/filters.yaml

@@ -0,0 +1,31 @@
+hard_keywords:
+  - "no sponsorship"
+  - "us citizen only"
+  - "unpaid"
+  - "volunteer"
+
+scoring_weights:
+  - keyword: golang
+    weight: 3.0
+  - keyword: go
+    weight: 2.0
+  - keyword: python
+    weight: 2.0
+  - keyword: kubernetes
+    weight: 3.0
+  - keyword: k8s
+    weight: 3.0
+  - keyword: docker
+    weight: 2.0
+  - keyword: aws
+    weight: 1.0
+  - keyword: gcp
+    weight: 1.0
+  - keyword: distributed systems
+    weight: 4.0
+  - keyword: backend
+    weight: 3.0
+  - keyword: ai
+    weight: -5.0
+  - keyword: machine learning
+    weight: -5.0

go.mod

@@ -10,4 +10,8 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 )
 
-require github.com/inconshreveable/mousetrap v1.1.0 // indirect
+require (
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	go.uber.org/multierr v1.10.0 // indirect
+	go.uber.org/zap v1.28.0 // indirect
+)

go.sum

@@ -8,6 +8,10 @@ github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
 github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
 github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
 github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
+go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
+go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
+go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
 golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
 golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

internal/config/config.go

@@ -11,6 +11,8 @@ type Config struct {
 	Port            string
 	DBPath          string
 	TargetConfig    string
+	FilterConfig    string
+	LogLevel        string
 	LLMProvider     string // "gemini", "deepseek", "mock"
 	GeminiAPIKey    string
 	DeepSeekAPIKey  string
@@ -23,6 +25,8 @@ func RegisterFlags(flags *pflag.FlagSet) *Config {
 	flags.StringVarP(&cfg.Port, "port", "p", getEnvOrDefault("PORT", "8080"), "HTTP server port")
 	flags.StringVarP(&cfg.DBPath, "db", "d", getEnvOrDefault("DB_PATH", "data/db/hireme.db"), "Path to database file")
 	flags.StringVarP(&cfg.TargetConfig, "target-config", "c", getEnvOrDefault("TARGET_CONFIG", "data/config/targets.yaml"), "Path to targets YAML file")
+	flags.StringVarP(&cfg.FilterConfig, "filter-config", "f", getEnvOrDefault("FILTER_CONFIG", "data/config/filters.yaml"), "Path to filters YAML file")
+	flags.StringVar(&cfg.LogLevel, "log-level", getEnvOrDefault("LOG_LEVEL", "info"), "Logging level (trace, debug, info, error)")
 	flags.StringVarP(&cfg.LLMProvider, "llm-provider", "l", getEnvOrDefault("LLM_PROVIDER", ""), "Active LLM Provider (gemini, deepseek, mock)")
 	flags.StringVar(&cfg.GeminiAPIKey, "gemini-api-key", os.Getenv("GEMINI_API_KEY"), "Gemini API Key")
 	flags.StringVar(&cfg.DeepSeekAPIKey, "deepseek-api-key", os.Getenv("DEEPSEEK_API_KEY"), "DeepSeek API Key")

internal/config/filters.go

@@ -0,0 +1,28 @@
+package config
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/hireme/hireme/internal/filter"
+	"gopkg.in/yaml.v3"
+)
+
+type FilterConfig struct {
+	HardKeywords   []string               `yaml:"hard_keywords"`
+	ScoringWeights []filter.KeywordWeight `yaml:"scoring_weights"`
+}
+
+func LoadFilterConfig(path string) (*FilterConfig, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read filter config file %s: %w", path, err)
+	}
+
+	var cfg FilterConfig
+	if err := yaml.Unmarshal(data, &cfg); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal filter config: %w", err)
+	}
+
+	return &cfg, nil
+}

internal/db/db.go

@@ -17,7 +17,7 @@ func InitDB(dbPath string) (*DB, error) {
 		dbPath = "data/db/hireme.db"
 	}
 
-	conn, err := sql.Open("sqlite3", dbPath+"?_foreign_keys=on")
+	conn, err := sql.Open("sqlite3", dbPath+"?_foreign_keys=on&_busy_timeout=5000&_journal_mode=WAL&_txlock=immediate")
 	if err != nil {
 		return nil, fmt.Errorf("failed to open sqlite database: %w", err)
 	}

internal/db/tasks.go

@@ -24,24 +24,21 @@ func (db *DB) EnqueueTask(taskType string, payload interface{}) error {
 
 // ClaimNextTask atomically finds and claims the next pending task
 func (db *DB) ClaimNextTask() (*Task, error) {
-	tx, err := db.Begin()
-	if err != nil {
-		return nil, err
-	}
-	defer func() { _ = tx.Rollback() }()
-
-	// Find the oldest PENDING task that is ready to be retried
 	query := `
-	SELECT id, task_type, payload, status, error_message, retry_count, next_retry_at, created_at, updated_at
-	FROM tasks
+	UPDATE tasks 
+	SET status = 'RUNNING', updated_at = CURRENT_TIMESTAMP 
+	WHERE id = (
+		SELECT id FROM tasks 
 		WHERE status = 'PENDING' AND next_retry_at <= CURRENT_TIMESTAMP 
 		ORDER BY created_at ASC 
 		LIMIT 1
+	)
+	RETURNING id, task_type, payload, status, error_message, retry_count, next_retry_at, created_at, updated_at
 	`
 	
 	var task Task
 	var errMsg sql.NullString
-	err = tx.QueryRow(query).Scan(
+	err := db.QueryRow(query).Scan(
 		&task.ID, &task.TaskType, &task.Payload, &task.Status, &errMsg,
 		&task.RetryCount, &task.NextRetryAt, &task.CreatedAt, &task.UpdatedAt,
 	)
@@ -57,17 +54,6 @@ func (db *DB) ClaimNextTask() (*Task, error) {
 		task.ErrorMessage = errMsg.String
 	}
 
-	// Claim the task
-	updateQuery := `UPDATE tasks SET status = 'RUNNING', updated_at = CURRENT_TIMESTAMP WHERE id = ?`
-	_, err = tx.Exec(updateQuery, task.ID)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := tx.Commit(); err != nil {
-		return nil, err
-	}
-
 	return &task, nil
 }
 
@@ -157,3 +143,49 @@ func (db *DB) GetQueueStatus() (map[string]int, error) {
 	}
 	return counts, nil
 }
+
+// ReprocessAllJobs wipes all job analysis results and re-queues PROCESS tasks for all existing raw jobs.
+func (db *DB) ReprocessAllJobs() error {
+	tx, err := db.Begin()
+	if err != nil {
+		return err
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	// 1. Delete all rows from job_analyses
+	if _, err := tx.Exec(`DELETE FROM job_analyses`); err != nil {
+		return fmt.Errorf("failed to clear job_analyses: %w", err)
+	}
+
+	// 2. Delete all existing PROCESS and ANALYZE tasks
+	if _, err := tx.Exec(`DELETE FROM tasks WHERE task_type IN ('PROCESS', 'ANALYZE')`); err != nil {
+		return fmt.Errorf("failed to clear old tasks: %w", err)
+	}
+
+	// 3. Queue up new PROCESS tasks for all raw jobs
+	rows, err := tx.Query(`SELECT id FROM raw_jobs`)
+	if err != nil {
+		return fmt.Errorf("failed to get raw jobs: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	var rawJobIDs []int64
+	for rows.Next() {
+		var rawJobID int64
+		if err := rows.Scan(&rawJobID); err != nil {
+			return err
+		}
+		rawJobIDs = append(rawJobIDs, rawJobID)
+	}
+	rows.Close() // Explicitly close to release any read locks
+
+	// 4. Enqueue PROCESS task for each job
+	for _, rawJobID := range rawJobIDs {
+		payloadJSON := fmt.Sprintf(`{"raw_job_id":%d}`, rawJobID)
+		if _, err := tx.Exec(`INSERT INTO tasks (task_type, payload, status) VALUES ('PROCESS', ?, 'PENDING')`, payloadJSON); err != nil {
+			return fmt.Errorf("failed to enqueue process task for job %d: %w", rawJobID, err)
+		}
+	}
+
+	return tx.Commit()
+}

internal/filter/filter.go

@@ -1,6 +1,10 @@
 package filter
 
-import "context"
+import (
+	"context"
+
+	"github.com/hireme/hireme/internal/db"
+)
 
 type FilterResult struct {
 	Keep   bool
@@ -10,7 +14,7 @@ type FilterResult struct {
 
 type JobFilter interface {
 	Name() string
-	Process(ctx context.Context, title, rawContent string) (FilterResult, error)
+	Process(ctx context.Context, job *db.RawJob) (FilterResult, error)
 }
 
 type FilterPipeline struct {
@@ -21,10 +25,10 @@ func NewPipeline(filters ...JobFilter) *FilterPipeline {
 	return &FilterPipeline{filters: filters}
 }
 
-func (p *FilterPipeline) Execute(ctx context.Context, title, rawContent string) (bool, float64, string) {
+func (p *FilterPipeline) Execute(ctx context.Context, job *db.RawJob) (bool, float64, string) {
 	var totalScore float64
 	for _, f := range p.filters {
-		res, err := f.Process(ctx, title, rawContent)
+		res, err := f.Process(ctx, job)
 		if err != nil {
 			return false, 0, err.Error()
 		}

internal/filter/filter_test.go

@@ -3,6 +3,8 @@ package filter
 import (
 	"context"
 	"testing"
+
+	"github.com/hireme/hireme/internal/db"
 )
 
 func TestHardKeywordFilter(t *testing.T) {
@@ -10,7 +12,7 @@ func TestHardKeywordFilter(t *testing.T) {
 	fk := NewHardKeywordFilter([]string{"DevOps", "No Sponsorship"})
 
 	// Should fail due to DevOps
-	res1, err := fk.Process(ctx, "Senior DevOps Engineer", "We build pipelines.")
+	res1, err := fk.Process(ctx, &db.RawJob{Title: "Senior DevOps Engineer", RawContent: "We build pipelines."})
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -19,7 +21,7 @@ func TestHardKeywordFilter(t *testing.T) {
 	}
 
 	// Should pass
-	res2, err := fk.Process(ctx, "Senior Backend Engineer", "We build scalable APIs in Go.")
+	res2, err := fk.Process(ctx, &db.RawJob{Title: "Senior Backend Engineer", RawContent: "We build scalable APIs in Go."})
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -39,7 +41,7 @@ func TestFilterPipeline(t *testing.T) {
 	pipeline := NewPipeline(hardFilter, scoringFilter)
 
 	// Test job 1: Unpaid job -> rejected by hard filter
-	keep1, score1, _ := pipeline.Execute(ctx, "Backend Intern", "Unpaid position working with Go.")
+	keep1, score1, _ := pipeline.Execute(ctx, &db.RawJob{Title: "Backend Intern", RawContent: "Unpaid position working with Go."})
 	if keep1 {
 		t.Errorf("expected job 1 to be filtered out, got keep=true")
 	}
@@ -48,7 +50,7 @@ func TestFilterPipeline(t *testing.T) {
 	}
 
 	// Test job 2: Qualified job -> passes both filters and accumulates score
-	keep2, score2, _ := pipeline.Execute(ctx, "Senior Go Backend Engineer", "Building high throughput services.")
+	keep2, score2, _ := pipeline.Execute(ctx, &db.RawJob{Title: "Senior Go Backend Engineer", RawContent: "Building high throughput services."})
 	if !keep2 {
 		t.Errorf("expected job 2 to pass filter pipeline, got keep=false")
 	}

internal/filter/hard_keyword.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 	"strings"
+
+	"github.com/hireme/hireme/internal/db"
 )
 
 type HardKeywordFilter struct {
@@ -26,8 +28,8 @@ func (f *HardKeywordFilter) Name() string {
 	return "HardKeywordFilter"
 }
 
-func (f *HardKeywordFilter) Process(ctx context.Context, title, rawContent string) (FilterResult, error) {
-	combined := strings.ToLower(title + " " + rawContent)
+func (f *HardKeywordFilter) Process(ctx context.Context, job *db.RawJob) (FilterResult, error) {
+	combined := strings.ToLower(job.Title + " " + job.RawContent)
 
 	for _, kw := range f.Blacklist {
 		term := strings.ToLower(strings.TrimSpace(kw))

internal/filter/scoring.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 	"strings"
+
+	"github.com/hireme/hireme/internal/db"
 )
 
 type KeywordWeight struct {
@@ -38,14 +40,14 @@ func (s *ScoringFilter) Name() string {
 	return "ScoringFilter"
 }
 
-func (s *ScoringFilter) Process(ctx context.Context, title, rawContent string) (FilterResult, error) {
-	combined := strings.ToLower(title + " " + rawContent)
+func (f *ScoringFilter) Process(ctx context.Context, job *db.RawJob) (FilterResult, error) {
+	combinedLower := strings.ToLower(job.Title + " " + job.RawContent)
 	var totalScore float64
 	var matched []string
 
-	for _, kw := range s.Weights {
+	for _, kw := range f.Weights {
 		term := strings.ToLower(strings.TrimSpace(kw.Keyword))
-		if term != "" && strings.Contains(combined, term) {
+		if term != "" && strings.Contains(combinedLower, term) {
 			totalScore += kw.Weight
 			matched = append(matched, kw.Keyword)
 		}

internal/logger/logger.go

@@ -0,0 +1,44 @@
+package logger
+
+import (
+	"os"
+	"strings"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+var Log *zap.SugaredLogger
+
+func init() {
+	levelStr := os.Getenv("LOG_LEVEL")
+	if levelStr == "" {
+		levelStr = "info"
+	}
+	SetLevel(levelStr)
+}
+
+func SetLevel(levelStr string) {
+	var level zapcore.Level
+	switch strings.ToLower(levelStr) {
+	case "trace", "debug":
+		level = zap.DebugLevel
+	case "info":
+		level = zap.InfoLevel
+	case "warn":
+		level = zap.WarnLevel
+	case "error":
+		level = zap.ErrorLevel
+	default:
+		level = zap.InfoLevel
+	}
+
+	config := zap.NewDevelopmentConfig()
+	config.Level = zap.NewAtomicLevelAt(level)
+	config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
+	// Customize the timestamp format if needed
+	config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+
+	logger, _ := config.Build()
+	Log = logger.Sugar()
+}

internal/ui/handlers.go

@@ -1,13 +1,13 @@
 package ui
 
 import (
-	"log"
 	"net/http"
 	"strconv"
 	"strings"
 
 	"github.com/hireme/hireme/internal/config"
 	"github.com/hireme/hireme/internal/db"
+	"github.com/hireme/hireme/internal/logger"
 	"github.com/hireme/hireme/internal/worker"
 )
 
@@ -37,6 +37,7 @@ func (s *Server) RegisterRoutes(mux *http.ServeMux) {
 	mux.HandleFunc("PATCH /applications/{id}/status", s.handleUpdateAppStatus)
 	mux.HandleFunc("POST /api/tasks/discover", s.handleDiscoverTasks)
 	mux.HandleFunc("POST /api/tasks/clean", s.handleCleanTasks)
+	mux.HandleFunc("POST /api/tasks/reprocess", s.handleReprocessJobs)
 	mux.HandleFunc("POST /api/tasks/toggle", s.handleToggleTasks)
 }
 
@@ -84,7 +85,7 @@ func (s *Server) handleIndex(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
 	if err := RenderPage(w, data); err != nil {
-		log.Printf("[UI] RenderPage error: %v", err)
+		logger.Log.Errorf("[UI] RenderPage error: %v", err)
 	}
 }
 
@@ -273,6 +274,7 @@ func (s *Server) handleUpdateAppStatus(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *Server) handleDiscoverTasks(w http.ResponseWriter, r *http.Request) {
+	logger.Log.Info("[API] Triggered /api/tasks/discover")
 	targets, err := s.db.GetAllTargets()
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -299,6 +301,7 @@ func (s *Server) handleDiscoverTasks(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *Server) handleCleanTasks(w http.ResponseWriter, r *http.Request) {
+	logger.Log.Info("[API] Triggered /api/tasks/clean")
 	err := s.db.ClearTaskQueue()
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -308,7 +311,19 @@ func (s *Server) handleCleanTasks(w http.ResponseWriter, r *http.Request) {
 	_, _ = w.Write([]byte(`{"status": "success"}`))
 }
 
+func (s *Server) handleReprocessJobs(w http.ResponseWriter, r *http.Request) {
+	logger.Log.Info("[API] Triggered /api/tasks/reprocess")
+	if err := s.db.ReprocessAllJobs(); err != nil {
+		logger.Log.Errorf("[API] Failed to reprocess jobs: %v", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_, _ = w.Write([]byte(`{"status": "success", "message": "All jobs queued for reprocessing"}`))
+}
+
 func (s *Server) handleToggleTasks(w http.ResponseWriter, r *http.Request) {
+	logger.Log.Info("[API] Triggered /api/tasks/toggle")
 	isRunning := s.pool.Toggle()
 	status := "paused"
 	if isRunning {

internal/ui/templates.go

@@ -193,6 +193,12 @@ const mainLayoutHTML = `{{ define "page" }}<!DOCTYPE html>
                             class="relative inline-flex items-center justify-center px-3 py-1.5 text-xs font-semibold text-white transition-all bg-red-600 rounded-lg shadow-md hover:bg-red-500 active:scale-95 shadow-red-500/25">
                         <span>🗑️ Clean Queue</span>
                     </button>
+                    <button hx-post="/api/tasks/reprocess" 
+                            hx-swap="none"
+                            hx-on::after-request="location.reload()"
+                            class="relative inline-flex items-center justify-center px-3 py-1.5 text-xs font-semibold text-white transition-all bg-amber-600 rounded-lg shadow-md hover:bg-amber-500 active:scale-95 shadow-amber-500/25">
+                        <span>🔄 Reprocess All Jobs</span>
+                    </button>
                     <button hx-post="/api/tasks/toggle" 
                             hx-swap="none"
                             hx-on::after-request="location.reload()"

internal/worker/pool.go

@@ -4,16 +4,17 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"log"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/hireme/hireme/internal/analyzer"
+	"github.com/hireme/hireme/internal/config"
 	"github.com/hireme/hireme/internal/crawler"
 	"github.com/hireme/hireme/internal/db"
 	"github.com/hireme/hireme/internal/filter"
+	"github.com/hireme/hireme/internal/logger"
 	"github.com/hireme/hireme/pkg/crawler/google"
 )
 
@@ -28,22 +29,11 @@ type WorkerPool struct {
 	cancel      context.CancelFunc
 }
 
-func NewWorkerPool(database *db.DB, analyzer analyzer.LLMAnalyzer, numWorkers int) *WorkerPool {
-	// Initialize default filters
-	hardKeywords := []string{"cleared", "clearance", "polygraph", "ts/sci", "secret", "citizen", "citizenship", "phd", "ph.d"}
-	scoringWeights := []filter.KeywordWeight{
-		{Keyword: "golang", Weight: 3.0}, {Keyword: "go", Weight: 2.0},
-		{Keyword: "python", Weight: 2.0}, {Keyword: "kubernetes", Weight: 3.0},
-		{Keyword: "k8s", Weight: 3.0}, {Keyword: "docker", Weight: 2.0},
-		{Keyword: "aws", Weight: 1.0}, {Keyword: "gcp", Weight: 1.0},
-		{Keyword: "distributed systems", Weight: 4.0},
-		{Keyword: "backend", Weight: 3.0}, {Keyword: "ai", Weight: -5.0},
-		{Keyword: "machine learning", Weight: -5.0},
-	}
-	
+func NewWorkerPool(database *db.DB, analyzer analyzer.LLMAnalyzer, numWorkers int, filterCfg *config.FilterConfig) *WorkerPool {
+	// Initialize filters using the provided config
 	filterPipe := filter.NewPipeline(
-		filter.NewHardKeywordFilter(hardKeywords),
-		filter.NewScoringFilter(scoringWeights),
+		filter.NewHardKeywordFilter(filterCfg.HardKeywords),
+		filter.NewScoringFilter(filterCfg.ScoringWeights),
 	)
 
 	return &WorkerPool{
@@ -78,7 +68,7 @@ func (p *WorkerPool) Start() {
 		p.wg.Add(1)
 		go p.workerLoop(i)
 	}
-	log.Printf("[WorkerPool] Started %d workers", p.workerCount)
+	logger.Log.Infof("[WorkerPool] Started %d workers", p.workerCount)
 }
 
 func (p *WorkerPool) Stop() {
@@ -88,23 +78,23 @@ func (p *WorkerPool) Stop() {
 	p.cancel()
 	p.isRunning.Store(false)
 	p.wg.Wait()
-	log.Println("[WorkerPool] All workers stopped")
+	logger.Log.Infof("[WorkerPool] All workers stopped")
 }
 
 func (p *WorkerPool) workerLoop(workerID int) {
 	defer p.wg.Done()
-	log.Printf("[Worker %d] Starting loop", workerID)
+	logger.Log.Debugf("[Worker %d] Starting loop", workerID)
 
 	for {
 		select {
 		case <-p.ctx.Done():
-			log.Printf("[Worker %d] Context cancelled, exiting", workerID)
+			logger.Log.Debugf("[Worker %d] Context cancelled, exiting", workerID)
 			return
 		default:
 			task, err := p.db.ClaimNextTask()
 			if err != nil {
-				log.Printf("[Worker %d] Error claiming task: %v", workerID, err)
-				time.Sleep(2 * time.Second)
+				logger.Log.Errorf("[Worker %d] Error claiming task: %v", workerID, err)
+				time.Sleep(1 * time.Second)
 				continue
 			}
 			
@@ -114,7 +104,7 @@ func (p *WorkerPool) workerLoop(workerID int) {
 				continue
 			}
 
-			log.Printf("[Worker %d] Picked up task %d (%s)", workerID, task.ID, task.TaskType)
+			logger.Log.Debugf("[Worker %d] Picked up task %d (%s)", workerID, task.ID, task.TaskType)
 			
 			// Process task based on type
 			var processErr error
@@ -132,10 +122,10 @@ func (p *WorkerPool) workerLoop(workerID int) {
 			}
 
 			if processErr != nil {
-				log.Printf("[Worker %d] Task %d FAILED: %v", workerID, task.ID, processErr)
+				logger.Log.Errorf("[Worker %d] Task %d FAILED: %v", workerID, task.ID, processErr)
 				_ = p.db.FailTask(task.ID, processErr.Error())
 			} else {
-				log.Printf("[Worker %d] Task %d COMPLETED", workerID, task.ID)
+				logger.Log.Debugf("[Worker %d] Task %d COMPLETED", workerID, task.ID)
 				_ = p.db.CompleteTask(task.ID)
 			}
 		}
@@ -249,7 +239,7 @@ func (p *WorkerPool) handleProcess(ctx context.Context, task *db.Task) error {
 		return err
 	}
 
-	keep, score, reason := p.filterPipe.Execute(ctx, job.Title, job.RawContent)
+	keep, score, reason := p.filterPipe.Execute(ctx, &job.RawJob)
 	if !keep {
 		// Create a rejected JobAnalysis record. Store reason in missing_highlights JSON array
 		reasonJSON, _ := json.Marshal([]string{reason})