feat: externalize filter configuration and add global structured logging and job reprocessing functionality

internal/config/config.go

@@ -11,6 +11,8 @@ type Config struct {
 	Port            string
 	DBPath          string
 	TargetConfig    string
+	FilterConfig    string
+	LogLevel        string
 	LLMProvider     string // "gemini", "deepseek", "mock"
 	GeminiAPIKey    string
 	DeepSeekAPIKey  string
@@ -23,6 +25,8 @@ func RegisterFlags(flags *pflag.FlagSet) *Config {
 	flags.StringVarP(&cfg.Port, "port", "p", getEnvOrDefault("PORT", "8080"), "HTTP server port")
 	flags.StringVarP(&cfg.DBPath, "db", "d", getEnvOrDefault("DB_PATH", "data/db/hireme.db"), "Path to database file")
 	flags.StringVarP(&cfg.TargetConfig, "target-config", "c", getEnvOrDefault("TARGET_CONFIG", "data/config/targets.yaml"), "Path to targets YAML file")
+	flags.StringVarP(&cfg.FilterConfig, "filter-config", "f", getEnvOrDefault("FILTER_CONFIG", "data/config/filters.yaml"), "Path to filters YAML file")
+	flags.StringVar(&cfg.LogLevel, "log-level", getEnvOrDefault("LOG_LEVEL", "info"), "Logging level (trace, debug, info, error)")
 	flags.StringVarP(&cfg.LLMProvider, "llm-provider", "l", getEnvOrDefault("LLM_PROVIDER", ""), "Active LLM Provider (gemini, deepseek, mock)")
 	flags.StringVar(&cfg.GeminiAPIKey, "gemini-api-key", os.Getenv("GEMINI_API_KEY"), "Gemini API Key")
 	flags.StringVar(&cfg.DeepSeekAPIKey, "deepseek-api-key", os.Getenv("DEEPSEEK_API_KEY"), "DeepSeek API Key")

internal/config/filters.go

@@ -0,0 +1,28 @@
+package config
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/hireme/hireme/internal/filter"
+	"gopkg.in/yaml.v3"
+)
+
+type FilterConfig struct {
+	HardKeywords   []string               `yaml:"hard_keywords"`
+	ScoringWeights []filter.KeywordWeight `yaml:"scoring_weights"`
+}
+
+func LoadFilterConfig(path string) (*FilterConfig, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read filter config file %s: %w", path, err)
+	}
+
+	var cfg FilterConfig
+	if err := yaml.Unmarshal(data, &cfg); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal filter config: %w", err)
+	}
+
+	return &cfg, nil
+}

internal/db/db.go

@@ -17,7 +17,7 @@ func InitDB(dbPath string) (*DB, error) {
 		dbPath = "data/db/hireme.db"
 	}
 
-	conn, err := sql.Open("sqlite3", dbPath+"?_foreign_keys=on")
+	conn, err := sql.Open("sqlite3", dbPath+"?_foreign_keys=on&_busy_timeout=5000&_journal_mode=WAL&_txlock=immediate")
 	if err != nil {
 		return nil, fmt.Errorf("failed to open sqlite database: %w", err)
 	}

internal/db/tasks.go

@@ -24,24 +24,21 @@ func (db *DB) EnqueueTask(taskType string, payload interface{}) error {
 
 // ClaimNextTask atomically finds and claims the next pending task
 func (db *DB) ClaimNextTask() (*Task, error) {
-	tx, err := db.Begin()
-	if err != nil {
-		return nil, err
-	}
-	defer func() { _ = tx.Rollback() }()
-
-	// Find the oldest PENDING task that is ready to be retried
 	query := `
-	SELECT id, task_type, payload, status, error_message, retry_count, next_retry_at, created_at, updated_at
-	FROM tasks
-	WHERE status = 'PENDING' AND next_retry_at <= CURRENT_TIMESTAMP
-	ORDER BY created_at ASC
-	LIMIT 1
+	UPDATE tasks 
+	SET status = 'RUNNING', updated_at = CURRENT_TIMESTAMP 
+	WHERE id = (
+		SELECT id FROM tasks 
+		WHERE status = 'PENDING' AND next_retry_at <= CURRENT_TIMESTAMP 
+		ORDER BY created_at ASC 
+		LIMIT 1
+	)
+	RETURNING id, task_type, payload, status, error_message, retry_count, next_retry_at, created_at, updated_at
 	`
 	
 	var task Task
 	var errMsg sql.NullString
-	err = tx.QueryRow(query).Scan(
+	err := db.QueryRow(query).Scan(
 		&task.ID, &task.TaskType, &task.Payload, &task.Status, &errMsg,
 		&task.RetryCount, &task.NextRetryAt, &task.CreatedAt, &task.UpdatedAt,
 	)
@@ -57,17 +54,6 @@ func (db *DB) ClaimNextTask() (*Task, error) {
 		task.ErrorMessage = errMsg.String
 	}
 
-	// Claim the task
-	updateQuery := `UPDATE tasks SET status = 'RUNNING', updated_at = CURRENT_TIMESTAMP WHERE id = ?`
-	_, err = tx.Exec(updateQuery, task.ID)
-	if err != nil {
-		return nil, err
-	}
-
-	if err := tx.Commit(); err != nil {
-		return nil, err
-	}
-
 	return &task, nil
 }
 
@@ -157,3 +143,49 @@ func (db *DB) GetQueueStatus() (map[string]int, error) {
 	}
 	return counts, nil
 }
+
+// ReprocessAllJobs wipes all job analysis results and re-queues PROCESS tasks for all existing raw jobs.
+func (db *DB) ReprocessAllJobs() error {
+	tx, err := db.Begin()
+	if err != nil {
+		return err
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	// 1. Delete all rows from job_analyses
+	if _, err := tx.Exec(`DELETE FROM job_analyses`); err != nil {
+		return fmt.Errorf("failed to clear job_analyses: %w", err)
+	}
+
+	// 2. Delete all existing PROCESS and ANALYZE tasks
+	if _, err := tx.Exec(`DELETE FROM tasks WHERE task_type IN ('PROCESS', 'ANALYZE')`); err != nil {
+		return fmt.Errorf("failed to clear old tasks: %w", err)
+	}
+
+	// 3. Queue up new PROCESS tasks for all raw jobs
+	rows, err := tx.Query(`SELECT id FROM raw_jobs`)
+	if err != nil {
+		return fmt.Errorf("failed to get raw jobs: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	var rawJobIDs []int64
+	for rows.Next() {
+		var rawJobID int64
+		if err := rows.Scan(&rawJobID); err != nil {
+			return err
+		}
+		rawJobIDs = append(rawJobIDs, rawJobID)
+	}
+	rows.Close() // Explicitly close to release any read locks
+
+	// 4. Enqueue PROCESS task for each job
+	for _, rawJobID := range rawJobIDs {
+		payloadJSON := fmt.Sprintf(`{"raw_job_id":%d}`, rawJobID)
+		if _, err := tx.Exec(`INSERT INTO tasks (task_type, payload, status) VALUES ('PROCESS', ?, 'PENDING')`, payloadJSON); err != nil {
+			return fmt.Errorf("failed to enqueue process task for job %d: %w", rawJobID, err)
+		}
+	}
+
+	return tx.Commit()
+}

internal/filter/filter.go

@@ -1,6 +1,10 @@
 package filter
 
-import "context"
+import (
+	"context"
+
+	"github.com/hireme/hireme/internal/db"
+)
 
 type FilterResult struct {
 	Keep   bool
@@ -10,7 +14,7 @@ type FilterResult struct {
 
 type JobFilter interface {
 	Name() string
-	Process(ctx context.Context, title, rawContent string) (FilterResult, error)
+	Process(ctx context.Context, job *db.RawJob) (FilterResult, error)
 }
 
 type FilterPipeline struct {
@@ -21,10 +25,10 @@ func NewPipeline(filters ...JobFilter) *FilterPipeline {
 	return &FilterPipeline{filters: filters}
 }
 
-func (p *FilterPipeline) Execute(ctx context.Context, title, rawContent string) (bool, float64, string) {
+func (p *FilterPipeline) Execute(ctx context.Context, job *db.RawJob) (bool, float64, string) {
 	var totalScore float64
 	for _, f := range p.filters {
-		res, err := f.Process(ctx, title, rawContent)
+		res, err := f.Process(ctx, job)
 		if err != nil {
 			return false, 0, err.Error()
 		}

internal/filter/filter_test.go

@@ -3,6 +3,8 @@ package filter
 import (
 	"context"
 	"testing"
+
+	"github.com/hireme/hireme/internal/db"
 )
 
 func TestHardKeywordFilter(t *testing.T) {
@@ -10,7 +12,7 @@ func TestHardKeywordFilter(t *testing.T) {
 	fk := NewHardKeywordFilter([]string{"DevOps", "No Sponsorship"})
 
 	// Should fail due to DevOps
-	res1, err := fk.Process(ctx, "Senior DevOps Engineer", "We build pipelines.")
+	res1, err := fk.Process(ctx, &db.RawJob{Title: "Senior DevOps Engineer", RawContent: "We build pipelines."})
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -19,7 +21,7 @@ func TestHardKeywordFilter(t *testing.T) {
 	}
 
 	// Should pass
-	res2, err := fk.Process(ctx, "Senior Backend Engineer", "We build scalable APIs in Go.")
+	res2, err := fk.Process(ctx, &db.RawJob{Title: "Senior Backend Engineer", RawContent: "We build scalable APIs in Go."})
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
@@ -39,7 +41,7 @@ func TestFilterPipeline(t *testing.T) {
 	pipeline := NewPipeline(hardFilter, scoringFilter)
 
 	// Test job 1: Unpaid job -> rejected by hard filter
-	keep1, score1, _ := pipeline.Execute(ctx, "Backend Intern", "Unpaid position working with Go.")
+	keep1, score1, _ := pipeline.Execute(ctx, &db.RawJob{Title: "Backend Intern", RawContent: "Unpaid position working with Go."})
 	if keep1 {
 		t.Errorf("expected job 1 to be filtered out, got keep=true")
 	}
@@ -48,7 +50,7 @@ func TestFilterPipeline(t *testing.T) {
 	}
 
 	// Test job 2: Qualified job -> passes both filters and accumulates score
-	keep2, score2, _ := pipeline.Execute(ctx, "Senior Go Backend Engineer", "Building high throughput services.")
+	keep2, score2, _ := pipeline.Execute(ctx, &db.RawJob{Title: "Senior Go Backend Engineer", RawContent: "Building high throughput services."})
 	if !keep2 {
 		t.Errorf("expected job 2 to pass filter pipeline, got keep=false")
 	}

internal/filter/hard_keyword.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 	"strings"
+
+	"github.com/hireme/hireme/internal/db"
 )
 
 type HardKeywordFilter struct {
@@ -26,8 +28,8 @@ func (f *HardKeywordFilter) Name() string {
 	return "HardKeywordFilter"
 }
 
-func (f *HardKeywordFilter) Process(ctx context.Context, title, rawContent string) (FilterResult, error) {
-	combined := strings.ToLower(title + " " + rawContent)
+func (f *HardKeywordFilter) Process(ctx context.Context, job *db.RawJob) (FilterResult, error) {
+	combined := strings.ToLower(job.Title + " " + job.RawContent)
 
 	for _, kw := range f.Blacklist {
 		term := strings.ToLower(strings.TrimSpace(kw))

internal/filter/scoring.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"fmt"
 	"strings"
+
+	"github.com/hireme/hireme/internal/db"
 )
 
 type KeywordWeight struct {
@@ -38,14 +40,14 @@ func (s *ScoringFilter) Name() string {
 	return "ScoringFilter"
 }
 
-func (s *ScoringFilter) Process(ctx context.Context, title, rawContent string) (FilterResult, error) {
-	combined := strings.ToLower(title + " " + rawContent)
+func (f *ScoringFilter) Process(ctx context.Context, job *db.RawJob) (FilterResult, error) {
+	combinedLower := strings.ToLower(job.Title + " " + job.RawContent)
 	var totalScore float64
 	var matched []string
 
-	for _, kw := range s.Weights {
+	for _, kw := range f.Weights {
 		term := strings.ToLower(strings.TrimSpace(kw.Keyword))
-		if term != "" && strings.Contains(combined, term) {
+		if term != "" && strings.Contains(combinedLower, term) {
 			totalScore += kw.Weight
 			matched = append(matched, kw.Keyword)
 		}

internal/logger/logger.go

@@ -0,0 +1,44 @@
+package logger
+
+import (
+	"os"
+	"strings"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+)
+
+var Log *zap.SugaredLogger
+
+func init() {
+	levelStr := os.Getenv("LOG_LEVEL")
+	if levelStr == "" {
+		levelStr = "info"
+	}
+	SetLevel(levelStr)
+}
+
+func SetLevel(levelStr string) {
+	var level zapcore.Level
+	switch strings.ToLower(levelStr) {
+	case "trace", "debug":
+		level = zap.DebugLevel
+	case "info":
+		level = zap.InfoLevel
+	case "warn":
+		level = zap.WarnLevel
+	case "error":
+		level = zap.ErrorLevel
+	default:
+		level = zap.InfoLevel
+	}
+
+	config := zap.NewDevelopmentConfig()
+	config.Level = zap.NewAtomicLevelAt(level)
+	config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
+	// Customize the timestamp format if needed
+	config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
+
+	logger, _ := config.Build()
+	Log = logger.Sugar()
+}

internal/ui/handlers.go

@@ -1,13 +1,13 @@
 package ui
 
 import (
-	"log"
 	"net/http"
 	"strconv"
 	"strings"
 
 	"github.com/hireme/hireme/internal/config"
 	"github.com/hireme/hireme/internal/db"
+	"github.com/hireme/hireme/internal/logger"
 	"github.com/hireme/hireme/internal/worker"
 )
 
@@ -37,6 +37,7 @@ func (s *Server) RegisterRoutes(mux *http.ServeMux) {
 	mux.HandleFunc("PATCH /applications/{id}/status", s.handleUpdateAppStatus)
 	mux.HandleFunc("POST /api/tasks/discover", s.handleDiscoverTasks)
 	mux.HandleFunc("POST /api/tasks/clean", s.handleCleanTasks)
+	mux.HandleFunc("POST /api/tasks/reprocess", s.handleReprocessJobs)
 	mux.HandleFunc("POST /api/tasks/toggle", s.handleToggleTasks)
 }
 
@@ -84,7 +85,7 @@ func (s *Server) handleIndex(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Type", "text/html; charset=utf-8")
 	if err := RenderPage(w, data); err != nil {
-		log.Printf("[UI] RenderPage error: %v", err)
+		logger.Log.Errorf("[UI] RenderPage error: %v", err)
 	}
 }
 
@@ -273,6 +274,7 @@ func (s *Server) handleUpdateAppStatus(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *Server) handleDiscoverTasks(w http.ResponseWriter, r *http.Request) {
+	logger.Log.Info("[API] Triggered /api/tasks/discover")
 	targets, err := s.db.GetAllTargets()
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -299,6 +301,7 @@ func (s *Server) handleDiscoverTasks(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *Server) handleCleanTasks(w http.ResponseWriter, r *http.Request) {
+	logger.Log.Info("[API] Triggered /api/tasks/clean")
 	err := s.db.ClearTaskQueue()
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -308,7 +311,19 @@ func (s *Server) handleCleanTasks(w http.ResponseWriter, r *http.Request) {
 	_, _ = w.Write([]byte(`{"status": "success"}`))
 }
 
+func (s *Server) handleReprocessJobs(w http.ResponseWriter, r *http.Request) {
+	logger.Log.Info("[API] Triggered /api/tasks/reprocess")
+	if err := s.db.ReprocessAllJobs(); err != nil {
+		logger.Log.Errorf("[API] Failed to reprocess jobs: %v", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	_, _ = w.Write([]byte(`{"status": "success", "message": "All jobs queued for reprocessing"}`))
+}
+
 func (s *Server) handleToggleTasks(w http.ResponseWriter, r *http.Request) {
+	logger.Log.Info("[API] Triggered /api/tasks/toggle")
 	isRunning := s.pool.Toggle()
 	status := "paused"
 	if isRunning {

internal/ui/templates.go

@@ -193,6 +193,12 @@ const mainLayoutHTML = `{{ define "page" }}<!DOCTYPE html>
                             class="relative inline-flex items-center justify-center px-3 py-1.5 text-xs font-semibold text-white transition-all bg-red-600 rounded-lg shadow-md hover:bg-red-500 active:scale-95 shadow-red-500/25">
                         <span>🗑️ Clean Queue</span>
                     </button>
+                    <button hx-post="/api/tasks/reprocess" 
+                            hx-swap="none"
+                            hx-on::after-request="location.reload()"
+                            class="relative inline-flex items-center justify-center px-3 py-1.5 text-xs font-semibold text-white transition-all bg-amber-600 rounded-lg shadow-md hover:bg-amber-500 active:scale-95 shadow-amber-500/25">
+                        <span>🔄 Reprocess All Jobs</span>
+                    </button>
                     <button hx-post="/api/tasks/toggle" 
                             hx-swap="none"
                             hx-on::after-request="location.reload()"

internal/worker/pool.go

@@ -4,16 +4,17 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"log"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/hireme/hireme/internal/analyzer"
+	"github.com/hireme/hireme/internal/config"
 	"github.com/hireme/hireme/internal/crawler"
 	"github.com/hireme/hireme/internal/db"
 	"github.com/hireme/hireme/internal/filter"
+	"github.com/hireme/hireme/internal/logger"
 	"github.com/hireme/hireme/pkg/crawler/google"
 )
 
@@ -28,22 +29,11 @@ type WorkerPool struct {
 	cancel      context.CancelFunc
 }
 
-func NewWorkerPool(database *db.DB, analyzer analyzer.LLMAnalyzer, numWorkers int) *WorkerPool {
-	// Initialize default filters
-	hardKeywords := []string{"cleared", "clearance", "polygraph", "ts/sci", "secret", "citizen", "citizenship", "phd", "ph.d"}
-	scoringWeights := []filter.KeywordWeight{
-		{Keyword: "golang", Weight: 3.0}, {Keyword: "go", Weight: 2.0},
-		{Keyword: "python", Weight: 2.0}, {Keyword: "kubernetes", Weight: 3.0},
-		{Keyword: "k8s", Weight: 3.0}, {Keyword: "docker", Weight: 2.0},
-		{Keyword: "aws", Weight: 1.0}, {Keyword: "gcp", Weight: 1.0},
-		{Keyword: "distributed systems", Weight: 4.0},
-		{Keyword: "backend", Weight: 3.0}, {Keyword: "ai", Weight: -5.0},
-		{Keyword: "machine learning", Weight: -5.0},
-	}
-	
+func NewWorkerPool(database *db.DB, analyzer analyzer.LLMAnalyzer, numWorkers int, filterCfg *config.FilterConfig) *WorkerPool {
+	// Initialize filters using the provided config
 	filterPipe := filter.NewPipeline(
-		filter.NewHardKeywordFilter(hardKeywords),
-		filter.NewScoringFilter(scoringWeights),
+		filter.NewHardKeywordFilter(filterCfg.HardKeywords),
+		filter.NewScoringFilter(filterCfg.ScoringWeights),
 	)
 
 	return &WorkerPool{
@@ -78,7 +68,7 @@ func (p *WorkerPool) Start() {
 		p.wg.Add(1)
 		go p.workerLoop(i)
 	}
-	log.Printf("[WorkerPool] Started %d workers", p.workerCount)
+	logger.Log.Infof("[WorkerPool] Started %d workers", p.workerCount)
 }
 
 func (p *WorkerPool) Stop() {
@@ -88,23 +78,23 @@ func (p *WorkerPool) Stop() {
 	p.cancel()
 	p.isRunning.Store(false)
 	p.wg.Wait()
-	log.Println("[WorkerPool] All workers stopped")
+	logger.Log.Infof("[WorkerPool] All workers stopped")
 }
 
 func (p *WorkerPool) workerLoop(workerID int) {
 	defer p.wg.Done()
-	log.Printf("[Worker %d] Starting loop", workerID)
+	logger.Log.Debugf("[Worker %d] Starting loop", workerID)
 
 	for {
 		select {
 		case <-p.ctx.Done():
-			log.Printf("[Worker %d] Context cancelled, exiting", workerID)
+			logger.Log.Debugf("[Worker %d] Context cancelled, exiting", workerID)
 			return
 		default:
 			task, err := p.db.ClaimNextTask()
 			if err != nil {
-				log.Printf("[Worker %d] Error claiming task: %v", workerID, err)
-				time.Sleep(2 * time.Second)
+				logger.Log.Errorf("[Worker %d] Error claiming task: %v", workerID, err)
+				time.Sleep(1 * time.Second)
 				continue
 			}
 			
@@ -114,7 +104,7 @@ func (p *WorkerPool) workerLoop(workerID int) {
 				continue
 			}
 
-			log.Printf("[Worker %d] Picked up task %d (%s)", workerID, task.ID, task.TaskType)
+			logger.Log.Debugf("[Worker %d] Picked up task %d (%s)", workerID, task.ID, task.TaskType)
 			
 			// Process task based on type
 			var processErr error
@@ -132,10 +122,10 @@ func (p *WorkerPool) workerLoop(workerID int) {
 			}
 
 			if processErr != nil {
-				log.Printf("[Worker %d] Task %d FAILED: %v", workerID, task.ID, processErr)
+				logger.Log.Errorf("[Worker %d] Task %d FAILED: %v", workerID, task.ID, processErr)
 				_ = p.db.FailTask(task.ID, processErr.Error())
 			} else {
-				log.Printf("[Worker %d] Task %d COMPLETED", workerID, task.ID)
+				logger.Log.Debugf("[Worker %d] Task %d COMPLETED", workerID, task.ID)
 				_ = p.db.CompleteTask(task.ID)
 			}
 		}
@@ -249,7 +239,7 @@ func (p *WorkerPool) handleProcess(ctx context.Context, task *db.Task) error {
 		return err
 	}
 
-	keep, score, reason := p.filterPipe.Execute(ctx, job.Title, job.RawContent)
+	keep, score, reason := p.filterPipe.Execute(ctx, &job.RawJob)
 	if !keep {
 		// Create a rejected JobAnalysis record. Store reason in missing_highlights JSON array
 		reasonJSON, _ := json.Marshal([]string{reason})