haopengzhan/hireme
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: externalize filter configuration and add global structured logging and job reprocessing functionality

Browse Source
This commit is contained in:
haopengzhan
2026-08-14 15:27:50 -07:00
parent 890469fed8
commit 5510e3ac82
17 changed files with 257 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/worker/pool.go
+16 -26
View File
@@ -4,16 +4,17 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/hireme/hireme/internal/analyzer"
"github.com/hireme/hireme/internal/config"
"github.com/hireme/hireme/internal/crawler"
"github.com/hireme/hireme/internal/db"
"github.com/hireme/hireme/internal/filter"
"github.com/hireme/hireme/internal/logger"
"github.com/hireme/hireme/pkg/crawler/google"
)
@@ -28,22 +29,11 @@ type WorkerPool struct {
cancel context.CancelFunc
}
func NewWorkerPool(database *db.DB, analyzer analyzer.LLMAnalyzer, numWorkers int) *WorkerPool {
// Initialize default filters
hardKeywords := []string{"cleared", "clearance", "polygraph", "ts/sci", "secret", "citizen", "citizenship", "phd", "ph.d"}
scoringWeights := []filter.KeywordWeight{
{Keyword: "golang", Weight: 3.0}, {Keyword: "go", Weight: 2.0},
{Keyword: "python", Weight: 2.0}, {Keyword: "kubernetes", Weight: 3.0},
{Keyword: "k8s", Weight: 3.0}, {Keyword: "docker", Weight: 2.0},
{Keyword: "aws", Weight: 1.0}, {Keyword: "gcp", Weight: 1.0},
{Keyword: "distributed systems", Weight: 4.0},
{Keyword: "backend", Weight: 3.0}, {Keyword: "ai", Weight: -5.0},
{Keyword: "machine learning", Weight: -5.0},
}
func NewWorkerPool(database *db.DB, analyzer analyzer.LLMAnalyzer, numWorkers int, filterCfg *config.FilterConfig) *WorkerPool {
// Initialize filters using the provided config
filterPipe := filter.NewPipeline(
filter.NewHardKeywordFilter(hardKeywords),
filter.NewScoringFilter(scoringWeights),
filter.NewHardKeywordFilter(filterCfg.HardKeywords),
filter.NewScoringFilter(filterCfg.ScoringWeights),
)
return &WorkerPool{
@@ -78,7 +68,7 @@ func (p *WorkerPool) Start() {
p.wg.Add(1)
go p.workerLoop(i)
}
log.Printf("[WorkerPool] Started %d workers", p.workerCount)
logger.Log.Infof("[WorkerPool] Started %d workers", p.workerCount)
}
func (p *WorkerPool) Stop() {
@@ -88,23 +78,23 @@ func (p *WorkerPool) Stop() {
p.cancel()
p.isRunning.Store(false)
p.wg.Wait()
log.Println("[WorkerPool] All workers stopped")
logger.Log.Infof("[WorkerPool] All workers stopped")
}
func (p *WorkerPool) workerLoop(workerID int) {
defer p.wg.Done()
log.Printf("[Worker %d] Starting loop", workerID)
logger.Log.Debugf("[Worker %d] Starting loop", workerID)
for {
select {
case <-p.ctx.Done():
log.Printf("[Worker %d] Context cancelled, exiting", workerID)
logger.Log.Debugf("[Worker %d] Context cancelled, exiting", workerID)
return
default:
task, err := p.db.ClaimNextTask()
if err != nil {
log.Printf("[Worker %d] Error claiming task: %v", workerID, err)
time.Sleep(2 * time.Second)
logger.Log.Errorf("[Worker %d] Error claiming task: %v", workerID, err)
time.Sleep(1 * time.Second)
continue
}
@@ -114,7 +104,7 @@ func (p *WorkerPool) workerLoop(workerID int) {
continue
}
log.Printf("[Worker %d] Picked up task %d (%s)", workerID, task.ID, task.TaskType)
logger.Log.Debugf("[Worker %d] Picked up task %d (%s)", workerID, task.ID, task.TaskType)
// Process task based on type
var processErr error
@@ -132,10 +122,10 @@ func (p *WorkerPool) workerLoop(workerID int) {
}
if processErr != nil {
log.Printf("[Worker %d] Task %d FAILED: %v", workerID, task.ID, processErr)
logger.Log.Errorf("[Worker %d] Task %d FAILED: %v", workerID, task.ID, processErr)
_ = p.db.FailTask(task.ID, processErr.Error())
} else {
log.Printf("[Worker %d] Task %d COMPLETED", workerID, task.ID)
logger.Log.Debugf("[Worker %d] Task %d COMPLETED", workerID, task.ID)
_ = p.db.CompleteTask(task.ID)
}
}
@@ -249,7 +239,7 @@ func (p *WorkerPool) handleProcess(ctx context.Context, task *db.Task) error {
return err
}
keep, score, reason := p.filterPipe.Execute(ctx, job.Title, job.RawContent)
keep, score, reason := p.filterPipe.Execute(ctx, &job.RawJob)
if !keep {
// Create a rejected JobAnalysis record. Store reason in missing_highlights JSON array
reasonJSON, _ := json.Marshal([]string{reason})