haopengzhan/hireme
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: implement worker pool pattern, add persistent task queue, and decouple target loading logic

Browse Source
This commit is contained in:
haopengzhan
2026-08-12 23:28:31 -07:00
parent 51b9468737
commit 890469fed8
18 changed files with 694 additions and 442 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/db/db.go
+12 -1
View File
@@ -55,7 +55,6 @@ func (db *DB) migrate() error {
location TEXT,
apply_url TEXT NOT NULL,
raw_content TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'PENDING',
crawled_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
@@ -79,6 +78,18 @@ func (db *DB) migrate() error {
applied_at DATETIME,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS tasks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
task_type TEXT NOT NULL,
payload TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'PENDING',
error_message TEXT,
retry_count INTEGER DEFAULT 0,
next_retry_at DATETIME DEFAULT CURRENT_TIMESTAMP,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
);
`
_, err := db.Exec(schema)
internal/db/models.go
+12 -1
View File
@@ -22,10 +22,21 @@ type RawJob struct {
Location string `json:"location"`
ApplyURL string `json:"apply_url"`
RawContent string `json:"raw_content"`
Status string `json:"status"` // 'PENDING', 'PROCESSED', 'FAILED', 'DUPLICATE'
CrawledAt time.Time `json:"crawled_at"`
}
type Task struct {
ID int64 `json:"id"`
TaskType string `json:"task_type"` // 'DISCOVER', 'FETCH', 'PROCESS', 'ANALYZE'
Payload string `json:"payload"` // JSON string
Status string `json:"status"` // 'PENDING', 'RUNNING', 'COMPLETED', 'FAILED'
ErrorMessage string `json:"error_message"`
RetryCount int `json:"retry_count"`
NextRetryAt time.Time `json:"next_retry_at"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
type JobAnalysis struct {
ID int64 `json:"id"`
RawJobID int64 `json:"raw_job_id"`
internal/db/queries.go
+37 -45
View File
@@ -31,7 +31,7 @@ func (db *DB) GetActiveTargets() ([]SearchTarget, error) {
if err != nil {
return nil, err
}
defer rows.Close()
defer func() { _ = rows.Close() }()
var targets []SearchTarget
for rows.Next() {
@@ -54,7 +54,7 @@ func (db *DB) GetAllTargets() ([]SearchTarget, error) {
if err != nil {
return nil, err
}
defer rows.Close()
defer func() { _ = rows.Close() }()
var targets []SearchTarget
for rows.Next() {
@@ -104,15 +104,15 @@ func (db *DB) InsertRawJob(canonicalID string, targetID *int64, title, company,
}
query := `
INSERT INTO raw_jobs (canonical_id, target_id, title, company, location, apply_url, raw_content, status)
VALUES (?, ?, ?, ?, ?, ?, ?, 'PENDING')
INSERT INTO raw_jobs (canonical_id, target_id, title, company, location, apply_url, raw_content)
VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(canonical_id) DO NOTHING
RETURNING id, canonical_id, target_id, title, company, location, apply_url, raw_content, status, crawled_at
RETURNING id, canonical_id, target_id, title, company, location, apply_url, raw_content, crawled_at
`
var job RawJob
var tID sql.NullInt64
err = db.QueryRow(query, canonicalID, targetID, title, company, location, applyURL, rawContent).Scan(
&job.ID, &job.CanonicalID, &tID, &job.Title, &job.Company, &job.Location, &job.ApplyURL, &job.RawContent, &job.Status, &job.CrawledAt,
&job.ID, &job.CanonicalID, &tID, &job.Title, &job.Company, &job.Location, &job.ApplyURL, &job.RawContent, &job.CrawledAt,
)
if err == sql.ErrNoRows {
return nil, false, nil // Duplicate job
@@ -139,7 +139,7 @@ func (db *DB) getAllRawJobsForDeduplication() ([]RawJob, error) {
if err != nil {
return nil, err
}
defer rows.Close()
defer func() { _ = rows.Close() }()
var jobs []RawJob
for rows.Next() {
@@ -207,34 +207,7 @@ func tokenizeToVector(text string) map[string]int {
return freqMap
}
func (db *DB) GetPendingRawJobs() ([]RawJob, error) {
query := `SELECT id, canonical_id, target_id, title, company, location, apply_url, raw_content, status, crawled_at FROM raw_jobs WHERE status = 'PENDING'`
rows, err := db.Query(query)
if err != nil {
return nil, err
}
defer rows.Close()
var jobs []RawJob
for rows.Next() {
var j RawJob
var tID sql.NullInt64
if err := rows.Scan(&j.ID, &j.CanonicalID, &tID, &j.Title, &j.Company, &j.Location, &j.ApplyURL, &j.RawContent, &j.Status, &j.CrawledAt); err != nil {
return nil, err
}
if tID.Valid {
j.TargetID = &tID.Int64
}
jobs = append(jobs, j)
}
return jobs, nil
}
func (db *DB) UpdateRawJobStatus(id int64, status string) error {
query := `UPDATE raw_jobs SET status = ? WHERE id = ?`
_, err := db.Exec(query, status, id)
return err
}
// Job Analyses Queries
func (db *DB) CreateJobAnalysis(rawJobID int64, score float64, isQualified bool, status string) error {
@@ -247,6 +220,25 @@ func (db *DB) CreateJobAnalysis(rawJobID int64, score float64, isQualified bool,
return err
}
func (db *DB) CreateJobAnalysisFull(rawJobID int64, score float64, isQualified bool, matched, missing, tailoring, status string) error {
query := `
INSERT INTO job_analyses (raw_job_id, score, is_qualified, matched_highlights, missing_highlights, tailoring_suggestions, status, analyzed_at)
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
`
_, err := db.Exec(query, rawJobID, score, isQualified, matched, missing, tailoring, status)
return err
}
func (db *DB) UpdateJobAnalysisFull(rawJobID int64, score float64, isQualified bool, matchedJSON, missingJSON, suggestionsJSON, status string) error {
query := `
UPDATE job_analyses
SET score = ?, is_qualified = ?, matched_highlights = ?, missing_highlights = ?, tailoring_suggestions = ?, status = ?, analyzed_at = CURRENT_TIMESTAMP
WHERE raw_job_id = ?
`
_, err := db.Exec(query, score, isQualified, matchedJSON, missingJSON, suggestionsJSON, status, rawJobID)
return err
}
func (db *DB) UpdateJobAnalysisLLM(rawJobID int64, matchedJSON, missingJSON, suggestionsJSON string) error {
query := `
UPDATE job_analyses
@@ -259,7 +251,7 @@ func (db *DB) UpdateJobAnalysisLLM(rawJobID int64, matchedJSON, missingJSON, sug
func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
query := `
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at,
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), a.status, a.analyzed_at
FROM raw_jobs r
JOIN job_analyses a ON r.id = a.raw_job_id
@@ -269,7 +261,7 @@ func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
if err != nil {
return nil, err
}
defer rows.Close()
defer func() { _ = rows.Close() }()
var results []JobWithAnalysis
for rows.Next() {
@@ -282,7 +274,7 @@ func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
var analyzedAt sql.NullTime
err := rows.Scan(
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt,
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
)
if err != nil {
@@ -312,7 +304,7 @@ func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
query := `
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at,
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
FROM raw_jobs r
@@ -324,7 +316,7 @@ func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
if err != nil {
return nil, err
}
defer rows.Close()
defer func() { _ = rows.Close() }()
var results []JobWithAnalysis
for rows.Next() {
@@ -340,7 +332,7 @@ func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
var updatedAt sql.NullTime
err := rows.Scan(
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt,
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
&appID, &appStatus, &notes, &appliedAt, &updatedAt,
)
@@ -386,7 +378,7 @@ func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
func (db *DB) GetJobByID(rawJobID int64) (*JobWithAnalysis, error) {
query := `
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at,
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
FROM raw_jobs r
@@ -407,7 +399,7 @@ func (db *DB) GetJobByID(rawJobID int64) (*JobWithAnalysis, error) {
var updatedAt sql.NullTime
err := row.Scan(
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt,
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
&appID, &appStatus, &notes, &appliedAt, &updatedAt,
)
@@ -483,7 +475,7 @@ func (db *DB) CreateOrUpdateApplication(rawJobID int64, status, notes string) (*
func (db *DB) GetApplications() ([]JobWithAnalysis, error) {
query := `
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at,
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
FROM job_applications app
@@ -495,7 +487,7 @@ func (db *DB) GetApplications() ([]JobWithAnalysis, error) {
if err != nil {
return nil, err
}
defer rows.Close()
defer func() { _ = rows.Close() }()
var results []JobWithAnalysis
for rows.Next() {
@@ -511,7 +503,7 @@ func (db *DB) GetApplications() ([]JobWithAnalysis, error) {
var updatedAt time.Time
err := rows.Scan(
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt,
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
&appID, &appStatus, &notes, &appliedAt, &updatedAt,
)
internal/db/queries_test.go
+3 -3
View File
@@ -13,7 +13,7 @@ func TestVectorSimilarityDeduplication(t *testing.T) {
if err != nil {
t.Fatalf("failed to init db: %v", err)
}
defer database.Close()
defer func() { _ = database.Close() }()
contentA := "Senior Software Engineer - Go Distributed Systems. Experience with microservices, gRPC, and SQLite database pipelines."
contentSimilar := "Senior Software Engineer - Go Distributed Systems. Experience with microservices, gRPC, and SQLite database pipelines."
@@ -41,9 +41,9 @@ func TestVectorSimilarityDeduplication(t *testing.T) {
}
// Verify only 2 raw jobs exist in DB
allJobs, err := database.GetPendingRawJobs()
allJobs, err := database.getAllRawJobsForDeduplication()
if err != nil {
t.Fatalf("GetPendingRawJobs failed: %v", err)
t.Fatalf("getAllRawJobsForDeduplication failed: %v", err)
}
if len(allJobs) != 2 {
t.Errorf("expected 2 raw jobs in DB after deduplication, got %d", len(allJobs))
internal/db/targets_loader.go
+48
View File
@@ -0,0 +1,48 @@
package db
import (
"fmt"
"os"
"gopkg.in/yaml.v3"
)
type targetConfig struct {
Targets []struct {
Name string `yaml:"name"`
TargetURL string `yaml:"target_url"`
Platform string `yaml:"platform"`
IsActive bool `yaml:"is_active"`
} `yaml:"targets"`
}
// LoadTargetsFromYAML reads a YAML file and inserts targets into the database if it's empty
func (db *DB) LoadTargetsFromYAML(filepath string) error {
data, err := os.ReadFile(filepath)
if err != nil {
return fmt.Errorf("could not read target config: %w", err)
}
var config targetConfig
if err := yaml.Unmarshal(data, &config); err != nil {
return fmt.Errorf("could not parse target config: %w", err)
}
// Check if we already have targets
targets, err := db.GetAllTargets()
if err != nil {
return err
}
if len(targets) > 0 {
// Do not overwrite existing targets
return nil
}
for _, t := range config.Targets {
_, err := db.CreateTarget(t.Name, t.TargetURL, t.Platform)
if err != nil {
return fmt.Errorf("failed to insert target %s: %w", t.Name, err)
}
}
return nil
}
internal/db/tasks.go
+159
View File
@@ -0,0 +1,159 @@
package db
import (
"database/sql"
"encoding/json"
"fmt"
"log"
)
// EnqueueTask adds a new task to the queue
func (db *DB) EnqueueTask(taskType string, payload interface{}) error {
payloadBytes, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("failed to marshal task payload: %w", err)
}
query := `
INSERT INTO tasks (task_type, payload, status)
VALUES (?, ?, 'PENDING')
`
_, err = db.Exec(query, taskType, string(payloadBytes))
return err
}
// ClaimNextTask atomically finds and claims the next pending task
func (db *DB) ClaimNextTask() (*Task, error) {
tx, err := db.Begin()
if err != nil {
return nil, err
}
defer func() { _ = tx.Rollback() }()
// Find the oldest PENDING task that is ready to be retried
query := `
SELECT id, task_type, payload, status, error_message, retry_count, next_retry_at, created_at, updated_at
FROM tasks
WHERE status = 'PENDING' AND next_retry_at <= CURRENT_TIMESTAMP
ORDER BY created_at ASC
LIMIT 1
`
var task Task
var errMsg sql.NullString
err = tx.QueryRow(query).Scan(
&task.ID, &task.TaskType, &task.Payload, &task.Status, &errMsg,
&task.RetryCount, &task.NextRetryAt, &task.CreatedAt, &task.UpdatedAt,
)
if err == sql.ErrNoRows {
return nil, nil // No tasks available
}
if err != nil {
return nil, err
}
if errMsg.Valid {
task.ErrorMessage = errMsg.String
}
// Claim the task
updateQuery := `UPDATE tasks SET status = 'RUNNING', updated_at = CURRENT_TIMESTAMP WHERE id = ?`
_, err = tx.Exec(updateQuery, task.ID)
if err != nil {
return nil, err
}
if err := tx.Commit(); err != nil {
return nil, err
}
return &task, nil
}
// CompleteTask marks a task as COMPLETED
func (db *DB) CompleteTask(id int64) error {
query := `UPDATE tasks SET status = 'COMPLETED', updated_at = CURRENT_TIMESTAMP WHERE id = ?`
_, err := db.Exec(query, id)
return err
}
// FailTask marks a task as FAILED or PENDING (for retry) using exponential backoff
func (db *DB) FailTask(id int64, errStr string) error {
// First get current retry count
var retryCount int
err := db.QueryRow(`SELECT retry_count FROM tasks WHERE id = ?`, id).Scan(&retryCount)
if err != nil {
log.Printf("[DB] FailTask: failed to get retry_count for task %d: %v", id, err)
return err
}
retryCount++
// Max retries = 5 (1s, 2s, 4s, 8s, 16s)
status := "PENDING"
if retryCount > 5 {
status = "FAILED"
}
// Exponential backoff
backoffSeconds := 1 << retryCount // 2^retryCount
query := `
UPDATE tasks
SET status = ?,
error_message = ?,
retry_count = ?,
next_retry_at = datetime('now', '+' || ? || ' seconds'),
updated_at = CURRENT_TIMESTAMP
WHERE id = ?
`
_, err = db.Exec(query, status, errStr, retryCount, fmt.Sprintf("%d", backoffSeconds), id)
return err
}
// HasPendingFetchTask checks if there is already a PENDING or RUNNING fetch task for a given canonical ID
func (db *DB) HasPendingFetchTask(canonicalID string) (bool, error) {
query := `
SELECT COUNT(*) FROM tasks
WHERE task_type = 'FETCH'
AND status IN ('PENDING', 'RUNNING')
AND payload LIKE ?
`
likePattern := fmt.Sprintf(`%%"canonical_id":"%s"%%`, canonicalID)
var count int
err := db.QueryRow(query, likePattern).Scan(&count)
if err != nil {
return false, err
}
return count > 0, nil
}
// ClearTaskQueue deletes all tasks or only pending/failed ones
func (db *DB) ClearTaskQueue() error {
query := `DELETE FROM tasks WHERE status IN ('PENDING', 'FAILED', 'COMPLETED')`
_, err := db.Exec(query)
return err
}
// GetQueueStatus returns counts of tasks by status
func (db *DB) GetQueueStatus() (map[string]int, error) {
query := `SELECT status, COUNT(*) FROM tasks GROUP BY status`
rows, err := db.Query(query)
if err != nil {
return nil, err
}
defer func() { _ = rows.Close() }()
counts := make(map[string]int)
for rows.Next() {
var status string
var count int
if err := rows.Scan(&status, &count); err != nil {
return nil, err
}
counts[status] = count
}
return counts, nil
}