feat: add job maintenance dashboard and automated stale job expiration system
This commit is contained in:
+34
-2
@@ -55,7 +55,8 @@ func (db *DB) migrate() error {
|
|||||||
location TEXT,
|
location TEXT,
|
||||||
apply_url TEXT NOT NULL,
|
apply_url TEXT NOT NULL,
|
||||||
raw_content TEXT NOT NULL,
|
raw_content TEXT NOT NULL,
|
||||||
crawled_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
crawled_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||||
|
last_seen_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS job_analyses (
|
CREATE TABLE IF NOT EXISTS job_analyses (
|
||||||
@@ -92,6 +93,37 @@ func (db *DB) migrate() error {
|
|||||||
);
|
);
|
||||||
`
|
`
|
||||||
|
|
||||||
_, err := db.Exec(schema)
|
if _, err := db.Exec(schema); err != nil {
|
||||||
return err
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Dynamic migration for existing databases: check if last_seen_at exists in raw_jobs
|
||||||
|
var hasLastSeen bool
|
||||||
|
rows, err := db.Query(`PRAGMA table_info(raw_jobs)`)
|
||||||
|
if err == nil {
|
||||||
|
for rows.Next() {
|
||||||
|
var cid int
|
||||||
|
var name, ctype string
|
||||||
|
var notnull, pk int
|
||||||
|
var dfltValue sql.NullString
|
||||||
|
if err := rows.Scan(&cid, &name, &ctype, ¬null, &dfltValue, &pk); err == nil {
|
||||||
|
if name == "last_seen_at" {
|
||||||
|
hasLastSeen = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = rows.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
if !hasLastSeen {
|
||||||
|
if _, err := db.Exec(`ALTER TABLE raw_jobs ADD COLUMN last_seen_at DATETIME`); err != nil {
|
||||||
|
logger.Log.Warnw("Failed to add last_seen_at column", "component", "DB", "error", err)
|
||||||
|
} else {
|
||||||
|
_, _ = db.Exec(`UPDATE raw_jobs SET last_seen_at = crawled_at WHERE last_seen_at IS NULL`)
|
||||||
|
_, _ = db.Exec(`UPDATE raw_jobs SET last_seen_at = CURRENT_TIMESTAMP WHERE last_seen_at IS NULL`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ type RawJob struct {
|
|||||||
ApplyURL string `json:"apply_url"`
|
ApplyURL string `json:"apply_url"`
|
||||||
RawContent string `json:"raw_content"`
|
RawContent string `json:"raw_content"`
|
||||||
CrawledAt time.Time `json:"crawled_at"`
|
CrawledAt time.Time `json:"crawled_at"`
|
||||||
|
LastSeenAt time.Time `json:"last_seen_at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type Task struct {
|
type Task struct {
|
||||||
|
|||||||
+131
-12
@@ -185,15 +185,15 @@ func (db *DB) InsertRawJob(canonicalID string, targetID *int64, title, company,
|
|||||||
}
|
}
|
||||||
|
|
||||||
query := `
|
query := `
|
||||||
INSERT INTO raw_jobs (canonical_id, target_id, title, company, location, apply_url, raw_content)
|
INSERT INTO raw_jobs (canonical_id, target_id, title, company, location, apply_url, raw_content, crawled_at, last_seen_at)
|
||||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||||
ON CONFLICT(canonical_id) DO NOTHING
|
ON CONFLICT(canonical_id) DO NOTHING
|
||||||
RETURNING id, canonical_id, target_id, title, company, location, apply_url, raw_content, crawled_at
|
RETURNING id, canonical_id, target_id, title, company, location, apply_url, raw_content, crawled_at, last_seen_at
|
||||||
`
|
`
|
||||||
var job RawJob
|
var job RawJob
|
||||||
var tID sql.NullInt64
|
var tID sql.NullInt64
|
||||||
err = db.QueryRow(query, canonicalID, targetID, title, company, location, applyURL, rawContent).Scan(
|
err = db.QueryRow(query, canonicalID, targetID, title, company, location, applyURL, rawContent).Scan(
|
||||||
&job.ID, &job.CanonicalID, &tID, &job.Title, &job.Company, &job.Location, &job.ApplyURL, &job.RawContent, &job.CrawledAt,
|
&job.ID, &job.CanonicalID, &tID, &job.Title, &job.Company, &job.Location, &job.ApplyURL, &job.RawContent, &job.CrawledAt, &job.LastSeenAt,
|
||||||
)
|
)
|
||||||
if err == sql.ErrNoRows {
|
if err == sql.ErrNoRows {
|
||||||
return nil, false, nil // Duplicate job
|
return nil, false, nil // Duplicate job
|
||||||
@@ -207,6 +207,125 @@ func (db *DB) InsertRawJob(canonicalID string, targetID *int64, title, company,
|
|||||||
return &job, true, nil
|
return &job, true, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (db *DB) TouchJobLastSeen(canonicalID string) error {
|
||||||
|
query := `UPDATE raw_jobs SET last_seen_at = CURRENT_TIMESTAMP WHERE canonical_id = ?`
|
||||||
|
_, err := db.Exec(query, canonicalID)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) TouchJobLastSeenByID(rawJobID int64) error {
|
||||||
|
query := `UPDATE raw_jobs SET last_seen_at = CURRENT_TIMESTAMP WHERE id = ?`
|
||||||
|
_, err := db.Exec(query, rawJobID)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) ExpireJob(rawJobID int64, reason string) error {
|
||||||
|
if reason == "" {
|
||||||
|
reason = "Marked expired"
|
||||||
|
}
|
||||||
|
query := `
|
||||||
|
INSERT INTO job_applications (raw_job_id, status, notes, updated_at)
|
||||||
|
VALUES (?, 'EXPIRED', ?, CURRENT_TIMESTAMP)
|
||||||
|
ON CONFLICT(raw_job_id) DO UPDATE SET
|
||||||
|
status='EXPIRED',
|
||||||
|
notes=COALESCE(job_applications.notes || ' | ', '') || excluded.notes,
|
||||||
|
updated_at=CURRENT_TIMESTAMP
|
||||||
|
`
|
||||||
|
_, err := db.Exec(query, rawJobID, reason)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) EnqueueExpireTasks(days int, preserveActivePipeline bool) (int, error) {
|
||||||
|
tx, err := db.Begin()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer func() { _ = tx.Rollback() }()
|
||||||
|
|
||||||
|
var query string
|
||||||
|
if preserveActivePipeline {
|
||||||
|
query = `
|
||||||
|
SELECT r.id FROM raw_jobs r
|
||||||
|
LEFT JOIN job_applications app ON r.id = app.raw_job_id
|
||||||
|
WHERE r.last_seen_at < datetime('now', '-' || ? || ' days')
|
||||||
|
AND (app.status IS NULL OR app.status NOT IN ('APPLIED', 'INTERVIEW', 'OFFER', 'EXPIRED', 'OPTED_OUT'))
|
||||||
|
`
|
||||||
|
} else {
|
||||||
|
query = `
|
||||||
|
SELECT r.id FROM raw_jobs r
|
||||||
|
LEFT JOIN job_applications app ON r.id = app.raw_job_id
|
||||||
|
WHERE r.last_seen_at < datetime('now', '-' || ? || ' days')
|
||||||
|
AND (app.status IS NULL OR app.status NOT IN ('EXPIRED', 'OPTED_OUT'))
|
||||||
|
`
|
||||||
|
}
|
||||||
|
|
||||||
|
rows, err := tx.Query(query, days)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var rawJobIDs []int64
|
||||||
|
for rows.Next() {
|
||||||
|
var id int64
|
||||||
|
if err := rows.Scan(&id); err != nil {
|
||||||
|
_ = rows.Close()
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
rawJobIDs = append(rawJobIDs, id)
|
||||||
|
}
|
||||||
|
_ = rows.Close()
|
||||||
|
|
||||||
|
for _, id := range rawJobIDs {
|
||||||
|
payloadJSON := fmt.Sprintf(`{"raw_job_id":%d,"reason":"Direct expire: not seen on target for %d+ days"}`, id, days)
|
||||||
|
if _, err := tx.Exec(`INSERT INTO tasks (task_type, payload, status) VALUES ('EXPIRE_JOB', ?, 'PENDING')`, payloadJSON); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return len(rawJobIDs), tx.Commit()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (db *DB) GetJobMaintenanceStats() (map[string]int, error) {
|
||||||
|
stats := make(map[string]int)
|
||||||
|
|
||||||
|
// Total raw jobs
|
||||||
|
var totalJobs int
|
||||||
|
_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs`).Scan(&totalJobs)
|
||||||
|
stats["total_jobs"] = totalJobs
|
||||||
|
|
||||||
|
// Active <= 7 days
|
||||||
|
var active7d int
|
||||||
|
_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs WHERE last_seen_at >= datetime('now', '-7 days')`).Scan(&active7d)
|
||||||
|
stats["active_7d"] = active7d
|
||||||
|
|
||||||
|
// Stale > 7 days
|
||||||
|
var stale7d int
|
||||||
|
_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs WHERE last_seen_at < datetime('now', '-7 days')`).Scan(&stale7d)
|
||||||
|
stats["stale_7d"] = stale7d
|
||||||
|
|
||||||
|
// Stale > 30 days
|
||||||
|
var stale30d int
|
||||||
|
_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs WHERE last_seen_at < datetime('now', '-30 days')`).Scan(&stale30d)
|
||||||
|
stats["stale_30d"] = stale30d
|
||||||
|
|
||||||
|
// Expired count
|
||||||
|
var expiredCount int
|
||||||
|
_ = db.QueryRow(`SELECT COUNT(*) FROM job_applications WHERE status = 'EXPIRED'`).Scan(&expiredCount)
|
||||||
|
stats["expired_count"] = expiredCount
|
||||||
|
|
||||||
|
// Opted out count
|
||||||
|
var optedOutCount int
|
||||||
|
_ = db.QueryRow(`SELECT COUNT(*) FROM job_applications WHERE status = 'OPTED_OUT'`).Scan(&optedOutCount)
|
||||||
|
stats["opted_out_count"] = optedOutCount
|
||||||
|
|
||||||
|
// In pipeline count
|
||||||
|
var inPipeline int
|
||||||
|
_ = db.QueryRow(`SELECT COUNT(*) FROM job_applications WHERE status IN ('SAVED', 'APPLIED', 'INTERVIEW', 'OFFER')`).Scan(&inPipeline)
|
||||||
|
stats["in_pipeline"] = inPipeline
|
||||||
|
|
||||||
|
return stats, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (db *DB) CanonicalIDExists(canonicalID string) (bool, error) {
|
func (db *DB) CanonicalIDExists(canonicalID string) (bool, error) {
|
||||||
query := `SELECT EXISTS(SELECT 1 FROM raw_jobs WHERE canonical_id = ?)`
|
query := `SELECT EXISTS(SELECT 1 FROM raw_jobs WHERE canonical_id = ?)`
|
||||||
var exists bool
|
var exists bool
|
||||||
@@ -330,7 +449,7 @@ func (db *DB) UpdateJobAnalysisLLM(rawJobID int64, matchedJSON, missingJSON, sug
|
|||||||
|
|
||||||
func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
|
func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
|
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
|
||||||
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), a.status, a.analyzed_at
|
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), a.status, a.analyzed_at
|
||||||
FROM raw_jobs r
|
FROM raw_jobs r
|
||||||
JOIN job_analyses a ON r.id = a.raw_job_id
|
JOIN job_analyses a ON r.id = a.raw_job_id
|
||||||
@@ -353,7 +472,7 @@ func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
|
|||||||
var analyzedAt sql.NullTime
|
var analyzedAt sql.NullTime
|
||||||
|
|
||||||
err := rows.Scan(
|
err := rows.Scan(
|
||||||
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
|
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
|
||||||
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -383,7 +502,7 @@ func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
|
|||||||
|
|
||||||
func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
|
func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
|
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
|
||||||
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
||||||
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
||||||
FROM raw_jobs r
|
FROM raw_jobs r
|
||||||
@@ -411,7 +530,7 @@ func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
|
|||||||
var updatedAt sql.NullTime
|
var updatedAt sql.NullTime
|
||||||
|
|
||||||
err := rows.Scan(
|
err := rows.Scan(
|
||||||
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
|
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
|
||||||
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
||||||
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
||||||
)
|
)
|
||||||
@@ -457,7 +576,7 @@ func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
|
|||||||
|
|
||||||
func (db *DB) GetJobByID(rawJobID int64) (*JobWithAnalysis, error) {
|
func (db *DB) GetJobByID(rawJobID int64) (*JobWithAnalysis, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
|
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
|
||||||
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
||||||
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
||||||
FROM raw_jobs r
|
FROM raw_jobs r
|
||||||
@@ -478,7 +597,7 @@ func (db *DB) GetJobByID(rawJobID int64) (*JobWithAnalysis, error) {
|
|||||||
var updatedAt sql.NullTime
|
var updatedAt sql.NullTime
|
||||||
|
|
||||||
err := row.Scan(
|
err := row.Scan(
|
||||||
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
|
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
|
||||||
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
||||||
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
||||||
)
|
)
|
||||||
@@ -554,7 +673,7 @@ func (db *DB) CreateOrUpdateApplication(rawJobID int64, status, notes string) (*
|
|||||||
|
|
||||||
func (db *DB) GetApplications() ([]JobWithAnalysis, error) {
|
func (db *DB) GetApplications() ([]JobWithAnalysis, error) {
|
||||||
query := `
|
query := `
|
||||||
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
|
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
|
||||||
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
||||||
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
||||||
FROM job_applications app
|
FROM job_applications app
|
||||||
@@ -582,7 +701,7 @@ func (db *DB) GetApplications() ([]JobWithAnalysis, error) {
|
|||||||
var updatedAt time.Time
|
var updatedAt time.Time
|
||||||
|
|
||||||
err := rows.Scan(
|
err := rows.Scan(
|
||||||
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
|
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
|
||||||
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
||||||
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -182,3 +182,118 @@ func TestRemoveTargetFromYAML(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestTouchJobLastSeenAndMaintenance(t *testing.T) {
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
dbPath := filepath.Join(tempDir, "test_maintenance.db")
|
||||||
|
|
||||||
|
database, err := InitDB(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to init db: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = database.Close() }()
|
||||||
|
|
||||||
|
// Insert jobs
|
||||||
|
job1, _, err := database.InsertRawJob("canon-1", nil, "Job 1", "Google", "Seattle", "https://google.com/1", "Software Engineer Go")
|
||||||
|
if err != nil || job1.ID == 0 {
|
||||||
|
t.Fatalf("failed to insert job 1: %v", err)
|
||||||
|
}
|
||||||
|
job2, _, err := database.InsertRawJob("canon-2", nil, "Job 2", "Apple", "Seattle", "https://apple.com/2", "Software Engineer Swift")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to insert job 2: %v", err)
|
||||||
|
}
|
||||||
|
job3, _, err := database.InsertRawJob("canon-3", nil, "Job 3", "Meta", "Seattle", "https://meta.com/3", "Software Engineer React")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to insert job 3: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1. Touch last seen for canon-1
|
||||||
|
if err := database.TouchJobLastSeen("canon-1"); err != nil {
|
||||||
|
t.Fatalf("TouchJobLastSeen failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make job2 and job3 artificially stale (20 days and 40 days old)
|
||||||
|
_, err = database.Exec(`UPDATE raw_jobs SET last_seen_at = datetime('now', '-20 days') WHERE id = ?`, job2.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to update last_seen_at: %v", err)
|
||||||
|
}
|
||||||
|
_, err = database.Exec(`UPDATE raw_jobs SET last_seen_at = datetime('now', '-40 days') WHERE id = ?`, job3.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to update last_seen_at: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Protect job3 in active pipeline (APPLIED)
|
||||||
|
_, _ = database.CreateOrUpdateApplication(job3.ID, "APPLIED", "Applied to Meta")
|
||||||
|
|
||||||
|
// 2. Enqueue expire tasks (> 14 days) with pipeline protection
|
||||||
|
queuedExpireCount, err := database.EnqueueExpireTasks(14, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("EnqueueExpireTasks failed: %v", err)
|
||||||
|
}
|
||||||
|
// job2 is stale and unprotected -> should be queued
|
||||||
|
// job3 is stale but protected (APPLIED) -> should NOT be queued
|
||||||
|
// job1 is fresh -> should NOT be queued
|
||||||
|
if queuedExpireCount != 1 {
|
||||||
|
t.Errorf("expected 1 job queued for expire, got %d", queuedExpireCount)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Directly expire job2
|
||||||
|
if err := database.ExpireJob(job2.ID, "Test expiration"); err != nil {
|
||||||
|
t.Fatalf("ExpireJob failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify job2 status in DB
|
||||||
|
j2Detail, err := database.GetJobByID(job2.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetJobByID failed: %v", err)
|
||||||
|
}
|
||||||
|
if j2Detail.App == nil || j2Detail.App.Status != "EXPIRED" {
|
||||||
|
t.Errorf("expected job2 to have EXPIRED application, got %v", j2Detail.App)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify job3 still APPLIED
|
||||||
|
j3Detail, err := database.GetJobByID(job3.ID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetJobByID failed: %v", err)
|
||||||
|
}
|
||||||
|
if j3Detail.App == nil || j3Detail.App.Status != "APPLIED" {
|
||||||
|
t.Errorf("expected job3 to remain APPLIED, got %v", j3Detail.App)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Stats check
|
||||||
|
stats, err := database.GetJobMaintenanceStats()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetJobMaintenanceStats failed: %v", err)
|
||||||
|
}
|
||||||
|
if stats["total_jobs"] != 3 {
|
||||||
|
t.Errorf("expected total_jobs=3, got %d", stats["total_jobs"])
|
||||||
|
}
|
||||||
|
if stats["active_7d"] != 1 {
|
||||||
|
t.Errorf("expected active_7d=1, got %d", stats["active_7d"])
|
||||||
|
}
|
||||||
|
if stats["stale_7d"] != 2 {
|
||||||
|
t.Errorf("expected stale_7d=2, got %d", stats["stale_7d"])
|
||||||
|
}
|
||||||
|
if stats["expired_count"] != 1 {
|
||||||
|
t.Errorf("expected expired_count=1, got %d", stats["expired_count"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDevDBMigration(t *testing.T) {
|
||||||
|
devDBPath := "../../data/db/dev/hireme.db"
|
||||||
|
if _, err := os.Stat(devDBPath); os.IsNotExist(err) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
database, err := InitDB(devDBPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("InitDB dev db failed: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = database.Close() }()
|
||||||
|
|
||||||
|
jobs, err := database.GetJobsWithAnalysis()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("GetJobsWithAnalysis on dev db failed: %v", err)
|
||||||
|
}
|
||||||
|
t.Logf("Successfully verified %d jobs from dev db", len(jobs))
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package ui
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
@@ -44,6 +45,8 @@ func (s *Server) RegisterRoutes(mux *http.ServeMux) {
|
|||||||
mux.HandleFunc("POST /api/tasks/reprocess", s.handleReprocessJobs)
|
mux.HandleFunc("POST /api/tasks/reprocess", s.handleReprocessJobs)
|
||||||
mux.HandleFunc("POST /api/tasks/toggle", s.handleToggleTasks)
|
mux.HandleFunc("POST /api/tasks/toggle", s.handleToggleTasks)
|
||||||
mux.HandleFunc("GET /api/status", s.handleGetStatus)
|
mux.HandleFunc("GET /api/status", s.handleGetStatus)
|
||||||
|
mux.HandleFunc("POST /api/maintenance/stale-expire", s.handleMaintenanceStaleExpire)
|
||||||
|
mux.HandleFunc("GET /api/maintenance/stats", s.handleMaintenanceStats)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) handleIndex(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) handleIndex(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -83,10 +86,16 @@ func (s *Server) handleIndex(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
mStats, err := s.db.GetJobMaintenanceStats()
|
||||||
|
if err != nil {
|
||||||
|
logger.Log.Warnw("Failed to get maintenance stats", "component", "UI", "error", err)
|
||||||
|
}
|
||||||
|
|
||||||
data := PageData{
|
data := PageData{
|
||||||
Targets: targets,
|
Targets: targets,
|
||||||
Jobs: jobs,
|
Jobs: jobs,
|
||||||
Applications: apps,
|
Applications: apps,
|
||||||
|
MaintenanceStats: mStats,
|
||||||
ActiveTab: "triage",
|
ActiveTab: "triage",
|
||||||
LLMProvider: s.cfg.LLMProvider,
|
LLMProvider: s.cfg.LLMProvider,
|
||||||
ToReviewCount: toReviewCount,
|
ToReviewCount: toReviewCount,
|
||||||
@@ -434,3 +443,39 @@ func (s *Server) handleGetStatus(w http.ResponseWriter, r *http.Request) {
|
|||||||
w.Header().Set("Content-Type", "application/json")
|
w.Header().Set("Content-Type", "application/json")
|
||||||
_ = json.NewEncoder(w).Encode(resp)
|
_ = json.NewEncoder(w).Encode(resp)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleMaintenanceStaleExpire(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if err := r.ParseForm(); err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
days, _ := strconv.Atoi(r.FormValue("days"))
|
||||||
|
if days <= 0 {
|
||||||
|
days = 14
|
||||||
|
}
|
||||||
|
protect := r.FormValue("protect") != "false"
|
||||||
|
|
||||||
|
count, err := s.db.EnqueueExpireTasks(days, protect)
|
||||||
|
if err != nil {
|
||||||
|
logger.Log.Errorw("Failed to enqueue expire tasks", "component", "UI", "error", err)
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]interface{}{
|
||||||
|
"status": "success",
|
||||||
|
"count": count,
|
||||||
|
"message": fmt.Sprintf("Enqueued %d expire task(s) for postings not seen in %d+ days.", count, days),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleMaintenanceStats(w http.ResponseWriter, r *http.Request) {
|
||||||
|
stats, err := s.db.GetJobMaintenanceStats()
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_ = json.NewEncoder(w).Encode(stats)
|
||||||
|
}
|
||||||
|
|||||||
@@ -82,3 +82,53 @@ func TestHandleDeleteTarget(t *testing.T) {
|
|||||||
func osWriteFile(path string, data []byte) error {
|
func osWriteFile(path string, data []byte) error {
|
||||||
return os.WriteFile(path, data, 0644)
|
return os.WriteFile(path, data, 0644)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMaintenanceHandlers(t *testing.T) {
|
||||||
|
tempDir := t.TempDir()
|
||||||
|
dbPath := filepath.Join(tempDir, "test_ui_maintenance.db")
|
||||||
|
|
||||||
|
database, err := db.InitDB(dbPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to init db: %v", err)
|
||||||
|
}
|
||||||
|
defer func() { _ = database.Close() }()
|
||||||
|
|
||||||
|
// Insert job
|
||||||
|
job, _, err := database.InsertRawJob("canon-m-1", nil, "Software Engineer", "Google", "Seattle", "https://example.com/job/1", "Go backend")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to insert raw job: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Make stale (25 days old)
|
||||||
|
_, _ = database.Exec(`UPDATE raw_jobs SET last_seen_at = datetime('now', '-25 days') WHERE id = ?`, job.ID)
|
||||||
|
|
||||||
|
cfg := &config.Config{}
|
||||||
|
server := NewServer(cfg, database, nil)
|
||||||
|
mux := http.NewServeMux()
|
||||||
|
server.RegisterRoutes(mux)
|
||||||
|
|
||||||
|
// 1. Test POST /api/maintenance/stale-expire
|
||||||
|
req := httptest.NewRequest("POST", "/api/maintenance/stale-expire", strings.NewReader("days=14&protect=true"))
|
||||||
|
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
mux.ServeHTTP(w, req)
|
||||||
|
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected status 200, got %d: %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(w.Body.String(), `"status":"success"`) || !strings.Contains(w.Body.String(), `"count":1`) {
|
||||||
|
t.Errorf("unexpected body: %s", w.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Test GET /api/maintenance/stats
|
||||||
|
reqStats := httptest.NewRequest("GET", "/api/maintenance/stats", nil)
|
||||||
|
wStats := httptest.NewRecorder()
|
||||||
|
mux.ServeHTTP(wStats, reqStats)
|
||||||
|
|
||||||
|
if wStats.Code != http.StatusOK {
|
||||||
|
t.Fatalf("expected status 200, got %d: %s", wStats.Code, wStats.Body.String())
|
||||||
|
}
|
||||||
|
if !strings.Contains(wStats.Body.String(), `"total_jobs":1`) {
|
||||||
|
t.Errorf("expected stats to show total_jobs:1, got %s", wStats.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -19,6 +19,8 @@ const applyTabHTML = `{{ define "applyTab" }}
|
|||||||
<option value="INTERVIEW">💬 Interview</option>
|
<option value="INTERVIEW">💬 Interview</option>
|
||||||
<option value="OFFER">🎉 Offer</option>
|
<option value="OFFER">🎉 Offer</option>
|
||||||
<option value="REJECTED">❌ Rejected</option>
|
<option value="REJECTED">❌ Rejected</option>
|
||||||
|
<option value="EXPIRED">⚠️ Expired / Closed</option>
|
||||||
|
<option value="OPTED_OUT">🚫 Opted Out</option>
|
||||||
<option value="all">All</option>
|
<option value="all">All</option>
|
||||||
</select>
|
</select>
|
||||||
<button hx-get="/applications" hx-target="#apply-container" hx-swap="innerHTML"
|
<button hx-get="/applications" hx-target="#apply-container" hx-swap="innerHTML"
|
||||||
@@ -69,6 +71,15 @@ const applyTableHTML = `{{ define "applyTable" }}
|
|||||||
|
|
||||||
<!-- Pipeline Stage (arrow indicator) -->
|
<!-- Pipeline Stage (arrow indicator) -->
|
||||||
<td class="px-4 py-3">
|
<td class="px-4 py-3">
|
||||||
|
{{ if eq .App.Status "EXPIRED" }}
|
||||||
|
<span class="px-2.5 py-0.5 rounded-full bg-amber-500/15 border border-amber-500/30 text-amber-300 text-[10px] font-bold">
|
||||||
|
⚠️ Expired / Closed
|
||||||
|
</span>
|
||||||
|
{{ else if eq .App.Status "OPTED_OUT" }}
|
||||||
|
<span class="px-2.5 py-0.5 rounded-full bg-slate-800 border border-slate-700 text-slate-400 text-[10px] font-medium">
|
||||||
|
🚫 Opted Out
|
||||||
|
</span>
|
||||||
|
{{ else }}
|
||||||
<div class="flex items-center gap-0.5 text-[10px] font-semibold">
|
<div class="flex items-center gap-0.5 text-[10px] font-semibold">
|
||||||
<span class="px-2 py-0.5 rounded-l-full {{ if eq .App.Status "SAVED" }}bg-slate-500 text-white{{ else }}bg-slate-800/60 text-slate-600{{ end }}">Saved</span>
|
<span class="px-2 py-0.5 rounded-l-full {{ if eq .App.Status "SAVED" }}bg-slate-500 text-white{{ else }}bg-slate-800/60 text-slate-600{{ end }}">Saved</span>
|
||||||
<span class="text-slate-700">▶</span>
|
<span class="text-slate-700">▶</span>
|
||||||
@@ -84,6 +95,7 @@ const applyTableHTML = `{{ define "applyTable" }}
|
|||||||
<span class="px-2 py-0.5 rounded-r-full bg-slate-800/60 text-slate-600">Offer</span>
|
<span class="px-2 py-0.5 rounded-r-full bg-slate-800/60 text-slate-600">Offer</span>
|
||||||
{{ end }}
|
{{ end }}
|
||||||
</div>
|
</div>
|
||||||
|
{{ end }}
|
||||||
</td>
|
</td>
|
||||||
|
|
||||||
<!-- Score -->
|
<!-- Score -->
|
||||||
@@ -122,6 +134,8 @@ const applyTableHTML = `{{ define "applyTable" }}
|
|||||||
<option value="INTERVIEW" {{ if eq .App.Status "INTERVIEW" }}selected{{ end }}>💬 Interview</option>
|
<option value="INTERVIEW" {{ if eq .App.Status "INTERVIEW" }}selected{{ end }}>💬 Interview</option>
|
||||||
<option value="OFFER" {{ if eq .App.Status "OFFER" }}selected{{ end }}>🎉 Offer</option>
|
<option value="OFFER" {{ if eq .App.Status "OFFER" }}selected{{ end }}>🎉 Offer</option>
|
||||||
<option value="REJECTED" {{ if eq .App.Status "REJECTED" }}selected{{ end }}>❌ Rejected</option>
|
<option value="REJECTED" {{ if eq .App.Status "REJECTED" }}selected{{ end }}>❌ Rejected</option>
|
||||||
|
<option value="EXPIRED" {{ if eq .App.Status "EXPIRED" }}selected{{ end }}>⚠️ Expired</option>
|
||||||
|
<option value="OPTED_OUT" {{ if eq .App.Status "OPTED_OUT" }}selected{{ end }}>🚫 Opted Out</option>
|
||||||
</select>
|
</select>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ const jobsTabHTML = `{{ define "jobsTab" }}
|
|||||||
<option value="not-qualified">❌ Not Qualified</option>
|
<option value="not-qualified">❌ Not Qualified</option>
|
||||||
<option value="saved">💾 Saved / In Pipeline</option>
|
<option value="saved">💾 Saved / In Pipeline</option>
|
||||||
<option value="all">🌐 All Jobs</option>
|
<option value="all">🌐 All Jobs</option>
|
||||||
|
<option value="expired">⚠️ Expired / Closed</option>
|
||||||
<option value="opted-out">🚫 Opted-out</option>
|
<option value="opted-out">🚫 Opted-out</option>
|
||||||
</select>
|
</select>
|
||||||
<button hx-get="/jobs" hx-target="#jobs-container" hx-swap="innerHTML"
|
<button hx-get="/jobs" hx-target="#jobs-container" hx-swap="innerHTML"
|
||||||
@@ -45,7 +46,7 @@ const jobListHTML = `{{ define "jobs" }}
|
|||||||
<th class="px-4 py-3" data-sort="status">Analysis</th>
|
<th class="px-4 py-3" data-sort="status">Analysis</th>
|
||||||
<th class="px-4 py-3" data-sort="score">Score & Qualification</th>
|
<th class="px-4 py-3" data-sort="score">Score & Qualification</th>
|
||||||
<th class="px-4 py-3">Reason / Tags</th>
|
<th class="px-4 py-3">Reason / Tags</th>
|
||||||
<th class="px-4 py-3" data-sort="date">Crawled</th>
|
<th class="px-4 py-3" data-sort="date">Last Updated</th>
|
||||||
<th class="px-4 py-3 text-right">Actions</th>
|
<th class="px-4 py-3 text-right">Actions</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
@@ -54,7 +55,7 @@ const jobListHTML = `{{ define "jobs" }}
|
|||||||
{{- $qual := "pending" -}}
|
{{- $qual := "pending" -}}
|
||||||
{{- if .Analysis }}{{ if .Analysis.IsQualified }}{{ $qual = "qualified" }}{{ else }}{{ $qual = "not-qualified" }}{{ end }}{{ end -}}
|
{{- if .Analysis }}{{ if .Analysis.IsQualified }}{{ $qual = "qualified" }}{{ else }}{{ $qual = "not-qualified" }}{{ end }}{{ end -}}
|
||||||
{{- $appClass := "none" -}}
|
{{- $appClass := "none" -}}
|
||||||
{{- if .App }}{{ if eq .App.Status "OPTED_OUT" }}{{ $appClass = "opted-out" }}{{ else }}{{ $appClass = "pipeline" }}{{ end }}{{ end -}}
|
{{- if .App }}{{ if eq .App.Status "OPTED_OUT" }}{{ $appClass = "opted-out" }}{{ else if eq .App.Status "EXPIRED" }}{{ $appClass = "expired" }}{{ else }}{{ $appClass = "pipeline" }}{{ end }}{{ end -}}
|
||||||
<tr class="hover:bg-slate-800/40 transition-colors group" data-row data-qual="{{ $qual }}" data-app="{{ $appClass }}">
|
<tr class="hover:bg-slate-800/40 transition-colors group" data-row data-qual="{{ $qual }}" data-app="{{ $appClass }}">
|
||||||
|
|
||||||
<!-- Title & Company -->
|
<!-- Title & Company -->
|
||||||
@@ -117,9 +118,13 @@ const jobListHTML = `{{ define "jobs" }}
|
|||||||
|
|
||||||
<!-- Reason / Matched Tags -->
|
<!-- Reason / Matched Tags -->
|
||||||
<td class="px-4 py-3 max-w-[180px]">
|
<td class="px-4 py-3 max-w-[180px]">
|
||||||
{{ if .App }}{{ if eq .App.Status "OPTED_OUT" }}
|
{{ if .App }}
|
||||||
<span class="text-[10px] text-slate-500 italic">Opted out</span>
|
{{ if eq .App.Status "OPTED_OUT" }}
|
||||||
{{ end }}{{ end }}
|
<span class="text-[10px] text-slate-500 italic block">Opted out</span>
|
||||||
|
{{ else if eq .App.Status "EXPIRED" }}
|
||||||
|
<span class="text-[10px] text-amber-400/80 italic block">⚠️ No longer active</span>
|
||||||
|
{{ end }}
|
||||||
|
{{ end }}
|
||||||
{{ if .Analysis }}
|
{{ if .Analysis }}
|
||||||
{{ if .Analysis.IsQualified }}
|
{{ if .Analysis.IsQualified }}
|
||||||
<div class="flex flex-wrap gap-1">
|
<div class="flex flex-wrap gap-1">
|
||||||
@@ -139,8 +144,8 @@ const jobListHTML = `{{ define "jobs" }}
|
|||||||
</td>
|
</td>
|
||||||
|
|
||||||
<!-- Date -->
|
<!-- Date -->
|
||||||
<td class="px-4 py-3 text-slate-400 text-[11px]" data-sort-val="{{ .CrawledAt.Unix }}">
|
<td class="px-4 py-3 text-slate-400 text-[11px]" data-sort-val="{{ .LastSeenAt.Unix }}">
|
||||||
{{ formatDate .CrawledAt }}
|
{{ formatDate .LastSeenAt }}
|
||||||
</td>
|
</td>
|
||||||
|
|
||||||
<!-- Actions: Save + Opt-out OR status badge -->
|
<!-- Actions: Save + Opt-out OR status badge -->
|
||||||
@@ -148,14 +153,32 @@ const jobListHTML = `{{ define "jobs" }}
|
|||||||
{{ if .App }}
|
{{ if .App }}
|
||||||
{{ if eq .App.Status "OPTED_OUT" }}
|
{{ if eq .App.Status "OPTED_OUT" }}
|
||||||
<!-- Opted out: show undo button -->
|
<!-- Opted out: show undo button -->
|
||||||
|
<div class="flex items-center justify-end gap-1.5">
|
||||||
|
<span class="text-[10px] text-slate-500">Opted-out</span>
|
||||||
<button hx-post="/applications"
|
<button hx-post="/applications"
|
||||||
hx-vals='{"raw_job_id": {{ .ID }}, "status": "SAVED"}'
|
hx-vals='{"raw_job_id": {{ .ID }}, "status": "SAVED"}'
|
||||||
hx-target="#jobs-container"
|
hx-target="#jobs-container"
|
||||||
hx-swap="innerHTML"
|
hx-swap="innerHTML"
|
||||||
title="Undo opt-out"
|
title="Undo opt-out"
|
||||||
class="px-2.5 py-1 rounded bg-slate-800 hover:bg-slate-700 text-slate-400 border border-slate-700 text-[10px] transition-colors">
|
class="px-2 py-0.5 rounded bg-slate-800 hover:bg-slate-700 text-slate-300 border border-slate-700 text-[10px] transition-colors">
|
||||||
↩ Undo
|
↩
|
||||||
</button>
|
</button>
|
||||||
|
</div>
|
||||||
|
{{ else if eq .App.Status "EXPIRED" }}
|
||||||
|
<!-- Expired -->
|
||||||
|
<div class="flex items-center justify-end gap-1.5">
|
||||||
|
<span class="px-2 py-0.5 rounded bg-slate-800 text-slate-400 text-[10px] font-bold border border-slate-700">
|
||||||
|
⚠️ EXPIRED
|
||||||
|
</span>
|
||||||
|
<button hx-post="/applications"
|
||||||
|
hx-vals='{"raw_job_id": {{ .ID }}, "status": "SAVED"}'
|
||||||
|
hx-target="#jobs-container"
|
||||||
|
hx-swap="innerHTML"
|
||||||
|
title="Re-open / Save"
|
||||||
|
class="px-2 py-0.5 rounded bg-slate-800 hover:bg-slate-700 text-slate-300 border border-slate-700 text-[10px] transition-colors">
|
||||||
|
↩
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
{{ else }}
|
{{ else }}
|
||||||
<!-- In pipeline -->
|
<!-- In pipeline -->
|
||||||
<span class="px-2 py-1 rounded bg-violet-500/20 text-violet-300 text-[10px] font-bold border border-violet-500/30">
|
<span class="px-2 py-1 rounded bg-violet-500/20 text-violet-300 text-[10px] font-bold border border-violet-500/30">
|
||||||
@@ -203,7 +226,7 @@ const jobDetailHTML = `{{ define "jobDetail" }}
|
|||||||
<div class="flex-1 min-w-0 pr-4">
|
<div class="flex-1 min-w-0 pr-4">
|
||||||
<span class="text-xs font-bold text-indigo-400 uppercase tracking-wide">{{ .Company }}</span>
|
<span class="text-xs font-bold text-indigo-400 uppercase tracking-wide">{{ .Company }}</span>
|
||||||
<h2 class="text-xl font-extrabold text-white mt-0.5 leading-tight">{{ .Title }}</h2>
|
<h2 class="text-xl font-extrabold text-white mt-0.5 leading-tight">{{ .Title }}</h2>
|
||||||
<p class="text-xs text-slate-400 mt-1">📍 {{ .Location }} • Ingested {{ formatDate .CrawledAt }}</p>
|
<p class="text-xs text-slate-400 mt-1">📍 {{ .Location }} • Last Updated {{ formatDate .LastSeenAt }}</p>
|
||||||
</div>
|
</div>
|
||||||
<button onclick="closeDetailModal()" class="flex-shrink-0 text-slate-400 hover:text-white text-lg p-1 rounded hover:bg-slate-800 transition-colors">✕</button>
|
<button onclick="closeDetailModal()" class="flex-shrink-0 text-slate-400 hover:text-white text-lg p-1 rounded hover:bg-slate-800 transition-colors">✕</button>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -102,7 +102,11 @@ const mainLayoutHTML = `{{ define "page" }}<!DOCTYPE html>
|
|||||||
</button>
|
</button>
|
||||||
<button id="tab-btn-targets" onclick="switchTab('targets')"
|
<button id="tab-btn-targets" onclick="switchTab('targets')"
|
||||||
class="tab-btn px-5 py-3 text-sm font-semibold border-b-2 transition-colors text-slate-400 border-transparent hover:text-slate-200">
|
class="tab-btn px-5 py-3 text-sm font-semibold border-b-2 transition-colors text-slate-400 border-transparent hover:text-slate-200">
|
||||||
⚙️ Targets
|
🎯 Targets
|
||||||
|
</button>
|
||||||
|
<button id="tab-btn-preferences" onclick="switchTab('preferences')"
|
||||||
|
class="tab-btn px-5 py-3 text-sm font-semibold border-b-2 transition-colors text-slate-400 border-transparent hover:text-slate-200">
|
||||||
|
⚙️ Preferences
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -126,6 +130,11 @@ const mainLayoutHTML = `{{ define "page" }}<!DOCTYPE html>
|
|||||||
{{ template "targetsTab" . }}
|
{{ template "targetsTab" . }}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Tab: Preferences -->
|
||||||
|
<div id="tab-content-preferences" class="hidden space-y-5">
|
||||||
|
{{ template "preferencesTab" . }}
|
||||||
|
</div>
|
||||||
|
|
||||||
</main>
|
</main>
|
||||||
|
|
||||||
<!-- Right Slide-Over Detail Panel -->
|
<!-- Right Slide-Over Detail Panel -->
|
||||||
@@ -230,19 +239,25 @@ const mainLayoutHTML = `{{ define "page" }}<!DOCTYPE html>
|
|||||||
|
|
||||||
<script>
|
<script>
|
||||||
// ── Tab Routing ──────────────────────────────────────────────────────
|
// ── Tab Routing ──────────────────────────────────────────────────────
|
||||||
const TABS = ['jobs', 'apply', 'targets'];
|
const TABS = ['jobs', 'apply', 'targets', 'preferences'];
|
||||||
|
|
||||||
function switchTab(tab) {
|
function switchTab(tab) {
|
||||||
TABS.forEach(t => {
|
TABS.forEach(t => {
|
||||||
document.getElementById('tab-content-' + t).classList.add('hidden');
|
const content = document.getElementById('tab-content-' + t);
|
||||||
|
if (content) content.classList.add('hidden');
|
||||||
const btn = document.getElementById('tab-btn-' + t);
|
const btn = document.getElementById('tab-btn-' + t);
|
||||||
|
if (btn) {
|
||||||
btn.classList.remove('text-indigo-400', 'border-indigo-500');
|
btn.classList.remove('text-indigo-400', 'border-indigo-500');
|
||||||
btn.classList.add('text-slate-400', 'border-transparent');
|
btn.classList.add('text-slate-400', 'border-transparent');
|
||||||
|
}
|
||||||
});
|
});
|
||||||
document.getElementById('tab-content-' + tab).classList.remove('hidden');
|
const activeContent = document.getElementById('tab-content-' + tab);
|
||||||
|
if (activeContent) activeContent.classList.remove('hidden');
|
||||||
const activeBtn = document.getElementById('tab-btn-' + tab);
|
const activeBtn = document.getElementById('tab-btn-' + tab);
|
||||||
|
if (activeBtn) {
|
||||||
activeBtn.classList.remove('text-slate-400', 'border-transparent');
|
activeBtn.classList.remove('text-slate-400', 'border-transparent');
|
||||||
activeBtn.classList.add('text-indigo-400', 'border-indigo-500');
|
activeBtn.classList.add('text-indigo-400', 'border-indigo-500');
|
||||||
|
}
|
||||||
history.replaceState(null, '', '#' + tab);
|
history.replaceState(null, '', '#' + tab);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -308,6 +323,49 @@ const mainLayoutHTML = `{{ define "page" }}<!DOCTYPE html>
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ── Maintenance Actions ──────────────────────────────────────────────
|
||||||
|
function runMaintenanceAction(url, daysInputId, protectCheckboxId, btn, originalText) {
|
||||||
|
const days = document.getElementById(daysInputId)?.value || '14';
|
||||||
|
const protect = document.getElementById(protectCheckboxId)?.checked ?? true;
|
||||||
|
const alertBox = document.getElementById('maintenance-alert');
|
||||||
|
|
||||||
|
btn.disabled = true;
|
||||||
|
btn.textContent = 'Processing…';
|
||||||
|
|
||||||
|
fetch(url, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||||||
|
body: new URLSearchParams({ days: days, protect: protect ? 'true' : 'false' })
|
||||||
|
})
|
||||||
|
.then(res => res.json())
|
||||||
|
.then(data => {
|
||||||
|
if (alertBox) {
|
||||||
|
alertBox.className = 'p-4 rounded-xl border text-xs font-semibold ' +
|
||||||
|
(data.status === 'success' ? 'bg-emerald-950/60 border-emerald-500/40 text-emerald-300' : 'bg-rose-950/60 border-rose-500/40 text-rose-300');
|
||||||
|
alertBox.innerHTML = '<span>✓</span> ' + (data.message || 'Operation completed successfully.');
|
||||||
|
alertBox.classList.remove('hidden');
|
||||||
|
setTimeout(() => { alertBox.classList.add('hidden'); }, 8000);
|
||||||
|
}
|
||||||
|
// Refresh Jobs and Apply tables & stats
|
||||||
|
const jobsContainer = document.getElementById('jobs-container');
|
||||||
|
if (jobsContainer) htmx.trigger(jobsContainer, 'refresh');
|
||||||
|
const applyContainer = document.getElementById('apply-container');
|
||||||
|
if (applyContainer) htmx.trigger(applyContainer, 'refresh');
|
||||||
|
refreshStatus();
|
||||||
|
})
|
||||||
|
.catch(err => {
|
||||||
|
if (alertBox) {
|
||||||
|
alertBox.className = 'p-4 rounded-xl border text-xs font-semibold bg-rose-950/60 border-rose-500/40 text-rose-300';
|
||||||
|
alertBox.innerHTML = '<span>✕</span> Error: ' + err.message;
|
||||||
|
alertBox.classList.remove('hidden');
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
btn.disabled = false;
|
||||||
|
btn.textContent = originalText;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
// ── Bottom Bar: Status Polling ────────────────────────────────────────
|
// ── Bottom Bar: Status Polling ────────────────────────────────────────
|
||||||
function refreshStatus() {
|
function refreshStatus() {
|
||||||
fetch('/api/status')
|
fetch('/api/status')
|
||||||
@@ -392,14 +450,15 @@ const mainLayoutHTML = `{{ define "page" }}<!DOCTYPE html>
|
|||||||
table.querySelectorAll('tr[data-row]').forEach(row => {
|
table.querySelectorAll('tr[data-row]').forEach(row => {
|
||||||
const rowText = row.textContent.toLowerCase();
|
const rowText = row.textContent.toLowerCase();
|
||||||
const rowQual = row.dataset.qual || ''; // qualified | not-qualified | pending
|
const rowQual = row.dataset.qual || ''; // qualified | not-qualified | pending
|
||||||
const rowApp = row.dataset.app || 'none'; // none | opted-out | pipeline
|
const rowApp = row.dataset.app || 'none'; // none | opted-out | expired | pipeline
|
||||||
const textMatch = !text || rowText.includes(text);
|
const textMatch = !text || rowText.includes(text);
|
||||||
let qualMatch;
|
let qualMatch;
|
||||||
switch (qual) {
|
switch (qual) {
|
||||||
case 'default': qualMatch = rowApp !== 'opted-out' && rowApp !== 'pipeline' && (rowQual === 'qualified' || rowQual === 'pending'); break;
|
case 'default': qualMatch = rowApp !== 'opted-out' && rowApp !== 'expired' && rowApp !== 'pipeline' && (rowQual === 'qualified' || rowQual === 'pending'); break;
|
||||||
case 'not-qualified': qualMatch = rowApp !== 'opted-out' && rowQual === 'not-qualified'; break;
|
case 'not-qualified': qualMatch = rowApp !== 'opted-out' && rowApp !== 'expired' && rowQual === 'not-qualified'; break;
|
||||||
case 'saved': qualMatch = rowApp === 'pipeline'; break;
|
case 'saved': qualMatch = rowApp === 'pipeline'; break;
|
||||||
case 'all': qualMatch = rowApp !== 'opted-out'; break;
|
case 'all': qualMatch = true; break;
|
||||||
|
case 'expired': qualMatch = rowApp === 'expired'; break;
|
||||||
case 'opted-out': qualMatch = rowApp === 'opted-out'; break;
|
case 'opted-out': qualMatch = rowApp === 'opted-out'; break;
|
||||||
default: qualMatch = true;
|
default: qualMatch = true;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package ui
|
||||||
|
|
||||||
|
const preferencesTabHTML = `{{ define "preferencesTab" }}
|
||||||
|
<div class="space-y-6 max-w-4xl">
|
||||||
|
<div>
|
||||||
|
<h2 class="text-base font-bold text-white flex items-center gap-2">
|
||||||
|
<span>⚙️</span> System Setup & Maintenance
|
||||||
|
</h2>
|
||||||
|
<p class="text-xs text-slate-400 mt-0.5">Monitor database health and purge stale job postings not seen on target career sites</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Maintenance Result Alert Banner -->
|
||||||
|
<div id="maintenance-alert" class="hidden p-4 rounded-xl border transition-all"></div>
|
||||||
|
|
||||||
|
<!-- Statistics Overview Grid -->
|
||||||
|
<div class="grid grid-cols-2 sm:grid-cols-3 lg:grid-cols-6 gap-3">
|
||||||
|
<div class="p-3.5 rounded-xl bg-slate-900/70 border border-slate-800">
|
||||||
|
<span class="text-[10px] uppercase font-bold text-slate-400 block tracking-wider">Total Ingested</span>
|
||||||
|
<span class="text-xl font-extrabold text-white mt-1 block">{{ .MaintenanceStats.total_jobs }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="p-3.5 rounded-xl bg-slate-900/70 border border-slate-800">
|
||||||
|
<span class="text-[10px] uppercase font-bold text-emerald-400 block tracking-wider">Active (≤7d)</span>
|
||||||
|
<span class="text-xl font-extrabold text-emerald-400 mt-1 block">{{ .MaintenanceStats.active_7d }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="p-3.5 rounded-xl bg-slate-900/70 border border-slate-800">
|
||||||
|
<span class="text-[10px] uppercase font-bold text-amber-400 block tracking-wider">Stale (>7d)</span>
|
||||||
|
<span class="text-xl font-extrabold text-amber-400 mt-1 block">{{ .MaintenanceStats.stale_7d }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="p-3.5 rounded-xl bg-slate-900/70 border border-slate-800">
|
||||||
|
<span class="text-[10px] uppercase font-bold text-rose-400 block tracking-wider">Stale (>30d)</span>
|
||||||
|
<span class="text-xl font-extrabold text-rose-400 mt-1 block">{{ .MaintenanceStats.stale_30d }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="p-3.5 rounded-xl bg-slate-900/70 border border-slate-800">
|
||||||
|
<span class="text-[10px] uppercase font-bold text-slate-400 block tracking-wider">Expired / Closed</span>
|
||||||
|
<span class="text-xl font-extrabold text-slate-300 mt-1 block">{{ .MaintenanceStats.expired_count }}</span>
|
||||||
|
</div>
|
||||||
|
<div class="p-3.5 rounded-xl bg-slate-900/70 border border-slate-800">
|
||||||
|
<span class="text-[10px] uppercase font-bold text-violet-400 block tracking-wider">In Pipeline</span>
|
||||||
|
<span class="text-xl font-extrabold text-violet-300 mt-1 block">{{ .MaintenanceStats.in_pipeline }}</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Maintenance Tool: Purge Stale Postings (Worker Queue Driven) -->
|
||||||
|
<div class="p-6 rounded-xl bg-slate-900/60 border border-slate-800 space-y-5">
|
||||||
|
<div class="flex items-start justify-between">
|
||||||
|
<div class="flex items-center gap-3">
|
||||||
|
<span class="w-9 h-9 rounded-xl bg-amber-500/10 border border-amber-500/30 flex items-center justify-center text-lg">⚠️</span>
|
||||||
|
<div>
|
||||||
|
<h3 class="text-sm font-bold text-white">Purge Stale Postings (Mark Expired)</h3>
|
||||||
|
<p class="text-xs text-slate-400 mt-0.5">Enqueues worker queue tasks to mark job postings as <strong class="text-amber-300 font-semibold">Expired</strong> if they have not been discovered on target career boards for X days.</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="pt-4 border-t border-slate-800/80 grid grid-cols-1 sm:grid-cols-3 gap-4 items-end">
|
||||||
|
<div>
|
||||||
|
<label for="expire-days" class="block text-xs font-semibold text-slate-300 mb-1.5">Not updated in:</label>
|
||||||
|
<select id="expire-days" class="w-full px-3 py-2 text-xs rounded-lg bg-slate-800 border border-slate-700 text-white focus:outline-none focus:border-indigo-500">
|
||||||
|
<option value="7">7+ days</option>
|
||||||
|
<option value="14" selected>14+ days</option>
|
||||||
|
<option value="30">30+ days</option>
|
||||||
|
<option value="60">60+ days</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="flex items-center gap-2 pb-2">
|
||||||
|
<input type="checkbox" id="expire-protect-pipeline" checked class="rounded bg-slate-800 border-slate-700 text-indigo-600 focus:ring-0">
|
||||||
|
<label for="expire-protect-pipeline" class="text-xs text-slate-300">Protect active applications</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<button type="button"
|
||||||
|
onclick="runMaintenanceAction('/api/maintenance/stale-expire', 'expire-days', 'expire-protect-pipeline', this, '⚠️ Queue Expire Tasks')"
|
||||||
|
class="w-full py-2 bg-amber-600 hover:bg-amber-500 text-xs font-semibold text-white rounded-lg shadow-md transition-all active:scale-95">
|
||||||
|
⚠️ Queue Expire Tasks
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Explanation Card -->
|
||||||
|
<div class="p-5 rounded-xl bg-slate-900/40 border border-slate-800/80 text-xs text-slate-400 space-y-2">
|
||||||
|
<h4 class="font-bold text-slate-200 uppercase tracking-wider text-[11px]">💡 How Job Verification Works</h4>
|
||||||
|
<p>
|
||||||
|
Every time <strong class="text-indigo-300">🔍 Discover</strong> runs, Hireme searches live listings directly on career sites (Google, Apple, Meta).
|
||||||
|
When an existing job appears in search results, its <strong class="text-emerald-400">Last Updated</strong> timestamp is refreshed.
|
||||||
|
If a role closes or is removed from search results, its Last Updated timestamp remains unchanged, allowing you to safely purge expired postings using the tool above.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{{ end }}`
|
||||||
@@ -72,6 +72,7 @@ type PageData struct {
|
|||||||
Targets []db.SearchTarget
|
Targets []db.SearchTarget
|
||||||
Jobs []db.JobWithAnalysis
|
Jobs []db.JobWithAnalysis
|
||||||
Applications []db.JobWithAnalysis
|
Applications []db.JobWithAnalysis
|
||||||
|
MaintenanceStats map[string]int
|
||||||
ActiveTab string
|
ActiveTab string
|
||||||
LLMProvider string
|
LLMProvider string
|
||||||
ToReviewCount int
|
ToReviewCount int
|
||||||
@@ -92,6 +93,7 @@ func init() {
|
|||||||
jobDetailHTML,
|
jobDetailHTML,
|
||||||
targetsTabHTML,
|
targetsTabHTML,
|
||||||
targetListHTML,
|
targetListHTML,
|
||||||
|
preferencesTabHTML,
|
||||||
applyTabHTML,
|
applyTabHTML,
|
||||||
applyTableHTML,
|
applyTableHTML,
|
||||||
kanbanBoardHTML,
|
kanbanBoardHTML,
|
||||||
@@ -131,3 +133,8 @@ func RenderKanbanBoard(w io.Writer, apps []db.JobWithAnalysis) error {
|
|||||||
func RenderApplyTable(w io.Writer, apps []db.JobWithAnalysis) error {
|
func RenderApplyTable(w io.Writer, apps []db.JobWithAnalysis) error {
|
||||||
return rootTemplate.ExecuteTemplate(w, "applyTable", apps)
|
return rootTemplate.ExecuteTemplate(w, "applyTable", apps)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func RenderPreferencesTab(w io.Writer, data PageData) error {
|
||||||
|
return rootTemplate.ExecuteTemplate(w, "preferencesTab", data)
|
||||||
|
}
|
||||||
|
|
||||||
|
|||||||
+32
-11
@@ -32,7 +32,6 @@ type WorkerPool struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewWorkerPool(database *db.DB, analyzer analyzer.LLMAnalyzer, numWorkers int, filterCfg *config.FilterConfig) *WorkerPool {
|
func NewWorkerPool(database *db.DB, analyzer analyzer.LLMAnalyzer, numWorkers int, filterCfg *config.FilterConfig) *WorkerPool {
|
||||||
// Initialize filters using the provided config
|
|
||||||
filterPipe := filter.NewPipeline(
|
filterPipe := filter.NewPipeline(
|
||||||
filter.NewHardKeywordFilter(filterCfg.HardKeywords),
|
filter.NewHardKeywordFilter(filterCfg.HardKeywords),
|
||||||
filter.NewScoringFilter(filterCfg.ScoringWeights),
|
filter.NewScoringFilter(filterCfg.ScoringWeights),
|
||||||
@@ -54,33 +53,37 @@ func (p *WorkerPool) Toggle() bool {
|
|||||||
if p.IsRunning() {
|
if p.IsRunning() {
|
||||||
p.Stop()
|
p.Stop()
|
||||||
return false
|
return false
|
||||||
}
|
} else {
|
||||||
p.Start()
|
p.Start()
|
||||||
return true
|
return true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *WorkerPool) Start() {
|
func (p *WorkerPool) Start() {
|
||||||
if p.isRunning.Load() {
|
if p.isRunning.Swap(true) {
|
||||||
return
|
return // Already running
|
||||||
}
|
}
|
||||||
|
|
||||||
p.ctx, p.cancel = context.WithCancel(context.Background())
|
p.ctx, p.cancel = context.WithCancel(context.Background())
|
||||||
p.isRunning.Store(true)
|
logger.Log.Infow("Started workers", "component", "WorkerPool", "count", p.workerCount)
|
||||||
|
|
||||||
for i := 0; i < p.workerCount; i++ {
|
for i := 0; i < p.workerCount; i++ {
|
||||||
p.wg.Add(1)
|
p.wg.Add(1)
|
||||||
go p.workerLoop(i)
|
go p.workerLoop(i)
|
||||||
}
|
}
|
||||||
logger.Log.Infow("Started workers", "component", "WorkerPool", "count", p.workerCount)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *WorkerPool) Stop() {
|
func (p *WorkerPool) Stop() {
|
||||||
if !p.isRunning.Load() {
|
if !p.isRunning.Swap(false) {
|
||||||
return
|
return // Already stopped
|
||||||
}
|
}
|
||||||
|
|
||||||
|
logger.Log.Infow("Stopping worker pool...", "component", "WorkerPool")
|
||||||
|
if p.cancel != nil {
|
||||||
p.cancel()
|
p.cancel()
|
||||||
p.isRunning.Store(false)
|
}
|
||||||
p.wg.Wait()
|
p.wg.Wait()
|
||||||
logger.Log.Infow("All workers stopped", "component", "WorkerPool")
|
logger.Log.Infow("Worker pool stopped", "component", "WorkerPool")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (p *WorkerPool) workerLoop(workerID int) {
|
func (p *WorkerPool) workerLoop(workerID int) {
|
||||||
@@ -95,7 +98,7 @@ func (p *WorkerPool) workerLoop(workerID int) {
|
|||||||
default:
|
default:
|
||||||
task, err := p.db.ClaimNextTask()
|
task, err := p.db.ClaimNextTask()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Log.Errorw("Error claiming task", "component", "Worker", "worker_id", workerID, "error", err)
|
logger.Log.Errorw("Failed to claim task", "component", "Worker", "worker_id", workerID, "error", err)
|
||||||
time.Sleep(1 * time.Second)
|
time.Sleep(1 * time.Second)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -119,6 +122,8 @@ func (p *WorkerPool) workerLoop(workerID int) {
|
|||||||
processErr = p.handleProcess(p.ctx, task)
|
processErr = p.handleProcess(p.ctx, task)
|
||||||
case "ANALYZE":
|
case "ANALYZE":
|
||||||
processErr = p.handleAnalyze(p.ctx, task)
|
processErr = p.handleAnalyze(p.ctx, task)
|
||||||
|
case "EXPIRE_JOB":
|
||||||
|
processErr = p.handleExpireJob(p.ctx, task)
|
||||||
default:
|
default:
|
||||||
processErr = fmt.Errorf("unknown task type: %s", task.TaskType)
|
processErr = fmt.Errorf("unknown task type: %s", task.TaskType)
|
||||||
}
|
}
|
||||||
@@ -159,6 +164,11 @@ type AnalyzePayload struct {
|
|||||||
RawJobID int64 `json:"raw_job_id"`
|
RawJobID int64 `json:"raw_job_id"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ExpireJobPayload struct {
|
||||||
|
RawJobID int64 `json:"raw_job_id"`
|
||||||
|
Reason string `json:"reason"`
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------
|
// ---------------------------------------------------------
|
||||||
// Task Handlers
|
// Task Handlers
|
||||||
// ---------------------------------------------------------
|
// ---------------------------------------------------------
|
||||||
@@ -179,6 +189,8 @@ func (p *WorkerPool) handleDiscover(ctx context.Context, task *db.Task) error {
|
|||||||
// Check deduplication (already fetched/inserted jobs)
|
// Check deduplication (already fetched/inserted jobs)
|
||||||
exists, err := p.db.CanonicalIDExists(disc.CanonicalID)
|
exists, err := p.db.CanonicalIDExists(disc.CanonicalID)
|
||||||
if err == nil && exists {
|
if err == nil && exists {
|
||||||
|
// Update last seen timestamp to confirm job is still active on target board
|
||||||
|
_ = p.db.TouchJobLastSeen(disc.CanonicalID)
|
||||||
continue // Skip existing job
|
continue // Skip existing job
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -289,6 +301,15 @@ func (p *WorkerPool) handleAnalyze(ctx context.Context, task *db.Task) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *WorkerPool) handleExpireJob(ctx context.Context, task *db.Task) error {
|
||||||
|
var payload ExpireJobPayload
|
||||||
|
if err := json.Unmarshal([]byte(task.Payload), &payload); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
logger.Log.Infow("Expiring job", "component", "Worker", "raw_job_id", payload.RawJobID, "reason", payload.Reason)
|
||||||
|
return p.db.ExpireJob(payload.RawJobID, payload.Reason)
|
||||||
|
}
|
||||||
|
|
||||||
func (p *WorkerPool) getCrawlerForPlatform(platform string) crawler.SiteCrawler {
|
func (p *WorkerPool) getCrawlerForPlatform(platform string) crawler.SiteCrawler {
|
||||||
plat := strings.ToLower(platform)
|
plat := strings.ToLower(platform)
|
||||||
if strings.Contains(plat, "google") {
|
if strings.Contains(plat, "google") {
|
||||||
|
|||||||
Reference in New Issue
Block a user