feat: add job maintenance dashboard and automated stale job expiration system

2026-08-22 01:51:43 -07:00
parent 367929e7d0
commit 7ec91f384f
12 changed files with 633 additions and 56 deletions
@@ -55,7 +55,8 @@ func (db *DB) migrate() error {
 		location TEXT,
 		apply_url TEXT NOT NULL,
 		raw_content TEXT NOT NULL,
-		crawled_at DATETIME DEFAULT CURRENT_TIMESTAMP
+		crawled_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+		last_seen_at DATETIME DEFAULT CURRENT_TIMESTAMP
 	);

 	CREATE TABLE IF NOT EXISTS job_analyses (
@@ -92,6 +93,37 @@ func (db *DB) migrate() error {
 	);
 	`

-	_, err := db.Exec(schema)
-	return err
+	if _, err := db.Exec(schema); err != nil {
+		return err
+	}
+
+	// Dynamic migration for existing databases: check if last_seen_at exists in raw_jobs
+	var hasLastSeen bool
+	rows, err := db.Query(`PRAGMA table_info(raw_jobs)`)
+	if err == nil {
+		for rows.Next() {
+			var cid int
+			var name, ctype string
+			var notnull, pk int
+			var dfltValue sql.NullString
+			if err := rows.Scan(&cid, &name, &ctype, &notnull, &dfltValue, &pk); err == nil {
+				if name == "last_seen_at" {
+					hasLastSeen = true
+					break
+				}
+			}
+		}
+		_ = rows.Close()
+	}
+
+	if !hasLastSeen {
+		if _, err := db.Exec(`ALTER TABLE raw_jobs ADD COLUMN last_seen_at DATETIME`); err != nil {
+			logger.Log.Warnw("Failed to add last_seen_at column", "component", "DB", "error", err)
+		} else {
+			_, _ = db.Exec(`UPDATE raw_jobs SET last_seen_at = crawled_at WHERE last_seen_at IS NULL`)
+			_, _ = db.Exec(`UPDATE raw_jobs SET last_seen_at = CURRENT_TIMESTAMP WHERE last_seen_at IS NULL`)
+		}
+	}
+
+	return nil
 }
@@ -23,6 +23,7 @@ type RawJob struct {
 	ApplyURL    string    `json:"apply_url"`
 	RawContent  string    `json:"raw_content"`
 	CrawledAt   time.Time `json:"crawled_at"`
+	LastSeenAt  time.Time `json:"last_seen_at"`
 }

 type Task struct {
@@ -185,15 +185,15 @@ func (db *DB) InsertRawJob(canonicalID string, targetID *int64, title, company,
 	}

 	query := `
-	INSERT INTO raw_jobs (canonical_id, target_id, title, company, location, apply_url, raw_content)
-	VALUES (?, ?, ?, ?, ?, ?, ?)
+	INSERT INTO raw_jobs (canonical_id, target_id, title, company, location, apply_url, raw_content, crawled_at, last_seen_at)
+	VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
 	ON CONFLICT(canonical_id) DO NOTHING
-	RETURNING id, canonical_id, target_id, title, company, location, apply_url, raw_content, crawled_at
+	RETURNING id, canonical_id, target_id, title, company, location, apply_url, raw_content, crawled_at, last_seen_at
 	`
 	var job RawJob
 	var tID sql.NullInt64
 	err = db.QueryRow(query, canonicalID, targetID, title, company, location, applyURL, rawContent).Scan(
-		&job.ID, &job.CanonicalID, &tID, &job.Title, &job.Company, &job.Location, &job.ApplyURL, &job.RawContent, &job.CrawledAt,
+		&job.ID, &job.CanonicalID, &tID, &job.Title, &job.Company, &job.Location, &job.ApplyURL, &job.RawContent, &job.CrawledAt, &job.LastSeenAt,
 	)
 	if err == sql.ErrNoRows {
 		return nil, false, nil // Duplicate job
@@ -207,6 +207,125 @@ func (db *DB) InsertRawJob(canonicalID string, targetID *int64, title, company,
 	return &job, true, nil
 }

+func (db *DB) TouchJobLastSeen(canonicalID string) error {
+	query := `UPDATE raw_jobs SET last_seen_at = CURRENT_TIMESTAMP WHERE canonical_id = ?`
+	_, err := db.Exec(query, canonicalID)
+	return err
+}
+
+func (db *DB) TouchJobLastSeenByID(rawJobID int64) error {
+	query := `UPDATE raw_jobs SET last_seen_at = CURRENT_TIMESTAMP WHERE id = ?`
+	_, err := db.Exec(query, rawJobID)
+	return err
+}
+
+func (db *DB) ExpireJob(rawJobID int64, reason string) error {
+	if reason == "" {
+		reason = "Marked expired"
+	}
+	query := `
+	INSERT INTO job_applications (raw_job_id, status, notes, updated_at)
+	VALUES (?, 'EXPIRED', ?, CURRENT_TIMESTAMP)
+	ON CONFLICT(raw_job_id) DO UPDATE SET 
+		status='EXPIRED',
+		notes=COALESCE(job_applications.notes || ' | ', '') || excluded.notes,
+		updated_at=CURRENT_TIMESTAMP
+	`
+	_, err := db.Exec(query, rawJobID, reason)
+	return err
+}
+
+func (db *DB) EnqueueExpireTasks(days int, preserveActivePipeline bool) (int, error) {
+	tx, err := db.Begin()
+	if err != nil {
+		return 0, err
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	var query string
+	if preserveActivePipeline {
+		query = `
+		SELECT r.id FROM raw_jobs r
+		LEFT JOIN job_applications app ON r.id = app.raw_job_id
+		WHERE r.last_seen_at < datetime('now', '-' || ? || ' days')
+		  AND (app.status IS NULL OR app.status NOT IN ('APPLIED', 'INTERVIEW', 'OFFER', 'EXPIRED', 'OPTED_OUT'))
+		`
+	} else {
+		query = `
+		SELECT r.id FROM raw_jobs r
+		LEFT JOIN job_applications app ON r.id = app.raw_job_id
+		WHERE r.last_seen_at < datetime('now', '-' || ? || ' days')
+		  AND (app.status IS NULL OR app.status NOT IN ('EXPIRED', 'OPTED_OUT'))
+		`
+	}
+
+	rows, err := tx.Query(query, days)
+	if err != nil {
+		return 0, err
+	}
+
+	var rawJobIDs []int64
+	for rows.Next() {
+		var id int64
+		if err := rows.Scan(&id); err != nil {
+			_ = rows.Close()
+			return 0, err
+		}
+		rawJobIDs = append(rawJobIDs, id)
+	}
+	_ = rows.Close()
+
+	for _, id := range rawJobIDs {
+		payloadJSON := fmt.Sprintf(`{"raw_job_id":%d,"reason":"Direct expire: not seen on target for %d+ days"}`, id, days)
+		if _, err := tx.Exec(`INSERT INTO tasks (task_type, payload, status) VALUES ('EXPIRE_JOB', ?, 'PENDING')`, payloadJSON); err != nil {
+			return 0, err
+		}
+	}
+
+	return len(rawJobIDs), tx.Commit()
+}
+
+func (db *DB) GetJobMaintenanceStats() (map[string]int, error) {
+	stats := make(map[string]int)
+
+	// Total raw jobs
+	var totalJobs int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs`).Scan(&totalJobs)
+	stats["total_jobs"] = totalJobs
+
+	// Active <= 7 days
+	var active7d int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs WHERE last_seen_at >= datetime('now', '-7 days')`).Scan(&active7d)
+	stats["active_7d"] = active7d
+
+	// Stale > 7 days
+	var stale7d int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs WHERE last_seen_at < datetime('now', '-7 days')`).Scan(&stale7d)
+	stats["stale_7d"] = stale7d
+
+	// Stale > 30 days
+	var stale30d int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs WHERE last_seen_at < datetime('now', '-30 days')`).Scan(&stale30d)
+	stats["stale_30d"] = stale30d
+
+	// Expired count
+	var expiredCount int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM job_applications WHERE status = 'EXPIRED'`).Scan(&expiredCount)
+	stats["expired_count"] = expiredCount
+
+	// Opted out count
+	var optedOutCount int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM job_applications WHERE status = 'OPTED_OUT'`).Scan(&optedOutCount)
+	stats["opted_out_count"] = optedOutCount
+
+	// In pipeline count
+	var inPipeline int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM job_applications WHERE status IN ('SAVED', 'APPLIED', 'INTERVIEW', 'OFFER')`).Scan(&inPipeline)
+	stats["in_pipeline"] = inPipeline
+
+	return stats, nil
+}
+
 func (db *DB) CanonicalIDExists(canonicalID string) (bool, error) {
 	query := `SELECT EXISTS(SELECT 1 FROM raw_jobs WHERE canonical_id = ?)`
 	var exists bool
@@ -330,7 +449,7 @@ func (db *DB) UpdateJobAnalysisLLM(rawJobID int64, matchedJSON, missingJSON, sug

 func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
 	query := `
-	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
+	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
 	       a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), a.status, a.analyzed_at
 	FROM raw_jobs r
 	JOIN job_analyses a ON r.id = a.raw_job_id
@@ -353,7 +472,7 @@ func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
 		var analyzedAt sql.NullTime

 		err := rows.Scan(
-			&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
+			&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
 			&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
 		)
 		if err != nil {
@@ -383,7 +502,7 @@ func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {

 func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
 	query := `
-	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
+	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
 	       a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
 	       app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
 	FROM raw_jobs r
@@ -411,7 +530,7 @@ func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
 		var updatedAt sql.NullTime

 		err := rows.Scan(
-			&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
+			&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
 			&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
 			&appID, &appStatus, &notes, &appliedAt, &updatedAt,
 		)
@@ -457,7 +576,7 @@ func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {

 func (db *DB) GetJobByID(rawJobID int64) (*JobWithAnalysis, error) {
 	query := `
-	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
+	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
 	       a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
 	       app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
 	FROM raw_jobs r
@@ -478,7 +597,7 @@ func (db *DB) GetJobByID(rawJobID int64) (*JobWithAnalysis, error) {
 	var updatedAt sql.NullTime

 	err := row.Scan(
-		&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
+		&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
 		&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
 		&appID, &appStatus, &notes, &appliedAt, &updatedAt,
 	)
@@ -554,7 +673,7 @@ func (db *DB) CreateOrUpdateApplication(rawJobID int64, status, notes string) (*

 func (db *DB) GetApplications() ([]JobWithAnalysis, error) {
 	query := `
-	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
+	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
 	       a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
 	       app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
 	FROM job_applications app
@@ -582,7 +701,7 @@ func (db *DB) GetApplications() ([]JobWithAnalysis, error) {
 		var updatedAt time.Time

 		err := rows.Scan(
-			&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
+			&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
 			&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
 			&appID, &appStatus, &notes, &appliedAt, &updatedAt,
 		)
@@ -182,3 +182,118 @@ func TestRemoveTargetFromYAML(t *testing.T) {
 	}
 }

+func TestTouchJobLastSeenAndMaintenance(t *testing.T) {
+	tempDir := t.TempDir()
+	dbPath := filepath.Join(tempDir, "test_maintenance.db")
+
+	database, err := InitDB(dbPath)
+	if err != nil {
+		t.Fatalf("failed to init db: %v", err)
+	}
+	defer func() { _ = database.Close() }()
+
+	// Insert jobs
+	job1, _, err := database.InsertRawJob("canon-1", nil, "Job 1", "Google", "Seattle", "https://google.com/1", "Software Engineer Go")
+	if err != nil || job1.ID == 0 {
+		t.Fatalf("failed to insert job 1: %v", err)
+	}
+	job2, _, err := database.InsertRawJob("canon-2", nil, "Job 2", "Apple", "Seattle", "https://apple.com/2", "Software Engineer Swift")
+	if err != nil {
+		t.Fatalf("failed to insert job 2: %v", err)
+	}
+	job3, _, err := database.InsertRawJob("canon-3", nil, "Job 3", "Meta", "Seattle", "https://meta.com/3", "Software Engineer React")
+	if err != nil {
+		t.Fatalf("failed to insert job 3: %v", err)
+	}
+
+	// 1. Touch last seen for canon-1
+	if err := database.TouchJobLastSeen("canon-1"); err != nil {
+		t.Fatalf("TouchJobLastSeen failed: %v", err)
+	}
+
+	// Make job2 and job3 artificially stale (20 days and 40 days old)
+	_, err = database.Exec(`UPDATE raw_jobs SET last_seen_at = datetime('now', '-20 days') WHERE id = ?`, job2.ID)
+	if err != nil {
+		t.Fatalf("failed to update last_seen_at: %v", err)
+	}
+	_, err = database.Exec(`UPDATE raw_jobs SET last_seen_at = datetime('now', '-40 days') WHERE id = ?`, job3.ID)
+	if err != nil {
+		t.Fatalf("failed to update last_seen_at: %v", err)
+	}
+
+	// Protect job3 in active pipeline (APPLIED)
+	_, _ = database.CreateOrUpdateApplication(job3.ID, "APPLIED", "Applied to Meta")
+
+	// 2. Enqueue expire tasks (> 14 days) with pipeline protection
+	queuedExpireCount, err := database.EnqueueExpireTasks(14, true)
+	if err != nil {
+		t.Fatalf("EnqueueExpireTasks failed: %v", err)
+	}
+	// job2 is stale and unprotected -> should be queued
+	// job3 is stale but protected (APPLIED) -> should NOT be queued
+	// job1 is fresh -> should NOT be queued
+	if queuedExpireCount != 1 {
+		t.Errorf("expected 1 job queued for expire, got %d", queuedExpireCount)
+	}
+
+	// 3. Directly expire job2
+	if err := database.ExpireJob(job2.ID, "Test expiration"); err != nil {
+		t.Fatalf("ExpireJob failed: %v", err)
+	}
+
+	// Verify job2 status in DB
+	j2Detail, err := database.GetJobByID(job2.ID)
+	if err != nil {
+		t.Fatalf("GetJobByID failed: %v", err)
+	}
+	if j2Detail.App == nil || j2Detail.App.Status != "EXPIRED" {
+		t.Errorf("expected job2 to have EXPIRED application, got %v", j2Detail.App)
+	}
+
+	// Verify job3 still APPLIED
+	j3Detail, err := database.GetJobByID(job3.ID)
+	if err != nil {
+		t.Fatalf("GetJobByID failed: %v", err)
+	}
+	if j3Detail.App == nil || j3Detail.App.Status != "APPLIED" {
+		t.Errorf("expected job3 to remain APPLIED, got %v", j3Detail.App)
+	}
+
+	// 4. Stats check
+	stats, err := database.GetJobMaintenanceStats()
+	if err != nil {
+		t.Fatalf("GetJobMaintenanceStats failed: %v", err)
+	}
+	if stats["total_jobs"] != 3 {
+		t.Errorf("expected total_jobs=3, got %d", stats["total_jobs"])
+	}
+	if stats["active_7d"] != 1 {
+		t.Errorf("expected active_7d=1, got %d", stats["active_7d"])
+	}
+	if stats["stale_7d"] != 2 {
+		t.Errorf("expected stale_7d=2, got %d", stats["stale_7d"])
+	}
+	if stats["expired_count"] != 1 {
+		t.Errorf("expected expired_count=1, got %d", stats["expired_count"])
+	}
+}
+
+func TestDevDBMigration(t *testing.T) {
+	devDBPath := "../../data/db/dev/hireme.db"
+	if _, err := os.Stat(devDBPath); os.IsNotExist(err) {
+		return
+	}
+	database, err := InitDB(devDBPath)
+	if err != nil {
+		t.Fatalf("InitDB dev db failed: %v", err)
+	}
+	defer func() { _ = database.Close() }()
+
+	jobs, err := database.GetJobsWithAnalysis()
+	if err != nil {
+		t.Fatalf("GetJobsWithAnalysis on dev db failed: %v", err)
+	}
+	t.Logf("Successfully verified %d jobs from dev db", len(jobs))
+}
+
+