feat: add job maintenance dashboard and automated stale job expiration system

2026-08-22 01:51:43 -07:00
parent 367929e7d0
commit 7ec91f384f
12 changed files with 633 additions and 56 deletions
@@ -185,15 +185,15 @@ func (db *DB) InsertRawJob(canonicalID string, targetID *int64, title, company,
 	}

 	query := `
-	INSERT INTO raw_jobs (canonical_id, target_id, title, company, location, apply_url, raw_content)
-	VALUES (?, ?, ?, ?, ?, ?, ?)
+	INSERT INTO raw_jobs (canonical_id, target_id, title, company, location, apply_url, raw_content, crawled_at, last_seen_at)
+	VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
 	ON CONFLICT(canonical_id) DO NOTHING
-	RETURNING id, canonical_id, target_id, title, company, location, apply_url, raw_content, crawled_at
+	RETURNING id, canonical_id, target_id, title, company, location, apply_url, raw_content, crawled_at, last_seen_at
 	`
 	var job RawJob
 	var tID sql.NullInt64
 	err = db.QueryRow(query, canonicalID, targetID, title, company, location, applyURL, rawContent).Scan(
-		&job.ID, &job.CanonicalID, &tID, &job.Title, &job.Company, &job.Location, &job.ApplyURL, &job.RawContent, &job.CrawledAt,
+		&job.ID, &job.CanonicalID, &tID, &job.Title, &job.Company, &job.Location, &job.ApplyURL, &job.RawContent, &job.CrawledAt, &job.LastSeenAt,
 	)
 	if err == sql.ErrNoRows {
 		return nil, false, nil // Duplicate job
@@ -207,6 +207,125 @@ func (db *DB) InsertRawJob(canonicalID string, targetID *int64, title, company,
 	return &job, true, nil
 }

+func (db *DB) TouchJobLastSeen(canonicalID string) error {
+	query := `UPDATE raw_jobs SET last_seen_at = CURRENT_TIMESTAMP WHERE canonical_id = ?`
+	_, err := db.Exec(query, canonicalID)
+	return err
+}
+
+func (db *DB) TouchJobLastSeenByID(rawJobID int64) error {
+	query := `UPDATE raw_jobs SET last_seen_at = CURRENT_TIMESTAMP WHERE id = ?`
+	_, err := db.Exec(query, rawJobID)
+	return err
+}
+
+func (db *DB) ExpireJob(rawJobID int64, reason string) error {
+	if reason == "" {
+		reason = "Marked expired"
+	}
+	query := `
+	INSERT INTO job_applications (raw_job_id, status, notes, updated_at)
+	VALUES (?, 'EXPIRED', ?, CURRENT_TIMESTAMP)
+	ON CONFLICT(raw_job_id) DO UPDATE SET 
+		status='EXPIRED',
+		notes=COALESCE(job_applications.notes || ' | ', '') || excluded.notes,
+		updated_at=CURRENT_TIMESTAMP
+	`
+	_, err := db.Exec(query, rawJobID, reason)
+	return err
+}
+
+func (db *DB) EnqueueExpireTasks(days int, preserveActivePipeline bool) (int, error) {
+	tx, err := db.Begin()
+	if err != nil {
+		return 0, err
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	var query string
+	if preserveActivePipeline {
+		query = `
+		SELECT r.id FROM raw_jobs r
+		LEFT JOIN job_applications app ON r.id = app.raw_job_id
+		WHERE r.last_seen_at < datetime('now', '-' || ? || ' days')
+		  AND (app.status IS NULL OR app.status NOT IN ('APPLIED', 'INTERVIEW', 'OFFER', 'EXPIRED', 'OPTED_OUT'))
+		`
+	} else {
+		query = `
+		SELECT r.id FROM raw_jobs r
+		LEFT JOIN job_applications app ON r.id = app.raw_job_id
+		WHERE r.last_seen_at < datetime('now', '-' || ? || ' days')
+		  AND (app.status IS NULL OR app.status NOT IN ('EXPIRED', 'OPTED_OUT'))
+		`
+	}
+
+	rows, err := tx.Query(query, days)
+	if err != nil {
+		return 0, err
+	}
+
+	var rawJobIDs []int64
+	for rows.Next() {
+		var id int64
+		if err := rows.Scan(&id); err != nil {
+			_ = rows.Close()
+			return 0, err
+		}
+		rawJobIDs = append(rawJobIDs, id)
+	}
+	_ = rows.Close()
+
+	for _, id := range rawJobIDs {
+		payloadJSON := fmt.Sprintf(`{"raw_job_id":%d,"reason":"Direct expire: not seen on target for %d+ days"}`, id, days)
+		if _, err := tx.Exec(`INSERT INTO tasks (task_type, payload, status) VALUES ('EXPIRE_JOB', ?, 'PENDING')`, payloadJSON); err != nil {
+			return 0, err
+		}
+	}
+
+	return len(rawJobIDs), tx.Commit()
+}
+
+func (db *DB) GetJobMaintenanceStats() (map[string]int, error) {
+	stats := make(map[string]int)
+
+	// Total raw jobs
+	var totalJobs int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs`).Scan(&totalJobs)
+	stats["total_jobs"] = totalJobs
+
+	// Active <= 7 days
+	var active7d int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs WHERE last_seen_at >= datetime('now', '-7 days')`).Scan(&active7d)
+	stats["active_7d"] = active7d
+
+	// Stale > 7 days
+	var stale7d int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs WHERE last_seen_at < datetime('now', '-7 days')`).Scan(&stale7d)
+	stats["stale_7d"] = stale7d
+
+	// Stale > 30 days
+	var stale30d int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs WHERE last_seen_at < datetime('now', '-30 days')`).Scan(&stale30d)
+	stats["stale_30d"] = stale30d
+
+	// Expired count
+	var expiredCount int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM job_applications WHERE status = 'EXPIRED'`).Scan(&expiredCount)
+	stats["expired_count"] = expiredCount
+
+	// Opted out count
+	var optedOutCount int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM job_applications WHERE status = 'OPTED_OUT'`).Scan(&optedOutCount)
+	stats["opted_out_count"] = optedOutCount
+
+	// In pipeline count
+	var inPipeline int
+	_ = db.QueryRow(`SELECT COUNT(*) FROM job_applications WHERE status IN ('SAVED', 'APPLIED', 'INTERVIEW', 'OFFER')`).Scan(&inPipeline)
+	stats["in_pipeline"] = inPipeline
+
+	return stats, nil
+}
+
 func (db *DB) CanonicalIDExists(canonicalID string) (bool, error) {
 	query := `SELECT EXISTS(SELECT 1 FROM raw_jobs WHERE canonical_id = ?)`
 	var exists bool
@@ -330,7 +449,7 @@ func (db *DB) UpdateJobAnalysisLLM(rawJobID int64, matchedJSON, missingJSON, sug

 func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
 	query := `
-	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
+	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
 	       a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), a.status, a.analyzed_at
 	FROM raw_jobs r
 	JOIN job_analyses a ON r.id = a.raw_job_id
@@ -353,7 +472,7 @@ func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
 		var analyzedAt sql.NullTime

 		err := rows.Scan(
-			&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
+			&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
 			&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
 		)
 		if err != nil {
@@ -383,7 +502,7 @@ func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {

 func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
 	query := `
-	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
+	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
 	       a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
 	       app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
 	FROM raw_jobs r
@@ -411,7 +530,7 @@ func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
 		var updatedAt sql.NullTime

 		err := rows.Scan(
-			&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
+			&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
 			&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
 			&appID, &appStatus, &notes, &appliedAt, &updatedAt,
 		)
@@ -457,7 +576,7 @@ func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {

 func (db *DB) GetJobByID(rawJobID int64) (*JobWithAnalysis, error) {
 	query := `
-	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
+	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
 	       a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
 	       app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
 	FROM raw_jobs r
@@ -478,7 +597,7 @@ func (db *DB) GetJobByID(rawJobID int64) (*JobWithAnalysis, error) {
 	var updatedAt sql.NullTime

 	err := row.Scan(
-		&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
+		&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
 		&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
 		&appID, &appStatus, &notes, &appliedAt, &updatedAt,
 	)
@@ -554,7 +673,7 @@ func (db *DB) CreateOrUpdateApplication(rawJobID int64, status, notes string) (*

 func (db *DB) GetApplications() ([]JobWithAnalysis, error) {
 	query := `
-	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at,
+	SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
 	       a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
 	       app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
 	FROM job_applications app
@@ -582,7 +701,7 @@ func (db *DB) GetApplications() ([]JobWithAnalysis, error) {
 		var updatedAt time.Time

 		err := rows.Scan(
-			&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt,
+			&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
 			&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
 			&appID, &appStatus, &notes, &appliedAt, &updatedAt,
 		)