769 lines
25 KiB
Go
769 lines
25 KiB
Go
package db
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
"math"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Target Queries
|
|
func (db *DB) CreateTarget(name, targetURL, platform string, isActive bool) (*SearchTarget, error) {
|
|
query := `INSERT INTO search_targets (name, target_url, platform, is_active) VALUES (?, ?, ?, ?) RETURNING id, name, target_url, platform, is_active, last_crawled_at`
|
|
var target SearchTarget
|
|
var lastCrawled sql.NullTime
|
|
err := db.QueryRow(query, name, targetURL, platform, isActive).Scan(
|
|
&target.ID, &target.Name, &target.TargetURL, &target.Platform, &target.IsActive, &lastCrawled,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if lastCrawled.Valid {
|
|
target.LastCrawledAt = &lastCrawled.Time
|
|
}
|
|
return &target, nil
|
|
}
|
|
|
|
func (db *DB) GetActiveTargets() ([]SearchTarget, error) {
|
|
query := `SELECT id, name, target_url, platform, is_active, last_crawled_at FROM search_targets WHERE is_active = 1`
|
|
rows, err := db.Query(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { _ = rows.Close() }()
|
|
|
|
var targets []SearchTarget
|
|
for rows.Next() {
|
|
var t SearchTarget
|
|
var lastCrawled sql.NullTime
|
|
if err := rows.Scan(&t.ID, &t.Name, &t.TargetURL, &t.Platform, &t.IsActive, &lastCrawled); err != nil {
|
|
return nil, err
|
|
}
|
|
if lastCrawled.Valid {
|
|
t.LastCrawledAt = &lastCrawled.Time
|
|
}
|
|
targets = append(targets, t)
|
|
}
|
|
return targets, nil
|
|
}
|
|
|
|
func (db *DB) GetAllTargets() ([]SearchTarget, error) {
|
|
query := `SELECT id, name, target_url, platform, is_active, last_crawled_at FROM search_targets ORDER BY id DESC`
|
|
rows, err := db.Query(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { _ = rows.Close() }()
|
|
|
|
var targets []SearchTarget
|
|
for rows.Next() {
|
|
var t SearchTarget
|
|
var lastCrawled sql.NullTime
|
|
if err := rows.Scan(&t.ID, &t.Name, &t.TargetURL, &t.Platform, &t.IsActive, &lastCrawled); err != nil {
|
|
return nil, err
|
|
}
|
|
if lastCrawled.Valid {
|
|
t.LastCrawledAt = &lastCrawled.Time
|
|
}
|
|
targets = append(targets, t)
|
|
}
|
|
return targets, nil
|
|
}
|
|
|
|
func (db *DB) ToggleTargetStatus(id int64) error {
|
|
query := `UPDATE search_targets SET is_active = CASE WHEN is_active = 1 THEN 0 ELSE 1 END WHERE id = ?`
|
|
_, err := db.Exec(query, id)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) SetTargetActive(id int64, isActive bool) error {
|
|
query := `UPDATE search_targets SET is_active = ? WHERE id = ?`
|
|
_, err := db.Exec(query, isActive, id)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) UpdateTargetCrawledAt(id int64) error {
|
|
query := `UPDATE search_targets SET last_crawled_at = CURRENT_TIMESTAMP WHERE id = ?`
|
|
_, err := db.Exec(query, id)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) DeleteTarget(id int64) (*SearchTarget, int64, error) {
|
|
tx, err := db.Begin()
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
defer func() { _ = tx.Rollback() }()
|
|
|
|
// 1. Get target info before deleting
|
|
var target SearchTarget
|
|
var lastCrawled sql.NullTime
|
|
err = tx.QueryRow(`SELECT id, name, target_url, platform, is_active, last_crawled_at FROM search_targets WHERE id = ?`, id).Scan(
|
|
&target.ID, &target.Name, &target.TargetURL, &target.Platform, &target.IsActive, &lastCrawled,
|
|
)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
if lastCrawled.Valid {
|
|
target.LastCrawledAt = &lastCrawled.Time
|
|
}
|
|
|
|
// 2. Find all raw_jobs IDs associated with this target
|
|
rows, err := tx.Query(`SELECT id, canonical_id FROM raw_jobs WHERE target_id = ?`, id)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
var rawJobIDs []int64
|
|
var canonicalIDs []string
|
|
for rows.Next() {
|
|
var rID int64
|
|
var cID string
|
|
if err := rows.Scan(&rID, &cID); err != nil {
|
|
_ = rows.Close()
|
|
return nil, 0, err
|
|
}
|
|
rawJobIDs = append(rawJobIDs, rID)
|
|
canonicalIDs = append(canonicalIDs, cID)
|
|
}
|
|
_ = rows.Close()
|
|
|
|
// 3. Delete tasks referencing this target or its discovered jobs
|
|
targetIDPattern := fmt.Sprintf(`%%"target_id":%d%%`, id)
|
|
targetIDPatternStr := fmt.Sprintf(`%%"target_id": %d%%`, id)
|
|
_, _ = tx.Exec(`DELETE FROM tasks WHERE payload LIKE ? OR payload LIKE ?`, targetIDPattern, targetIDPatternStr)
|
|
|
|
for _, cid := range canonicalIDs {
|
|
cPattern := fmt.Sprintf(`%%"canonical_id":"%s"%%`, cid)
|
|
_, _ = tx.Exec(`DELETE FROM tasks WHERE payload LIKE ?`, cPattern)
|
|
}
|
|
for _, rID := range rawJobIDs {
|
|
rPattern := fmt.Sprintf(`%%"raw_job_id":%d%%`, rID)
|
|
rPatternStr := fmt.Sprintf(`%%"raw_job_id": %d%%`, rID)
|
|
_, _ = tx.Exec(`DELETE FROM tasks WHERE payload LIKE ? OR payload LIKE ?`, rPattern, rPatternStr)
|
|
}
|
|
|
|
// 4. Delete job_applications and job_analyses for these raw_jobs
|
|
for _, rID := range rawJobIDs {
|
|
_, _ = tx.Exec(`DELETE FROM job_applications WHERE raw_job_id = ?`, rID)
|
|
_, _ = tx.Exec(`DELETE FROM job_analyses WHERE raw_job_id = ?`, rID)
|
|
}
|
|
|
|
// 5. Delete raw_jobs for this target
|
|
res, err := tx.Exec(`DELETE FROM raw_jobs WHERE target_id = ?`, id)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
deletedJobsCount, _ := res.RowsAffected()
|
|
|
|
// 6. Delete search_target
|
|
_, err = tx.Exec(`DELETE FROM search_targets WHERE id = ?`, id)
|
|
if err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
if err := tx.Commit(); err != nil {
|
|
return nil, 0, err
|
|
}
|
|
|
|
return &target, deletedJobsCount, nil
|
|
}
|
|
|
|
// Raw Jobs Queries & Vector Similarity Deduplication
|
|
func (db *DB) InsertRawJob(canonicalID string, targetID *int64, title, company, location, applyURL, rawContent string) (*RawJob, bool, error) {
|
|
// Deduplication Check 1 & 2: Check canonical_id exist AND vector similarity >= 99%
|
|
existingJobs, err := db.getAllRawJobsForDeduplication()
|
|
if err == nil {
|
|
for _, existing := range existingJobs {
|
|
if existing.CanonicalID == canonicalID {
|
|
return nil, false, nil // Duplicate Canonical ID
|
|
}
|
|
|
|
// Vector Cosine Similarity check on raw_content (run only for substantial bodies >100 chars)
|
|
if len(existing.RawContent) >= 100 && len(rawContent) >= 100 {
|
|
similarity := CalculateCosineSimilarity(existing.RawContent, rawContent)
|
|
if similarity >= 0.99 {
|
|
return nil, false, nil // Content >= 99% vector match -> Deduplicate
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
query := `
|
|
INSERT INTO raw_jobs (canonical_id, target_id, title, company, location, apply_url, raw_content, crawled_at, last_seen_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
ON CONFLICT(canonical_id) DO NOTHING
|
|
RETURNING id, canonical_id, target_id, title, company, location, apply_url, raw_content, crawled_at, last_seen_at
|
|
`
|
|
var job RawJob
|
|
var tID sql.NullInt64
|
|
err = db.QueryRow(query, canonicalID, targetID, title, company, location, applyURL, rawContent).Scan(
|
|
&job.ID, &job.CanonicalID, &tID, &job.Title, &job.Company, &job.Location, &job.ApplyURL, &job.RawContent, &job.CrawledAt, &job.LastSeenAt,
|
|
)
|
|
if err == sql.ErrNoRows {
|
|
return nil, false, nil // Duplicate job
|
|
}
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
if tID.Valid {
|
|
job.TargetID = &tID.Int64
|
|
}
|
|
return &job, true, nil
|
|
}
|
|
|
|
func (db *DB) TouchJobLastSeen(canonicalID string) error {
|
|
query := `UPDATE raw_jobs SET last_seen_at = CURRENT_TIMESTAMP WHERE canonical_id = ?`
|
|
_, err := db.Exec(query, canonicalID)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) TouchJobLastSeenByID(rawJobID int64) error {
|
|
query := `UPDATE raw_jobs SET last_seen_at = CURRENT_TIMESTAMP WHERE id = ?`
|
|
_, err := db.Exec(query, rawJobID)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) ExpireJob(rawJobID int64, reason string) error {
|
|
if reason == "" {
|
|
reason = "Marked expired"
|
|
}
|
|
query := `
|
|
INSERT INTO job_applications (raw_job_id, status, notes, updated_at)
|
|
VALUES (?, 'EXPIRED', ?, CURRENT_TIMESTAMP)
|
|
ON CONFLICT(raw_job_id) DO UPDATE SET
|
|
status='EXPIRED',
|
|
notes=COALESCE(job_applications.notes || ' | ', '') || excluded.notes,
|
|
updated_at=CURRENT_TIMESTAMP
|
|
`
|
|
_, err := db.Exec(query, rawJobID, reason)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) EnqueueExpireTasks(days int, preserveActivePipeline bool) (int, error) {
|
|
tx, err := db.Begin()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer func() { _ = tx.Rollback() }()
|
|
|
|
var query string
|
|
if preserveActivePipeline {
|
|
query = `
|
|
SELECT r.id FROM raw_jobs r
|
|
LEFT JOIN job_applications app ON r.id = app.raw_job_id
|
|
WHERE r.last_seen_at < datetime('now', '-' || ? || ' days')
|
|
AND (app.status IS NULL OR app.status NOT IN ('APPLIED', 'INTERVIEW', 'OFFER', 'EXPIRED', 'OPTED_OUT'))
|
|
`
|
|
} else {
|
|
query = `
|
|
SELECT r.id FROM raw_jobs r
|
|
LEFT JOIN job_applications app ON r.id = app.raw_job_id
|
|
WHERE r.last_seen_at < datetime('now', '-' || ? || ' days')
|
|
AND (app.status IS NULL OR app.status NOT IN ('EXPIRED', 'OPTED_OUT'))
|
|
`
|
|
}
|
|
|
|
rows, err := tx.Query(query, days)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
var rawJobIDs []int64
|
|
for rows.Next() {
|
|
var id int64
|
|
if err := rows.Scan(&id); err != nil {
|
|
_ = rows.Close()
|
|
return 0, err
|
|
}
|
|
rawJobIDs = append(rawJobIDs, id)
|
|
}
|
|
_ = rows.Close()
|
|
|
|
for _, id := range rawJobIDs {
|
|
payloadJSON := fmt.Sprintf(`{"raw_job_id":%d,"reason":"Direct expire: not seen on target for %d+ days"}`, id, days)
|
|
if _, err := tx.Exec(`INSERT INTO tasks (task_type, payload, status) VALUES ('EXPIRE_JOB', ?, 'PENDING')`, payloadJSON); err != nil {
|
|
return 0, err
|
|
}
|
|
}
|
|
|
|
return len(rawJobIDs), tx.Commit()
|
|
}
|
|
|
|
func (db *DB) GetJobMaintenanceStats() (map[string]int, error) {
|
|
stats := make(map[string]int)
|
|
|
|
// Total raw jobs
|
|
var totalJobs int
|
|
_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs`).Scan(&totalJobs)
|
|
stats["total_jobs"] = totalJobs
|
|
|
|
// Active <= 7 days
|
|
var active7d int
|
|
_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs WHERE last_seen_at >= datetime('now', '-7 days')`).Scan(&active7d)
|
|
stats["active_7d"] = active7d
|
|
|
|
// Stale > 7 days
|
|
var stale7d int
|
|
_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs WHERE last_seen_at < datetime('now', '-7 days')`).Scan(&stale7d)
|
|
stats["stale_7d"] = stale7d
|
|
|
|
// Stale > 30 days
|
|
var stale30d int
|
|
_ = db.QueryRow(`SELECT COUNT(*) FROM raw_jobs WHERE last_seen_at < datetime('now', '-30 days')`).Scan(&stale30d)
|
|
stats["stale_30d"] = stale30d
|
|
|
|
// Expired count
|
|
var expiredCount int
|
|
_ = db.QueryRow(`SELECT COUNT(*) FROM job_applications WHERE status = 'EXPIRED'`).Scan(&expiredCount)
|
|
stats["expired_count"] = expiredCount
|
|
|
|
// Opted out count
|
|
var optedOutCount int
|
|
_ = db.QueryRow(`SELECT COUNT(*) FROM job_applications WHERE status = 'OPTED_OUT'`).Scan(&optedOutCount)
|
|
stats["opted_out_count"] = optedOutCount
|
|
|
|
// In pipeline count
|
|
var inPipeline int
|
|
_ = db.QueryRow(`SELECT COUNT(*) FROM job_applications WHERE status IN ('SAVED', 'APPLIED', 'INTERVIEW', 'OFFER')`).Scan(&inPipeline)
|
|
stats["in_pipeline"] = inPipeline
|
|
|
|
return stats, nil
|
|
}
|
|
|
|
func (db *DB) CanonicalIDExists(canonicalID string) (bool, error) {
|
|
query := `SELECT EXISTS(SELECT 1 FROM raw_jobs WHERE canonical_id = ?)`
|
|
var exists bool
|
|
err := db.QueryRow(query, canonicalID).Scan(&exists)
|
|
return exists, err
|
|
}
|
|
|
|
func (db *DB) getAllRawJobsForDeduplication() ([]RawJob, error) {
|
|
query := `SELECT id, canonical_id, raw_content FROM raw_jobs`
|
|
rows, err := db.Query(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { _ = rows.Close() }()
|
|
|
|
var jobs []RawJob
|
|
for rows.Next() {
|
|
var j RawJob
|
|
if err := rows.Scan(&j.ID, &j.CanonicalID, &j.RawContent); err != nil {
|
|
return nil, err
|
|
}
|
|
jobs = append(jobs, j)
|
|
}
|
|
return jobs, nil
|
|
}
|
|
|
|
// CalculateCosineSimilarity tokenizes text into term frequency vectors and returns cosine similarity in range [0.0, 1.0]
|
|
func CalculateCosineSimilarity(text1, text2 string) float64 {
|
|
vec1 := tokenizeToVector(text1)
|
|
vec2 := tokenizeToVector(text2)
|
|
|
|
if len(vec1) == 0 || len(vec2) == 0 {
|
|
return 0.0
|
|
}
|
|
|
|
var dotProduct float64
|
|
for term, freq1 := range vec1 {
|
|
if freq2, exists := vec2[term]; exists {
|
|
dotProduct += float64(freq1 * freq2)
|
|
}
|
|
}
|
|
|
|
var norm1 float64
|
|
for _, freq := range vec1 {
|
|
norm1 += float64(freq * freq)
|
|
}
|
|
norm1 = math.Sqrt(norm1)
|
|
|
|
var norm2 float64
|
|
for _, freq := range vec2 {
|
|
norm2 += float64(freq * freq)
|
|
}
|
|
norm2 = math.Sqrt(norm2)
|
|
|
|
if norm1 == 0 || norm2 == 0 {
|
|
return 0.0
|
|
}
|
|
|
|
return dotProduct / (norm1 * norm2)
|
|
}
|
|
|
|
var wordRegex = regexp.MustCompile(`[a-zA-Z0-9]+`)
|
|
|
|
var stopWords = map[string]bool{
|
|
"the": true, "and": true, "for": true, "with": true, "this": true,
|
|
"that": true, "from": true, "are": true, "was": true, "were": true,
|
|
"google": true, "position": true, "apply": true, "url": true, "location": true,
|
|
"seattle": true, "usa": true, "job": true, "http": true, "https": true,
|
|
}
|
|
|
|
func tokenizeToVector(text string) map[string]int {
|
|
words := wordRegex.FindAllString(strings.ToLower(text), -1)
|
|
freqMap := make(map[string]int)
|
|
for _, w := range words {
|
|
if len(w) > 2 && !stopWords[w] {
|
|
freqMap[w]++
|
|
}
|
|
}
|
|
return freqMap
|
|
}
|
|
|
|
// Job Analyses Queries
|
|
func (db *DB) CreateJobAnalysis(rawJobID int64, filterScore, score float64, isQualified bool, status string) error {
|
|
query := `
|
|
INSERT INTO job_analyses (raw_job_id, filter_score, score, is_qualified, status)
|
|
VALUES (?, ?, ?, ?, ?)
|
|
ON CONFLICT(raw_job_id) DO UPDATE SET filter_score=excluded.filter_score, score=excluded.score, is_qualified=excluded.is_qualified, status=excluded.status
|
|
`
|
|
_, err := db.Exec(query, rawJobID, filterScore, score, isQualified, status)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) CreateJobAnalysisFull(rawJobID int64, filterScore, score float64, isQualified bool, summary, matched, missing, tailoring, status string) error {
|
|
query := `
|
|
INSERT INTO job_analyses (raw_job_id, filter_score, score, is_qualified, summary, matched_highlights, missing_highlights, tailoring_suggestions, status, analyzed_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP)
|
|
ON CONFLICT(raw_job_id) DO UPDATE SET
|
|
filter_score=excluded.filter_score,
|
|
score=excluded.score,
|
|
is_qualified=excluded.is_qualified,
|
|
summary=excluded.summary,
|
|
matched_highlights=excluded.matched_highlights,
|
|
missing_highlights=excluded.missing_highlights,
|
|
tailoring_suggestions=excluded.tailoring_suggestions,
|
|
status=excluded.status,
|
|
analyzed_at=CURRENT_TIMESTAMP
|
|
`
|
|
_, err := db.Exec(query, rawJobID, filterScore, score, isQualified, summary, matched, missing, tailoring, status)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) UpdateJobAnalysisFull(rawJobID int64, filterScore, score float64, isQualified bool, summary, matchedJSON, missingJSON, suggestionsJSON, status string) error {
|
|
query := `
|
|
UPDATE job_analyses
|
|
SET filter_score = ?, score = ?, is_qualified = ?, summary = ?, matched_highlights = ?, missing_highlights = ?, tailoring_suggestions = ?, status = ?, analyzed_at = CURRENT_TIMESTAMP
|
|
WHERE raw_job_id = ?
|
|
`
|
|
_, err := db.Exec(query, filterScore, score, isQualified, summary, matchedJSON, missingJSON, suggestionsJSON, status, rawJobID)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) UpdateJobAnalysisLLM(rawJobID int64, aiScore float64, isQualified bool, summary, matchedJSON, missingJSON, suggestionsJSON string) error {
|
|
query := `
|
|
UPDATE job_analyses
|
|
SET score = ?, is_qualified = ?, summary = ?, matched_highlights = ?, missing_highlights = ?, tailoring_suggestions = ?, status = 'PARSED', analyzed_at = CURRENT_TIMESTAMP
|
|
WHERE raw_job_id = ?
|
|
`
|
|
_, err := db.Exec(query, aiScore, isQualified, summary, matchedJSON, missingJSON, suggestionsJSON, rawJobID)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
|
|
query := `
|
|
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
|
|
a.id, COALESCE(a.filter_score, 0), COALESCE(a.score, 0), a.is_qualified, COALESCE(a.summary, ''), COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), a.status, a.analyzed_at
|
|
FROM raw_jobs r
|
|
JOIN job_analyses a ON r.id = a.raw_job_id
|
|
WHERE a.is_qualified = 1 AND a.status = 'PENDING'
|
|
`
|
|
rows, err := db.Query(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { _ = rows.Close() }()
|
|
|
|
var results []JobWithAnalysis
|
|
for rows.Next() {
|
|
var item JobWithAnalysis
|
|
var tID sql.NullInt64
|
|
var aID int64
|
|
var filterScore, score float64
|
|
var isQualified bool
|
|
var summary, matched, missing, suggestions, aStatus string
|
|
var analyzedAt sql.NullTime
|
|
|
|
err := rows.Scan(
|
|
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
|
|
&aID, &filterScore, &score, &isQualified, &summary, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if tID.Valid {
|
|
item.TargetID = &tID.Int64
|
|
}
|
|
analysis := &JobAnalysis{
|
|
ID: aID,
|
|
RawJobID: item.ID,
|
|
FilterScore: filterScore,
|
|
Score: score,
|
|
IsQualified: isQualified,
|
|
Summary: summary,
|
|
MatchedHighlights: matched,
|
|
MissingHighlights: missing,
|
|
TailoringSuggestions: suggestions,
|
|
Status: aStatus,
|
|
}
|
|
if analyzedAt.Valid {
|
|
analysis.AnalyzedAt = &analyzedAt.Time
|
|
}
|
|
item.Analysis = analysis
|
|
results = append(results, item)
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
|
|
query := `
|
|
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
|
|
a.id, COALESCE(a.filter_score, 0), COALESCE(a.score, 0), a.is_qualified, COALESCE(a.summary, ''), COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
|
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
|
FROM raw_jobs r
|
|
LEFT JOIN job_analyses a ON r.id = a.raw_job_id
|
|
LEFT JOIN job_applications app ON r.id = app.raw_job_id
|
|
ORDER BY COALESCE(a.score, a.filter_score, 0) DESC, r.crawled_at DESC
|
|
`
|
|
rows, err := db.Query(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { _ = rows.Close() }()
|
|
|
|
var results []JobWithAnalysis
|
|
for rows.Next() {
|
|
var item JobWithAnalysis
|
|
var tID sql.NullInt64
|
|
var aID, appID sql.NullInt64
|
|
var filterScore, score sql.NullFloat64
|
|
var isQualified sql.NullBool
|
|
var summary, matched, missing, suggestions, aStatus sql.NullString
|
|
var analyzedAt sql.NullTime
|
|
var appStatus, notes sql.NullString
|
|
var appliedAt sql.NullTime
|
|
var updatedAt sql.NullTime
|
|
|
|
err := rows.Scan(
|
|
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
|
|
&aID, &filterScore, &score, &isQualified, &summary, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
|
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if tID.Valid {
|
|
item.TargetID = &tID.Int64
|
|
}
|
|
if aID.Valid {
|
|
analysis := &JobAnalysis{
|
|
ID: aID.Int64,
|
|
RawJobID: item.ID,
|
|
FilterScore: filterScore.Float64,
|
|
Score: score.Float64,
|
|
IsQualified: isQualified.Bool,
|
|
Summary: summary.String,
|
|
MatchedHighlights: matched.String,
|
|
MissingHighlights: missing.String,
|
|
TailoringSuggestions: suggestions.String,
|
|
Status: aStatus.String,
|
|
}
|
|
if analyzedAt.Valid {
|
|
analysis.AnalyzedAt = &analyzedAt.Time
|
|
}
|
|
item.Analysis = analysis
|
|
}
|
|
if appID.Valid {
|
|
app := &JobApplication{
|
|
ID: appID.Int64,
|
|
RawJobID: item.ID,
|
|
Status: appStatus.String,
|
|
Notes: notes.String,
|
|
UpdatedAt: updatedAt.Time,
|
|
}
|
|
if appliedAt.Valid {
|
|
app.AppliedAt = &appliedAt.Time
|
|
}
|
|
item.App = app
|
|
}
|
|
results = append(results, item)
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
func (db *DB) GetJobByID(rawJobID int64) (*JobWithAnalysis, error) {
|
|
query := `
|
|
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
|
|
a.id, COALESCE(a.filter_score, 0), COALESCE(a.score, 0), a.is_qualified, COALESCE(a.summary, ''), COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
|
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
|
FROM raw_jobs r
|
|
LEFT JOIN job_analyses a ON r.id = a.raw_job_id
|
|
LEFT JOIN job_applications app ON r.id = app.raw_job_id
|
|
WHERE r.id = ?
|
|
`
|
|
row := db.QueryRow(query, rawJobID)
|
|
var item JobWithAnalysis
|
|
var tID sql.NullInt64
|
|
var aID, appID sql.NullInt64
|
|
var filterScore, score sql.NullFloat64
|
|
var isQualified sql.NullBool
|
|
var summary, matched, missing, suggestions, aStatus sql.NullString
|
|
var analyzedAt sql.NullTime
|
|
var appStatus, notes sql.NullString
|
|
var appliedAt sql.NullTime
|
|
var updatedAt sql.NullTime
|
|
|
|
err := row.Scan(
|
|
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
|
|
&aID, &filterScore, &score, &isQualified, &summary, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
|
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if tID.Valid {
|
|
item.TargetID = &tID.Int64
|
|
}
|
|
if aID.Valid {
|
|
analysis := &JobAnalysis{
|
|
ID: aID.Int64,
|
|
RawJobID: item.ID,
|
|
FilterScore: filterScore.Float64,
|
|
Score: score.Float64,
|
|
IsQualified: isQualified.Bool,
|
|
Summary: summary.String,
|
|
MatchedHighlights: matched.String,
|
|
MissingHighlights: missing.String,
|
|
TailoringSuggestions: suggestions.String,
|
|
Status: aStatus.String,
|
|
}
|
|
if analyzedAt.Valid {
|
|
analysis.AnalyzedAt = &analyzedAt.Time
|
|
}
|
|
item.Analysis = analysis
|
|
}
|
|
if appID.Valid {
|
|
app := &JobApplication{
|
|
ID: appID.Int64,
|
|
RawJobID: item.ID,
|
|
Status: appStatus.String,
|
|
Notes: notes.String,
|
|
UpdatedAt: updatedAt.Time,
|
|
}
|
|
if appliedAt.Valid {
|
|
app.AppliedAt = &appliedAt.Time
|
|
}
|
|
item.App = app
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
// Applications Queries
|
|
func (db *DB) CreateOrUpdateApplication(rawJobID int64, status, notes string) (*JobApplication, error) {
|
|
now := time.Now()
|
|
var appliedAt *time.Time
|
|
if status == "APPLIED" {
|
|
appliedAt = &now
|
|
}
|
|
|
|
query := `
|
|
INSERT INTO job_applications (raw_job_id, status, notes, applied_at, updated_at)
|
|
VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
|
|
ON CONFLICT(raw_job_id) DO UPDATE SET
|
|
status=excluded.status,
|
|
notes=COALESCE(excluded.notes, job_applications.notes),
|
|
applied_at=COALESCE(excluded.applied_at, job_applications.applied_at),
|
|
updated_at=CURRENT_TIMESTAMP
|
|
RETURNING id, raw_job_id, status, notes, applied_at, updated_at
|
|
`
|
|
var app JobApplication
|
|
var appAt sql.NullTime
|
|
err := db.QueryRow(query, rawJobID, status, notes, appliedAt).Scan(
|
|
&app.ID, &app.RawJobID, &app.Status, &app.Notes, &appAt, &app.UpdatedAt,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if appAt.Valid {
|
|
app.AppliedAt = &appAt.Time
|
|
}
|
|
return &app, nil
|
|
}
|
|
|
|
func (db *DB) GetApplications() ([]JobWithAnalysis, error) {
|
|
query := `
|
|
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.crawled_at, r.last_seen_at,
|
|
a.id, COALESCE(a.filter_score, 0), COALESCE(a.score, 0), a.is_qualified, COALESCE(a.summary, ''), COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
|
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
|
FROM job_applications app
|
|
JOIN raw_jobs r ON app.raw_job_id = r.id
|
|
LEFT JOIN job_analyses a ON r.id = a.raw_job_id
|
|
ORDER BY app.updated_at DESC
|
|
`
|
|
rows, err := db.Query(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer func() { _ = rows.Close() }()
|
|
|
|
var results []JobWithAnalysis
|
|
for rows.Next() {
|
|
var item JobWithAnalysis
|
|
var tID sql.NullInt64
|
|
var aID, appID sql.NullInt64
|
|
var filterScore, score sql.NullFloat64
|
|
var isQualified sql.NullBool
|
|
var summary, matched, missing, suggestions, aStatus sql.NullString
|
|
var analyzedAt sql.NullTime
|
|
var appStatus, notes sql.NullString
|
|
var appliedAt sql.NullTime
|
|
var updatedAt time.Time
|
|
|
|
err := rows.Scan(
|
|
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.CrawledAt, &item.LastSeenAt,
|
|
&aID, &filterScore, &score, &isQualified, &summary, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
|
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if tID.Valid {
|
|
item.TargetID = &tID.Int64
|
|
}
|
|
if aID.Valid {
|
|
analysis := &JobAnalysis{
|
|
ID: aID.Int64,
|
|
RawJobID: item.ID,
|
|
FilterScore: filterScore.Float64,
|
|
Score: score.Float64,
|
|
IsQualified: isQualified.Bool,
|
|
Summary: summary.String,
|
|
MatchedHighlights: matched.String,
|
|
MissingHighlights: missing.String,
|
|
TailoringSuggestions: suggestions.String,
|
|
Status: aStatus.String,
|
|
}
|
|
if analyzedAt.Valid {
|
|
analysis.AnalyzedAt = &analyzedAt.Time
|
|
}
|
|
item.Analysis = analysis
|
|
}
|
|
app := &JobApplication{
|
|
ID: appID.Int64,
|
|
RawJobID: item.ID,
|
|
Status: appStatus.String,
|
|
Notes: notes.String,
|
|
UpdatedAt: updatedAt,
|
|
}
|
|
if appliedAt.Valid {
|
|
app.AppliedAt = &appliedAt.Time
|
|
}
|
|
item.App = app
|
|
results = append(results, item)
|
|
}
|
|
return results, nil
|
|
}
|