package db import ( "database/sql" "math" "regexp" "strings" "time" ) // Target Queries func (db *DB) CreateTarget(name, targetURL, platform string) (*SearchTarget, error) { query := `INSERT INTO search_targets (name, target_url, platform, is_active) VALUES (?, ?, ?, 1) RETURNING id, name, target_url, platform, is_active, last_crawled_at` var target SearchTarget var lastCrawled sql.NullTime err := db.QueryRow(query, name, targetURL, platform).Scan( &target.ID, &target.Name, &target.TargetURL, &target.Platform, &target.IsActive, &lastCrawled, ) if err != nil { return nil, err } if lastCrawled.Valid { target.LastCrawledAt = &lastCrawled.Time } return &target, nil } func (db *DB) GetActiveTargets() ([]SearchTarget, error) { query := `SELECT id, name, target_url, platform, is_active, last_crawled_at FROM search_targets WHERE is_active = 1` rows, err := db.Query(query) if err != nil { return nil, err } defer rows.Close() var targets []SearchTarget for rows.Next() { var t SearchTarget var lastCrawled sql.NullTime if err := rows.Scan(&t.ID, &t.Name, &t.TargetURL, &t.Platform, &t.IsActive, &lastCrawled); err != nil { return nil, err } if lastCrawled.Valid { t.LastCrawledAt = &lastCrawled.Time } targets = append(targets, t) } return targets, nil } func (db *DB) GetAllTargets() ([]SearchTarget, error) { query := `SELECT id, name, target_url, platform, is_active, last_crawled_at FROM search_targets ORDER BY id DESC` rows, err := db.Query(query) if err != nil { return nil, err } defer rows.Close() var targets []SearchTarget for rows.Next() { var t SearchTarget var lastCrawled sql.NullTime if err := rows.Scan(&t.ID, &t.Name, &t.TargetURL, &t.Platform, &t.IsActive, &lastCrawled); err != nil { return nil, err } if lastCrawled.Valid { t.LastCrawledAt = &lastCrawled.Time } targets = append(targets, t) } return targets, nil } func (db *DB) ToggleTargetStatus(id int64) error { query := `UPDATE search_targets SET is_active = CASE WHEN is_active = 1 THEN 0 ELSE 1 END WHERE id = ?` _, err := db.Exec(query, id) return err } func (db *DB) UpdateTargetCrawledAt(id int64) error { query := `UPDATE search_targets SET last_crawled_at = CURRENT_TIMESTAMP WHERE id = ?` _, err := db.Exec(query, id) return err } // Raw Jobs Queries & Vector Similarity Deduplication func (db *DB) InsertRawJob(canonicalID string, targetID *int64, title, company, location, applyURL, rawContent string) (*RawJob, bool, error) { // Deduplication Check 1 & 2: Check canonical_id exist AND vector similarity >= 99% existingJobs, err := db.getAllRawJobsForDeduplication() if err == nil { for _, existing := range existingJobs { if existing.CanonicalID == canonicalID { return nil, false, nil // Duplicate Canonical ID } // Vector Cosine Similarity check on raw_content (run only for substantial bodies >100 chars) if len(existing.RawContent) >= 100 && len(rawContent) >= 100 { similarity := CalculateCosineSimilarity(existing.RawContent, rawContent) if similarity >= 0.99 { return nil, false, nil // Content >= 99% vector match -> Deduplicate } } } } query := ` INSERT INTO raw_jobs (canonical_id, target_id, title, company, location, apply_url, raw_content, status) VALUES (?, ?, ?, ?, ?, ?, ?, 'PENDING') ON CONFLICT(canonical_id) DO NOTHING RETURNING id, canonical_id, target_id, title, company, location, apply_url, raw_content, status, crawled_at ` var job RawJob var tID sql.NullInt64 err = db.QueryRow(query, canonicalID, targetID, title, company, location, applyURL, rawContent).Scan( &job.ID, &job.CanonicalID, &tID, &job.Title, &job.Company, &job.Location, &job.ApplyURL, &job.RawContent, &job.Status, &job.CrawledAt, ) if err == sql.ErrNoRows { return nil, false, nil // Duplicate job } if err != nil { return nil, false, err } if tID.Valid { job.TargetID = &tID.Int64 } return &job, true, nil } func (db *DB) CanonicalIDExists(canonicalID string) (bool, error) { query := `SELECT EXISTS(SELECT 1 FROM raw_jobs WHERE canonical_id = ?)` var exists bool err := db.QueryRow(query, canonicalID).Scan(&exists) return exists, err } func (db *DB) getAllRawJobsForDeduplication() ([]RawJob, error) { query := `SELECT id, canonical_id, raw_content FROM raw_jobs` rows, err := db.Query(query) if err != nil { return nil, err } defer rows.Close() var jobs []RawJob for rows.Next() { var j RawJob if err := rows.Scan(&j.ID, &j.CanonicalID, &j.RawContent); err != nil { return nil, err } jobs = append(jobs, j) } return jobs, nil } // CalculateCosineSimilarity tokenizes text into term frequency vectors and returns cosine similarity in range [0.0, 1.0] func CalculateCosineSimilarity(text1, text2 string) float64 { vec1 := tokenizeToVector(text1) vec2 := tokenizeToVector(text2) if len(vec1) == 0 || len(vec2) == 0 { return 0.0 } var dotProduct float64 for term, freq1 := range vec1 { if freq2, exists := vec2[term]; exists { dotProduct += float64(freq1 * freq2) } } var norm1 float64 for _, freq := range vec1 { norm1 += float64(freq * freq) } norm1 = math.Sqrt(norm1) var norm2 float64 for _, freq := range vec2 { norm2 += float64(freq * freq) } norm2 = math.Sqrt(norm2) if norm1 == 0 || norm2 == 0 { return 0.0 } return dotProduct / (norm1 * norm2) } var wordRegex = regexp.MustCompile(`[a-zA-Z0-9]+`) var stopWords = map[string]bool{ "the": true, "and": true, "for": true, "with": true, "this": true, "that": true, "from": true, "are": true, "was": true, "were": true, "google": true, "position": true, "apply": true, "url": true, "location": true, "seattle": true, "usa": true, "job": true, "http": true, "https": true, } func tokenizeToVector(text string) map[string]int { words := wordRegex.FindAllString(strings.ToLower(text), -1) freqMap := make(map[string]int) for _, w := range words { if len(w) > 2 && !stopWords[w] { freqMap[w]++ } } return freqMap } func (db *DB) GetPendingRawJobs() ([]RawJob, error) { query := `SELECT id, canonical_id, target_id, title, company, location, apply_url, raw_content, status, crawled_at FROM raw_jobs WHERE status = 'PENDING'` rows, err := db.Query(query) if err != nil { return nil, err } defer rows.Close() var jobs []RawJob for rows.Next() { var j RawJob var tID sql.NullInt64 if err := rows.Scan(&j.ID, &j.CanonicalID, &tID, &j.Title, &j.Company, &j.Location, &j.ApplyURL, &j.RawContent, &j.Status, &j.CrawledAt); err != nil { return nil, err } if tID.Valid { j.TargetID = &tID.Int64 } jobs = append(jobs, j) } return jobs, nil } func (db *DB) UpdateRawJobStatus(id int64, status string) error { query := `UPDATE raw_jobs SET status = ? WHERE id = ?` _, err := db.Exec(query, status, id) return err } // Job Analyses Queries func (db *DB) CreateJobAnalysis(rawJobID int64, score float64, isQualified bool, status string) error { query := ` INSERT INTO job_analyses (raw_job_id, score, is_qualified, status) VALUES (?, ?, ?, ?) ON CONFLICT(raw_job_id) DO UPDATE SET score=excluded.score, is_qualified=excluded.is_qualified, status=excluded.status ` _, err := db.Exec(query, rawJobID, score, isQualified, status) return err } func (db *DB) UpdateJobAnalysisLLM(rawJobID int64, matchedJSON, missingJSON, suggestionsJSON string) error { query := ` UPDATE job_analyses SET matched_highlights = ?, missing_highlights = ?, tailoring_suggestions = ?, status = 'PARSED', analyzed_at = CURRENT_TIMESTAMP WHERE raw_job_id = ? ` _, err := db.Exec(query, matchedJSON, missingJSON, suggestionsJSON, rawJobID) return err } func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) { query := ` SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at, a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), a.status, a.analyzed_at FROM raw_jobs r JOIN job_analyses a ON r.id = a.raw_job_id WHERE a.is_qualified = 1 AND a.status = 'PENDING' ` rows, err := db.Query(query) if err != nil { return nil, err } defer rows.Close() var results []JobWithAnalysis for rows.Next() { var item JobWithAnalysis var tID sql.NullInt64 var aID int64 var score float64 var isQualified bool var matched, missing, suggestions, aStatus string var analyzedAt sql.NullTime err := rows.Scan( &item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt, &aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt, ) if err != nil { return nil, err } if tID.Valid { item.TargetID = &tID.Int64 } analysis := &JobAnalysis{ ID: aID, RawJobID: item.ID, Score: score, IsQualified: isQualified, MatchedHighlights: matched, MissingHighlights: missing, TailoringSuggestions: suggestions, Status: aStatus, } if analyzedAt.Valid { analysis.AnalyzedAt = &analyzedAt.Time } item.Analysis = analysis results = append(results, item) } return results, nil } func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) { query := ` SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at, a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at, app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at FROM raw_jobs r LEFT JOIN job_analyses a ON r.id = a.raw_job_id LEFT JOIN job_applications app ON r.id = app.raw_job_id ORDER BY COALESCE(a.score, 0) DESC, r.crawled_at DESC ` rows, err := db.Query(query) if err != nil { return nil, err } defer rows.Close() var results []JobWithAnalysis for rows.Next() { var item JobWithAnalysis var tID sql.NullInt64 var aID, appID sql.NullInt64 var score sql.NullFloat64 var isQualified sql.NullBool var matched, missing, suggestions, aStatus sql.NullString var analyzedAt sql.NullTime var appStatus, notes sql.NullString var appliedAt sql.NullTime var updatedAt sql.NullTime err := rows.Scan( &item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt, &aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt, &appID, &appStatus, ¬es, &appliedAt, &updatedAt, ) if err != nil { return nil, err } if tID.Valid { item.TargetID = &tID.Int64 } if aID.Valid { analysis := &JobAnalysis{ ID: aID.Int64, RawJobID: item.ID, Score: score.Float64, IsQualified: isQualified.Bool, MatchedHighlights: matched.String, MissingHighlights: missing.String, TailoringSuggestions: suggestions.String, Status: aStatus.String, } if analyzedAt.Valid { analysis.AnalyzedAt = &analyzedAt.Time } item.Analysis = analysis } if appID.Valid { app := &JobApplication{ ID: appID.Int64, RawJobID: item.ID, Status: appStatus.String, Notes: notes.String, UpdatedAt: updatedAt.Time, } if appliedAt.Valid { app.AppliedAt = &appliedAt.Time } item.App = app } results = append(results, item) } return results, nil } func (db *DB) GetJobByID(rawJobID int64) (*JobWithAnalysis, error) { query := ` SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at, a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at, app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at FROM raw_jobs r LEFT JOIN job_analyses a ON r.id = a.raw_job_id LEFT JOIN job_applications app ON r.id = app.raw_job_id WHERE r.id = ? ` row := db.QueryRow(query, rawJobID) var item JobWithAnalysis var tID sql.NullInt64 var aID, appID sql.NullInt64 var score sql.NullFloat64 var isQualified sql.NullBool var matched, missing, suggestions, aStatus sql.NullString var analyzedAt sql.NullTime var appStatus, notes sql.NullString var appliedAt sql.NullTime var updatedAt sql.NullTime err := row.Scan( &item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt, &aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt, &appID, &appStatus, ¬es, &appliedAt, &updatedAt, ) if err != nil { return nil, err } if tID.Valid { item.TargetID = &tID.Int64 } if aID.Valid { analysis := &JobAnalysis{ ID: aID.Int64, RawJobID: item.ID, Score: score.Float64, IsQualified: isQualified.Bool, MatchedHighlights: matched.String, MissingHighlights: missing.String, TailoringSuggestions: suggestions.String, Status: aStatus.String, } if analyzedAt.Valid { analysis.AnalyzedAt = &analyzedAt.Time } item.Analysis = analysis } if appID.Valid { app := &JobApplication{ ID: appID.Int64, RawJobID: item.ID, Status: appStatus.String, Notes: notes.String, UpdatedAt: updatedAt.Time, } if appliedAt.Valid { app.AppliedAt = &appliedAt.Time } item.App = app } return &item, nil } // Applications Queries func (db *DB) CreateOrUpdateApplication(rawJobID int64, status, notes string) (*JobApplication, error) { now := time.Now() var appliedAt *time.Time if status == "APPLIED" { appliedAt = &now } query := ` INSERT INTO job_applications (raw_job_id, status, notes, applied_at, updated_at) VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP) ON CONFLICT(raw_job_id) DO UPDATE SET status=excluded.status, notes=COALESCE(excluded.notes, job_applications.notes), applied_at=COALESCE(excluded.applied_at, job_applications.applied_at), updated_at=CURRENT_TIMESTAMP RETURNING id, raw_job_id, status, notes, applied_at, updated_at ` var app JobApplication var appAt sql.NullTime err := db.QueryRow(query, rawJobID, status, notes, appliedAt).Scan( &app.ID, &app.RawJobID, &app.Status, &app.Notes, &appAt, &app.UpdatedAt, ) if err != nil { return nil, err } if appAt.Valid { app.AppliedAt = &appAt.Time } return &app, nil } func (db *DB) GetApplications() ([]JobWithAnalysis, error) { query := ` SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at, a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at, app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at FROM job_applications app JOIN raw_jobs r ON app.raw_job_id = r.id LEFT JOIN job_analyses a ON r.id = a.raw_job_id ORDER BY app.updated_at DESC ` rows, err := db.Query(query) if err != nil { return nil, err } defer rows.Close() var results []JobWithAnalysis for rows.Next() { var item JobWithAnalysis var tID sql.NullInt64 var aID, appID sql.NullInt64 var score sql.NullFloat64 var isQualified sql.NullBool var matched, missing, suggestions, aStatus sql.NullString var analyzedAt sql.NullTime var appStatus, notes sql.NullString var appliedAt sql.NullTime var updatedAt time.Time err := rows.Scan( &item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt, &aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt, &appID, &appStatus, ¬es, &appliedAt, &updatedAt, ) if err != nil { return nil, err } if tID.Valid { item.TargetID = &tID.Int64 } if aID.Valid { analysis := &JobAnalysis{ ID: aID.Int64, RawJobID: item.ID, Score: score.Float64, IsQualified: isQualified.Bool, MatchedHighlights: matched.String, MissingHighlights: missing.String, TailoringSuggestions: suggestions.String, Status: aStatus.String, } if analyzedAt.Valid { analysis.AnalyzedAt = &analyzedAt.Time } item.Analysis = analysis } app := &JobApplication{ ID: appID.Int64, RawJobID: item.ID, Status: appStatus.String, Notes: notes.String, UpdatedAt: updatedAt, } if appliedAt.Valid { app.AppliedAt = &appliedAt.Time } item.App = app results = append(results, item) } return results, nil }