555 lines
17 KiB
Go
555 lines
17 KiB
Go
package db
|
|
|
|
import (
|
|
"database/sql"
|
|
"math"
|
|
"regexp"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Target Queries
|
|
func (db *DB) CreateTarget(name, targetURL, platform string) (*SearchTarget, error) {
|
|
query := `INSERT INTO search_targets (name, target_url, platform, is_active) VALUES (?, ?, ?, 1) RETURNING id, name, target_url, platform, is_active, last_crawled_at`
|
|
var target SearchTarget
|
|
var lastCrawled sql.NullTime
|
|
err := db.QueryRow(query, name, targetURL, platform).Scan(
|
|
&target.ID, &target.Name, &target.TargetURL, &target.Platform, &target.IsActive, &lastCrawled,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if lastCrawled.Valid {
|
|
target.LastCrawledAt = &lastCrawled.Time
|
|
}
|
|
return &target, nil
|
|
}
|
|
|
|
func (db *DB) GetActiveTargets() ([]SearchTarget, error) {
|
|
query := `SELECT id, name, target_url, platform, is_active, last_crawled_at FROM search_targets WHERE is_active = 1`
|
|
rows, err := db.Query(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var targets []SearchTarget
|
|
for rows.Next() {
|
|
var t SearchTarget
|
|
var lastCrawled sql.NullTime
|
|
if err := rows.Scan(&t.ID, &t.Name, &t.TargetURL, &t.Platform, &t.IsActive, &lastCrawled); err != nil {
|
|
return nil, err
|
|
}
|
|
if lastCrawled.Valid {
|
|
t.LastCrawledAt = &lastCrawled.Time
|
|
}
|
|
targets = append(targets, t)
|
|
}
|
|
return targets, nil
|
|
}
|
|
|
|
func (db *DB) GetAllTargets() ([]SearchTarget, error) {
|
|
query := `SELECT id, name, target_url, platform, is_active, last_crawled_at FROM search_targets ORDER BY id DESC`
|
|
rows, err := db.Query(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var targets []SearchTarget
|
|
for rows.Next() {
|
|
var t SearchTarget
|
|
var lastCrawled sql.NullTime
|
|
if err := rows.Scan(&t.ID, &t.Name, &t.TargetURL, &t.Platform, &t.IsActive, &lastCrawled); err != nil {
|
|
return nil, err
|
|
}
|
|
if lastCrawled.Valid {
|
|
t.LastCrawledAt = &lastCrawled.Time
|
|
}
|
|
targets = append(targets, t)
|
|
}
|
|
return targets, nil
|
|
}
|
|
|
|
func (db *DB) ToggleTargetStatus(id int64) error {
|
|
query := `UPDATE search_targets SET is_active = CASE WHEN is_active = 1 THEN 0 ELSE 1 END WHERE id = ?`
|
|
_, err := db.Exec(query, id)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) UpdateTargetCrawledAt(id int64) error {
|
|
query := `UPDATE search_targets SET last_crawled_at = CURRENT_TIMESTAMP WHERE id = ?`
|
|
_, err := db.Exec(query, id)
|
|
return err
|
|
}
|
|
|
|
// Raw Jobs Queries & Vector Similarity Deduplication
|
|
func (db *DB) InsertRawJob(canonicalID string, targetID *int64, title, company, location, applyURL, rawContent string) (*RawJob, bool, error) {
|
|
// Deduplication Check 1 & 2: Check canonical_id exist AND vector similarity >= 99%
|
|
existingJobs, err := db.getAllRawJobsForDeduplication()
|
|
if err == nil {
|
|
for _, existing := range existingJobs {
|
|
if existing.CanonicalID == canonicalID {
|
|
return nil, false, nil // Duplicate Canonical ID
|
|
}
|
|
|
|
// Vector Cosine Similarity check on raw_content (run only for substantial bodies >100 chars)
|
|
if len(existing.RawContent) >= 100 && len(rawContent) >= 100 {
|
|
similarity := CalculateCosineSimilarity(existing.RawContent, rawContent)
|
|
if similarity >= 0.99 {
|
|
return nil, false, nil // Content >= 99% vector match -> Deduplicate
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
query := `
|
|
INSERT INTO raw_jobs (canonical_id, target_id, title, company, location, apply_url, raw_content, status)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, 'PENDING')
|
|
ON CONFLICT(canonical_id) DO NOTHING
|
|
RETURNING id, canonical_id, target_id, title, company, location, apply_url, raw_content, status, crawled_at
|
|
`
|
|
var job RawJob
|
|
var tID sql.NullInt64
|
|
err = db.QueryRow(query, canonicalID, targetID, title, company, location, applyURL, rawContent).Scan(
|
|
&job.ID, &job.CanonicalID, &tID, &job.Title, &job.Company, &job.Location, &job.ApplyURL, &job.RawContent, &job.Status, &job.CrawledAt,
|
|
)
|
|
if err == sql.ErrNoRows {
|
|
return nil, false, nil // Duplicate job
|
|
}
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
if tID.Valid {
|
|
job.TargetID = &tID.Int64
|
|
}
|
|
return &job, true, nil
|
|
}
|
|
|
|
func (db *DB) CanonicalIDExists(canonicalID string) (bool, error) {
|
|
query := `SELECT EXISTS(SELECT 1 FROM raw_jobs WHERE canonical_id = ?)`
|
|
var exists bool
|
|
err := db.QueryRow(query, canonicalID).Scan(&exists)
|
|
return exists, err
|
|
}
|
|
|
|
func (db *DB) getAllRawJobsForDeduplication() ([]RawJob, error) {
|
|
query := `SELECT id, canonical_id, raw_content FROM raw_jobs`
|
|
rows, err := db.Query(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var jobs []RawJob
|
|
for rows.Next() {
|
|
var j RawJob
|
|
if err := rows.Scan(&j.ID, &j.CanonicalID, &j.RawContent); err != nil {
|
|
return nil, err
|
|
}
|
|
jobs = append(jobs, j)
|
|
}
|
|
return jobs, nil
|
|
}
|
|
|
|
// CalculateCosineSimilarity tokenizes text into term frequency vectors and returns cosine similarity in range [0.0, 1.0]
|
|
func CalculateCosineSimilarity(text1, text2 string) float64 {
|
|
vec1 := tokenizeToVector(text1)
|
|
vec2 := tokenizeToVector(text2)
|
|
|
|
if len(vec1) == 0 || len(vec2) == 0 {
|
|
return 0.0
|
|
}
|
|
|
|
var dotProduct float64
|
|
for term, freq1 := range vec1 {
|
|
if freq2, exists := vec2[term]; exists {
|
|
dotProduct += float64(freq1 * freq2)
|
|
}
|
|
}
|
|
|
|
var norm1 float64
|
|
for _, freq := range vec1 {
|
|
norm1 += float64(freq * freq)
|
|
}
|
|
norm1 = math.Sqrt(norm1)
|
|
|
|
var norm2 float64
|
|
for _, freq := range vec2 {
|
|
norm2 += float64(freq * freq)
|
|
}
|
|
norm2 = math.Sqrt(norm2)
|
|
|
|
if norm1 == 0 || norm2 == 0 {
|
|
return 0.0
|
|
}
|
|
|
|
return dotProduct / (norm1 * norm2)
|
|
}
|
|
|
|
var wordRegex = regexp.MustCompile(`[a-zA-Z0-9]+`)
|
|
|
|
var stopWords = map[string]bool{
|
|
"the": true, "and": true, "for": true, "with": true, "this": true,
|
|
"that": true, "from": true, "are": true, "was": true, "were": true,
|
|
"google": true, "position": true, "apply": true, "url": true, "location": true,
|
|
"seattle": true, "usa": true, "job": true, "http": true, "https": true,
|
|
}
|
|
|
|
func tokenizeToVector(text string) map[string]int {
|
|
words := wordRegex.FindAllString(strings.ToLower(text), -1)
|
|
freqMap := make(map[string]int)
|
|
for _, w := range words {
|
|
if len(w) > 2 && !stopWords[w] {
|
|
freqMap[w]++
|
|
}
|
|
}
|
|
return freqMap
|
|
}
|
|
|
|
func (db *DB) GetPendingRawJobs() ([]RawJob, error) {
|
|
query := `SELECT id, canonical_id, target_id, title, company, location, apply_url, raw_content, status, crawled_at FROM raw_jobs WHERE status = 'PENDING'`
|
|
rows, err := db.Query(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var jobs []RawJob
|
|
for rows.Next() {
|
|
var j RawJob
|
|
var tID sql.NullInt64
|
|
if err := rows.Scan(&j.ID, &j.CanonicalID, &tID, &j.Title, &j.Company, &j.Location, &j.ApplyURL, &j.RawContent, &j.Status, &j.CrawledAt); err != nil {
|
|
return nil, err
|
|
}
|
|
if tID.Valid {
|
|
j.TargetID = &tID.Int64
|
|
}
|
|
jobs = append(jobs, j)
|
|
}
|
|
return jobs, nil
|
|
}
|
|
|
|
func (db *DB) UpdateRawJobStatus(id int64, status string) error {
|
|
query := `UPDATE raw_jobs SET status = ? WHERE id = ?`
|
|
_, err := db.Exec(query, status, id)
|
|
return err
|
|
}
|
|
|
|
// Job Analyses Queries
|
|
func (db *DB) CreateJobAnalysis(rawJobID int64, score float64, isQualified bool, status string) error {
|
|
query := `
|
|
INSERT INTO job_analyses (raw_job_id, score, is_qualified, status)
|
|
VALUES (?, ?, ?, ?)
|
|
ON CONFLICT(raw_job_id) DO UPDATE SET score=excluded.score, is_qualified=excluded.is_qualified, status=excluded.status
|
|
`
|
|
_, err := db.Exec(query, rawJobID, score, isQualified, status)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) UpdateJobAnalysisLLM(rawJobID int64, matchedJSON, missingJSON, suggestionsJSON string) error {
|
|
query := `
|
|
UPDATE job_analyses
|
|
SET matched_highlights = ?, missing_highlights = ?, tailoring_suggestions = ?, status = 'PARSED', analyzed_at = CURRENT_TIMESTAMP
|
|
WHERE raw_job_id = ?
|
|
`
|
|
_, err := db.Exec(query, matchedJSON, missingJSON, suggestionsJSON, rawJobID)
|
|
return err
|
|
}
|
|
|
|
func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
|
|
query := `
|
|
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at,
|
|
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), a.status, a.analyzed_at
|
|
FROM raw_jobs r
|
|
JOIN job_analyses a ON r.id = a.raw_job_id
|
|
WHERE a.is_qualified = 1 AND a.status = 'PENDING'
|
|
`
|
|
rows, err := db.Query(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var results []JobWithAnalysis
|
|
for rows.Next() {
|
|
var item JobWithAnalysis
|
|
var tID sql.NullInt64
|
|
var aID int64
|
|
var score float64
|
|
var isQualified bool
|
|
var matched, missing, suggestions, aStatus string
|
|
var analyzedAt sql.NullTime
|
|
|
|
err := rows.Scan(
|
|
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt,
|
|
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if tID.Valid {
|
|
item.TargetID = &tID.Int64
|
|
}
|
|
analysis := &JobAnalysis{
|
|
ID: aID,
|
|
RawJobID: item.ID,
|
|
Score: score,
|
|
IsQualified: isQualified,
|
|
MatchedHighlights: matched,
|
|
MissingHighlights: missing,
|
|
TailoringSuggestions: suggestions,
|
|
Status: aStatus,
|
|
}
|
|
if analyzedAt.Valid {
|
|
analysis.AnalyzedAt = &analyzedAt.Time
|
|
}
|
|
item.Analysis = analysis
|
|
results = append(results, item)
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
|
|
query := `
|
|
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at,
|
|
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
|
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
|
FROM raw_jobs r
|
|
LEFT JOIN job_analyses a ON r.id = a.raw_job_id
|
|
LEFT JOIN job_applications app ON r.id = app.raw_job_id
|
|
ORDER BY COALESCE(a.score, 0) DESC, r.crawled_at DESC
|
|
`
|
|
rows, err := db.Query(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var results []JobWithAnalysis
|
|
for rows.Next() {
|
|
var item JobWithAnalysis
|
|
var tID sql.NullInt64
|
|
var aID, appID sql.NullInt64
|
|
var score sql.NullFloat64
|
|
var isQualified sql.NullBool
|
|
var matched, missing, suggestions, aStatus sql.NullString
|
|
var analyzedAt sql.NullTime
|
|
var appStatus, notes sql.NullString
|
|
var appliedAt sql.NullTime
|
|
var updatedAt sql.NullTime
|
|
|
|
err := rows.Scan(
|
|
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt,
|
|
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
|
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if tID.Valid {
|
|
item.TargetID = &tID.Int64
|
|
}
|
|
if aID.Valid {
|
|
analysis := &JobAnalysis{
|
|
ID: aID.Int64,
|
|
RawJobID: item.ID,
|
|
Score: score.Float64,
|
|
IsQualified: isQualified.Bool,
|
|
MatchedHighlights: matched.String,
|
|
MissingHighlights: missing.String,
|
|
TailoringSuggestions: suggestions.String,
|
|
Status: aStatus.String,
|
|
}
|
|
if analyzedAt.Valid {
|
|
analysis.AnalyzedAt = &analyzedAt.Time
|
|
}
|
|
item.Analysis = analysis
|
|
}
|
|
if appID.Valid {
|
|
app := &JobApplication{
|
|
ID: appID.Int64,
|
|
RawJobID: item.ID,
|
|
Status: appStatus.String,
|
|
Notes: notes.String,
|
|
UpdatedAt: updatedAt.Time,
|
|
}
|
|
if appliedAt.Valid {
|
|
app.AppliedAt = &appliedAt.Time
|
|
}
|
|
item.App = app
|
|
}
|
|
results = append(results, item)
|
|
}
|
|
return results, nil
|
|
}
|
|
|
|
func (db *DB) GetJobByID(rawJobID int64) (*JobWithAnalysis, error) {
|
|
query := `
|
|
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at,
|
|
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
|
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
|
FROM raw_jobs r
|
|
LEFT JOIN job_analyses a ON r.id = a.raw_job_id
|
|
LEFT JOIN job_applications app ON r.id = app.raw_job_id
|
|
WHERE r.id = ?
|
|
`
|
|
row := db.QueryRow(query, rawJobID)
|
|
var item JobWithAnalysis
|
|
var tID sql.NullInt64
|
|
var aID, appID sql.NullInt64
|
|
var score sql.NullFloat64
|
|
var isQualified sql.NullBool
|
|
var matched, missing, suggestions, aStatus sql.NullString
|
|
var analyzedAt sql.NullTime
|
|
var appStatus, notes sql.NullString
|
|
var appliedAt sql.NullTime
|
|
var updatedAt sql.NullTime
|
|
|
|
err := row.Scan(
|
|
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt,
|
|
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
|
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if tID.Valid {
|
|
item.TargetID = &tID.Int64
|
|
}
|
|
if aID.Valid {
|
|
analysis := &JobAnalysis{
|
|
ID: aID.Int64,
|
|
RawJobID: item.ID,
|
|
Score: score.Float64,
|
|
IsQualified: isQualified.Bool,
|
|
MatchedHighlights: matched.String,
|
|
MissingHighlights: missing.String,
|
|
TailoringSuggestions: suggestions.String,
|
|
Status: aStatus.String,
|
|
}
|
|
if analyzedAt.Valid {
|
|
analysis.AnalyzedAt = &analyzedAt.Time
|
|
}
|
|
item.Analysis = analysis
|
|
}
|
|
if appID.Valid {
|
|
app := &JobApplication{
|
|
ID: appID.Int64,
|
|
RawJobID: item.ID,
|
|
Status: appStatus.String,
|
|
Notes: notes.String,
|
|
UpdatedAt: updatedAt.Time,
|
|
}
|
|
if appliedAt.Valid {
|
|
app.AppliedAt = &appliedAt.Time
|
|
}
|
|
item.App = app
|
|
}
|
|
return &item, nil
|
|
}
|
|
|
|
// Applications Queries
|
|
func (db *DB) CreateOrUpdateApplication(rawJobID int64, status, notes string) (*JobApplication, error) {
|
|
now := time.Now()
|
|
var appliedAt *time.Time
|
|
if status == "APPLIED" {
|
|
appliedAt = &now
|
|
}
|
|
|
|
query := `
|
|
INSERT INTO job_applications (raw_job_id, status, notes, applied_at, updated_at)
|
|
VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
|
|
ON CONFLICT(raw_job_id) DO UPDATE SET
|
|
status=excluded.status,
|
|
notes=COALESCE(excluded.notes, job_applications.notes),
|
|
applied_at=COALESCE(excluded.applied_at, job_applications.applied_at),
|
|
updated_at=CURRENT_TIMESTAMP
|
|
RETURNING id, raw_job_id, status, notes, applied_at, updated_at
|
|
`
|
|
var app JobApplication
|
|
var appAt sql.NullTime
|
|
err := db.QueryRow(query, rawJobID, status, notes, appliedAt).Scan(
|
|
&app.ID, &app.RawJobID, &app.Status, &app.Notes, &appAt, &app.UpdatedAt,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if appAt.Valid {
|
|
app.AppliedAt = &appAt.Time
|
|
}
|
|
return &app, nil
|
|
}
|
|
|
|
func (db *DB) GetApplications() ([]JobWithAnalysis, error) {
|
|
query := `
|
|
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at,
|
|
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
|
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
|
FROM job_applications app
|
|
JOIN raw_jobs r ON app.raw_job_id = r.id
|
|
LEFT JOIN job_analyses a ON r.id = a.raw_job_id
|
|
ORDER BY app.updated_at DESC
|
|
`
|
|
rows, err := db.Query(query)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
|
|
var results []JobWithAnalysis
|
|
for rows.Next() {
|
|
var item JobWithAnalysis
|
|
var tID sql.NullInt64
|
|
var aID, appID sql.NullInt64
|
|
var score sql.NullFloat64
|
|
var isQualified sql.NullBool
|
|
var matched, missing, suggestions, aStatus sql.NullString
|
|
var analyzedAt sql.NullTime
|
|
var appStatus, notes sql.NullString
|
|
var appliedAt sql.NullTime
|
|
var updatedAt time.Time
|
|
|
|
err := rows.Scan(
|
|
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt,
|
|
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
|
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if tID.Valid {
|
|
item.TargetID = &tID.Int64
|
|
}
|
|
if aID.Valid {
|
|
analysis := &JobAnalysis{
|
|
ID: aID.Int64,
|
|
RawJobID: item.ID,
|
|
Score: score.Float64,
|
|
IsQualified: isQualified.Bool,
|
|
MatchedHighlights: matched.String,
|
|
MissingHighlights: missing.String,
|
|
TailoringSuggestions: suggestions.String,
|
|
Status: aStatus.String,
|
|
}
|
|
if analyzedAt.Valid {
|
|
analysis.AnalyzedAt = &analyzedAt.Time
|
|
}
|
|
item.Analysis = analysis
|
|
}
|
|
app := &JobApplication{
|
|
ID: appID.Int64,
|
|
RawJobID: item.ID,
|
|
Status: appStatus.String,
|
|
Notes: notes.String,
|
|
UpdatedAt: updatedAt,
|
|
}
|
|
if appliedAt.Valid {
|
|
app.AppliedAt = &appliedAt.Time
|
|
}
|
|
item.App = app
|
|
results = append(results, item)
|
|
}
|
|
return results, nil
|
|
}
|