feat: initialize project structure with SQLite database, crawler framework, and LLM-based JD analysis integration
This commit is contained in:
@@ -0,0 +1,86 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
type DB struct {
|
||||
*sql.DB
|
||||
}
|
||||
|
||||
func InitDB(dbPath string) (*DB, error) {
|
||||
if dbPath == "" {
|
||||
dbPath = "data/db/hireme.db"
|
||||
}
|
||||
|
||||
conn, err := sql.Open("sqlite3", dbPath+"?_foreign_keys=on")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open sqlite database: %w", err)
|
||||
}
|
||||
|
||||
if err := conn.Ping(); err != nil {
|
||||
return nil, fmt.Errorf("failed to ping sqlite database: %w", err)
|
||||
}
|
||||
|
||||
database := &DB{DB: conn}
|
||||
if err := database.migrate(); err != nil {
|
||||
return nil, fmt.Errorf("failed to run sqlite migrations: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("[DB] SQLite database initialized successfully at %s", dbPath)
|
||||
return database, nil
|
||||
}
|
||||
|
||||
func (db *DB) migrate() error {
|
||||
schema := `
|
||||
CREATE TABLE IF NOT EXISTS search_targets (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
target_url TEXT NOT NULL,
|
||||
platform TEXT NOT NULL,
|
||||
is_active BOOLEAN NOT NULL DEFAULT 1,
|
||||
last_crawled_at DATETIME
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS raw_jobs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
canonical_id TEXT UNIQUE NOT NULL,
|
||||
target_id INTEGER REFERENCES search_targets(id),
|
||||
title TEXT NOT NULL,
|
||||
company TEXT NOT NULL,
|
||||
location TEXT,
|
||||
apply_url TEXT NOT NULL,
|
||||
raw_content TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'PENDING',
|
||||
crawled_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS job_analyses (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
raw_job_id INTEGER UNIQUE NOT NULL REFERENCES raw_jobs(id) ON DELETE CASCADE,
|
||||
score REAL DEFAULT 0.0,
|
||||
is_qualified BOOLEAN NOT NULL DEFAULT 0,
|
||||
matched_highlights JSON,
|
||||
missing_highlights JSON,
|
||||
tailoring_suggestions JSON,
|
||||
status TEXT NOT NULL DEFAULT 'PENDING',
|
||||
analyzed_at DATETIME
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS job_applications (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
raw_job_id INTEGER UNIQUE NOT NULL REFERENCES raw_jobs(id) ON DELETE CASCADE,
|
||||
status TEXT NOT NULL DEFAULT 'SAVED',
|
||||
notes TEXT,
|
||||
applied_at DATETIME,
|
||||
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
`
|
||||
|
||||
_, err := db.Exec(schema)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type SearchTarget struct {
|
||||
ID int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
TargetURL string `json:"target_url"`
|
||||
Platform string `json:"platform"` // 'linkedin', 'google_jobs', 'greenhouse', 'lever'
|
||||
IsActive bool `json:"is_active"`
|
||||
LastCrawledAt *time.Time `json:"last_crawled_at"`
|
||||
}
|
||||
|
||||
type RawJob struct {
|
||||
ID int64 `json:"id"`
|
||||
CanonicalID string `json:"canonical_id"` // SHA256(company + title + applyURL)
|
||||
TargetID *int64 `json:"target_id"`
|
||||
Title string `json:"title"`
|
||||
Company string `json:"company"`
|
||||
Location string `json:"location"`
|
||||
ApplyURL string `json:"apply_url"`
|
||||
RawContent string `json:"raw_content"`
|
||||
Status string `json:"status"` // 'PENDING', 'PROCESSED', 'FAILED', 'DUPLICATE'
|
||||
CrawledAt time.Time `json:"crawled_at"`
|
||||
}
|
||||
|
||||
type JobAnalysis struct {
|
||||
ID int64 `json:"id"`
|
||||
RawJobID int64 `json:"raw_job_id"`
|
||||
Score float64 `json:"score"`
|
||||
IsQualified bool `json:"is_qualified"`
|
||||
MatchedHighlights string `json:"matched_highlights"` // JSON array string
|
||||
MissingHighlights string `json:"missing_highlights"` // JSON array string
|
||||
TailoringSuggestions string `json:"tailoring_suggestions"` // JSON array string
|
||||
Status string `json:"status"` // 'PENDING', 'PARSED', 'FILTERED_OUT', 'FAILED'
|
||||
AnalyzedAt *time.Time `json:"analyzed_at"`
|
||||
}
|
||||
|
||||
type JobApplication struct {
|
||||
ID int64 `json:"id"`
|
||||
RawJobID int64 `json:"raw_job_id"`
|
||||
Status string `json:"status"` // 'SAVED', 'APPLIED', 'INTERVIEW', 'REJECTED', 'OFFER'
|
||||
Notes string `json:"notes"`
|
||||
AppliedAt *time.Time `json:"applied_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
// JobWithAnalysis combines RawJob and JobAnalysis for UI rendering
|
||||
type JobWithAnalysis struct {
|
||||
RawJob
|
||||
Analysis *JobAnalysis `json:"analysis"`
|
||||
App *JobApplication `json:"application"`
|
||||
}
|
||||
@@ -0,0 +1,554 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"math"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Target Queries
|
||||
func (db *DB) CreateTarget(name, targetURL, platform string) (*SearchTarget, error) {
|
||||
query := `INSERT INTO search_targets (name, target_url, platform, is_active) VALUES (?, ?, ?, 1) RETURNING id, name, target_url, platform, is_active, last_crawled_at`
|
||||
var target SearchTarget
|
||||
var lastCrawled sql.NullTime
|
||||
err := db.QueryRow(query, name, targetURL, platform).Scan(
|
||||
&target.ID, &target.Name, &target.TargetURL, &target.Platform, &target.IsActive, &lastCrawled,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if lastCrawled.Valid {
|
||||
target.LastCrawledAt = &lastCrawled.Time
|
||||
}
|
||||
return &target, nil
|
||||
}
|
||||
|
||||
func (db *DB) GetActiveTargets() ([]SearchTarget, error) {
|
||||
query := `SELECT id, name, target_url, platform, is_active, last_crawled_at FROM search_targets WHERE is_active = 1`
|
||||
rows, err := db.Query(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var targets []SearchTarget
|
||||
for rows.Next() {
|
||||
var t SearchTarget
|
||||
var lastCrawled sql.NullTime
|
||||
if err := rows.Scan(&t.ID, &t.Name, &t.TargetURL, &t.Platform, &t.IsActive, &lastCrawled); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if lastCrawled.Valid {
|
||||
t.LastCrawledAt = &lastCrawled.Time
|
||||
}
|
||||
targets = append(targets, t)
|
||||
}
|
||||
return targets, nil
|
||||
}
|
||||
|
||||
func (db *DB) GetAllTargets() ([]SearchTarget, error) {
|
||||
query := `SELECT id, name, target_url, platform, is_active, last_crawled_at FROM search_targets ORDER BY id DESC`
|
||||
rows, err := db.Query(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var targets []SearchTarget
|
||||
for rows.Next() {
|
||||
var t SearchTarget
|
||||
var lastCrawled sql.NullTime
|
||||
if err := rows.Scan(&t.ID, &t.Name, &t.TargetURL, &t.Platform, &t.IsActive, &lastCrawled); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if lastCrawled.Valid {
|
||||
t.LastCrawledAt = &lastCrawled.Time
|
||||
}
|
||||
targets = append(targets, t)
|
||||
}
|
||||
return targets, nil
|
||||
}
|
||||
|
||||
func (db *DB) ToggleTargetStatus(id int64) error {
|
||||
query := `UPDATE search_targets SET is_active = CASE WHEN is_active = 1 THEN 0 ELSE 1 END WHERE id = ?`
|
||||
_, err := db.Exec(query, id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) UpdateTargetCrawledAt(id int64) error {
|
||||
query := `UPDATE search_targets SET last_crawled_at = CURRENT_TIMESTAMP WHERE id = ?`
|
||||
_, err := db.Exec(query, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// Raw Jobs Queries & Vector Similarity Deduplication
|
||||
func (db *DB) InsertRawJob(canonicalID string, targetID *int64, title, company, location, applyURL, rawContent string) (*RawJob, bool, error) {
|
||||
// Deduplication Check 1 & 2: Check canonical_id exist AND vector similarity >= 99%
|
||||
existingJobs, err := db.getAllRawJobsForDeduplication()
|
||||
if err == nil {
|
||||
for _, existing := range existingJobs {
|
||||
if existing.CanonicalID == canonicalID {
|
||||
return nil, false, nil // Duplicate Canonical ID
|
||||
}
|
||||
|
||||
// Vector Cosine Similarity check on raw_content (run only for substantial bodies >100 chars)
|
||||
if len(existing.RawContent) >= 100 && len(rawContent) >= 100 {
|
||||
similarity := CalculateCosineSimilarity(existing.RawContent, rawContent)
|
||||
if similarity >= 0.99 {
|
||||
return nil, false, nil // Content >= 99% vector match -> Deduplicate
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
query := `
|
||||
INSERT INTO raw_jobs (canonical_id, target_id, title, company, location, apply_url, raw_content, status)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, 'PENDING')
|
||||
ON CONFLICT(canonical_id) DO NOTHING
|
||||
RETURNING id, canonical_id, target_id, title, company, location, apply_url, raw_content, status, crawled_at
|
||||
`
|
||||
var job RawJob
|
||||
var tID sql.NullInt64
|
||||
err = db.QueryRow(query, canonicalID, targetID, title, company, location, applyURL, rawContent).Scan(
|
||||
&job.ID, &job.CanonicalID, &tID, &job.Title, &job.Company, &job.Location, &job.ApplyURL, &job.RawContent, &job.Status, &job.CrawledAt,
|
||||
)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, false, nil // Duplicate job
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if tID.Valid {
|
||||
job.TargetID = &tID.Int64
|
||||
}
|
||||
return &job, true, nil
|
||||
}
|
||||
|
||||
func (db *DB) CanonicalIDExists(canonicalID string) (bool, error) {
|
||||
query := `SELECT EXISTS(SELECT 1 FROM raw_jobs WHERE canonical_id = ?)`
|
||||
var exists bool
|
||||
err := db.QueryRow(query, canonicalID).Scan(&exists)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
func (db *DB) getAllRawJobsForDeduplication() ([]RawJob, error) {
|
||||
query := `SELECT id, canonical_id, raw_content FROM raw_jobs`
|
||||
rows, err := db.Query(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var jobs []RawJob
|
||||
for rows.Next() {
|
||||
var j RawJob
|
||||
if err := rows.Scan(&j.ID, &j.CanonicalID, &j.RawContent); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jobs = append(jobs, j)
|
||||
}
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
// CalculateCosineSimilarity tokenizes text into term frequency vectors and returns cosine similarity in range [0.0, 1.0]
|
||||
func CalculateCosineSimilarity(text1, text2 string) float64 {
|
||||
vec1 := tokenizeToVector(text1)
|
||||
vec2 := tokenizeToVector(text2)
|
||||
|
||||
if len(vec1) == 0 || len(vec2) == 0 {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
var dotProduct float64
|
||||
for term, freq1 := range vec1 {
|
||||
if freq2, exists := vec2[term]; exists {
|
||||
dotProduct += float64(freq1 * freq2)
|
||||
}
|
||||
}
|
||||
|
||||
var norm1 float64
|
||||
for _, freq := range vec1 {
|
||||
norm1 += float64(freq * freq)
|
||||
}
|
||||
norm1 = math.Sqrt(norm1)
|
||||
|
||||
var norm2 float64
|
||||
for _, freq := range vec2 {
|
||||
norm2 += float64(freq * freq)
|
||||
}
|
||||
norm2 = math.Sqrt(norm2)
|
||||
|
||||
if norm1 == 0 || norm2 == 0 {
|
||||
return 0.0
|
||||
}
|
||||
|
||||
return dotProduct / (norm1 * norm2)
|
||||
}
|
||||
|
||||
var wordRegex = regexp.MustCompile(`[a-zA-Z0-9]+`)
|
||||
|
||||
var stopWords = map[string]bool{
|
||||
"the": true, "and": true, "for": true, "with": true, "this": true,
|
||||
"that": true, "from": true, "are": true, "was": true, "were": true,
|
||||
"google": true, "position": true, "apply": true, "url": true, "location": true,
|
||||
"seattle": true, "usa": true, "job": true, "http": true, "https": true,
|
||||
}
|
||||
|
||||
func tokenizeToVector(text string) map[string]int {
|
||||
words := wordRegex.FindAllString(strings.ToLower(text), -1)
|
||||
freqMap := make(map[string]int)
|
||||
for _, w := range words {
|
||||
if len(w) > 2 && !stopWords[w] {
|
||||
freqMap[w]++
|
||||
}
|
||||
}
|
||||
return freqMap
|
||||
}
|
||||
|
||||
func (db *DB) GetPendingRawJobs() ([]RawJob, error) {
|
||||
query := `SELECT id, canonical_id, target_id, title, company, location, apply_url, raw_content, status, crawled_at FROM raw_jobs WHERE status = 'PENDING'`
|
||||
rows, err := db.Query(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var jobs []RawJob
|
||||
for rows.Next() {
|
||||
var j RawJob
|
||||
var tID sql.NullInt64
|
||||
if err := rows.Scan(&j.ID, &j.CanonicalID, &tID, &j.Title, &j.Company, &j.Location, &j.ApplyURL, &j.RawContent, &j.Status, &j.CrawledAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tID.Valid {
|
||||
j.TargetID = &tID.Int64
|
||||
}
|
||||
jobs = append(jobs, j)
|
||||
}
|
||||
return jobs, nil
|
||||
}
|
||||
|
||||
func (db *DB) UpdateRawJobStatus(id int64, status string) error {
|
||||
query := `UPDATE raw_jobs SET status = ? WHERE id = ?`
|
||||
_, err := db.Exec(query, status, id)
|
||||
return err
|
||||
}
|
||||
|
||||
// Job Analyses Queries
|
||||
func (db *DB) CreateJobAnalysis(rawJobID int64, score float64, isQualified bool, status string) error {
|
||||
query := `
|
||||
INSERT INTO job_analyses (raw_job_id, score, is_qualified, status)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT(raw_job_id) DO UPDATE SET score=excluded.score, is_qualified=excluded.is_qualified, status=excluded.status
|
||||
`
|
||||
_, err := db.Exec(query, rawJobID, score, isQualified, status)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) UpdateJobAnalysisLLM(rawJobID int64, matchedJSON, missingJSON, suggestionsJSON string) error {
|
||||
query := `
|
||||
UPDATE job_analyses
|
||||
SET matched_highlights = ?, missing_highlights = ?, tailoring_suggestions = ?, status = 'PARSED', analyzed_at = CURRENT_TIMESTAMP
|
||||
WHERE raw_job_id = ?
|
||||
`
|
||||
_, err := db.Exec(query, matchedJSON, missingJSON, suggestionsJSON, rawJobID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) GetPendingQualifiedAnalyses() ([]JobWithAnalysis, error) {
|
||||
query := `
|
||||
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at,
|
||||
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), a.status, a.analyzed_at
|
||||
FROM raw_jobs r
|
||||
JOIN job_analyses a ON r.id = a.raw_job_id
|
||||
WHERE a.is_qualified = 1 AND a.status = 'PENDING'
|
||||
`
|
||||
rows, err := db.Query(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var results []JobWithAnalysis
|
||||
for rows.Next() {
|
||||
var item JobWithAnalysis
|
||||
var tID sql.NullInt64
|
||||
var aID int64
|
||||
var score float64
|
||||
var isQualified bool
|
||||
var matched, missing, suggestions, aStatus string
|
||||
var analyzedAt sql.NullTime
|
||||
|
||||
err := rows.Scan(
|
||||
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt,
|
||||
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tID.Valid {
|
||||
item.TargetID = &tID.Int64
|
||||
}
|
||||
analysis := &JobAnalysis{
|
||||
ID: aID,
|
||||
RawJobID: item.ID,
|
||||
Score: score,
|
||||
IsQualified: isQualified,
|
||||
MatchedHighlights: matched,
|
||||
MissingHighlights: missing,
|
||||
TailoringSuggestions: suggestions,
|
||||
Status: aStatus,
|
||||
}
|
||||
if analyzedAt.Valid {
|
||||
analysis.AnalyzedAt = &analyzedAt.Time
|
||||
}
|
||||
item.Analysis = analysis
|
||||
results = append(results, item)
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (db *DB) GetJobsWithAnalysis() ([]JobWithAnalysis, error) {
|
||||
query := `
|
||||
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at,
|
||||
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
||||
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
||||
FROM raw_jobs r
|
||||
LEFT JOIN job_analyses a ON r.id = a.raw_job_id
|
||||
LEFT JOIN job_applications app ON r.id = app.raw_job_id
|
||||
ORDER BY COALESCE(a.score, 0) DESC, r.crawled_at DESC
|
||||
`
|
||||
rows, err := db.Query(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var results []JobWithAnalysis
|
||||
for rows.Next() {
|
||||
var item JobWithAnalysis
|
||||
var tID sql.NullInt64
|
||||
var aID, appID sql.NullInt64
|
||||
var score sql.NullFloat64
|
||||
var isQualified sql.NullBool
|
||||
var matched, missing, suggestions, aStatus sql.NullString
|
||||
var analyzedAt sql.NullTime
|
||||
var appStatus, notes sql.NullString
|
||||
var appliedAt sql.NullTime
|
||||
var updatedAt sql.NullTime
|
||||
|
||||
err := rows.Scan(
|
||||
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt,
|
||||
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
||||
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tID.Valid {
|
||||
item.TargetID = &tID.Int64
|
||||
}
|
||||
if aID.Valid {
|
||||
analysis := &JobAnalysis{
|
||||
ID: aID.Int64,
|
||||
RawJobID: item.ID,
|
||||
Score: score.Float64,
|
||||
IsQualified: isQualified.Bool,
|
||||
MatchedHighlights: matched.String,
|
||||
MissingHighlights: missing.String,
|
||||
TailoringSuggestions: suggestions.String,
|
||||
Status: aStatus.String,
|
||||
}
|
||||
if analyzedAt.Valid {
|
||||
analysis.AnalyzedAt = &analyzedAt.Time
|
||||
}
|
||||
item.Analysis = analysis
|
||||
}
|
||||
if appID.Valid {
|
||||
app := &JobApplication{
|
||||
ID: appID.Int64,
|
||||
RawJobID: item.ID,
|
||||
Status: appStatus.String,
|
||||
Notes: notes.String,
|
||||
UpdatedAt: updatedAt.Time,
|
||||
}
|
||||
if appliedAt.Valid {
|
||||
app.AppliedAt = &appliedAt.Time
|
||||
}
|
||||
item.App = app
|
||||
}
|
||||
results = append(results, item)
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (db *DB) GetJobByID(rawJobID int64) (*JobWithAnalysis, error) {
|
||||
query := `
|
||||
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at,
|
||||
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
||||
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
||||
FROM raw_jobs r
|
||||
LEFT JOIN job_analyses a ON r.id = a.raw_job_id
|
||||
LEFT JOIN job_applications app ON r.id = app.raw_job_id
|
||||
WHERE r.id = ?
|
||||
`
|
||||
row := db.QueryRow(query, rawJobID)
|
||||
var item JobWithAnalysis
|
||||
var tID sql.NullInt64
|
||||
var aID, appID sql.NullInt64
|
||||
var score sql.NullFloat64
|
||||
var isQualified sql.NullBool
|
||||
var matched, missing, suggestions, aStatus sql.NullString
|
||||
var analyzedAt sql.NullTime
|
||||
var appStatus, notes sql.NullString
|
||||
var appliedAt sql.NullTime
|
||||
var updatedAt sql.NullTime
|
||||
|
||||
err := row.Scan(
|
||||
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt,
|
||||
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
||||
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tID.Valid {
|
||||
item.TargetID = &tID.Int64
|
||||
}
|
||||
if aID.Valid {
|
||||
analysis := &JobAnalysis{
|
||||
ID: aID.Int64,
|
||||
RawJobID: item.ID,
|
||||
Score: score.Float64,
|
||||
IsQualified: isQualified.Bool,
|
||||
MatchedHighlights: matched.String,
|
||||
MissingHighlights: missing.String,
|
||||
TailoringSuggestions: suggestions.String,
|
||||
Status: aStatus.String,
|
||||
}
|
||||
if analyzedAt.Valid {
|
||||
analysis.AnalyzedAt = &analyzedAt.Time
|
||||
}
|
||||
item.Analysis = analysis
|
||||
}
|
||||
if appID.Valid {
|
||||
app := &JobApplication{
|
||||
ID: appID.Int64,
|
||||
RawJobID: item.ID,
|
||||
Status: appStatus.String,
|
||||
Notes: notes.String,
|
||||
UpdatedAt: updatedAt.Time,
|
||||
}
|
||||
if appliedAt.Valid {
|
||||
app.AppliedAt = &appliedAt.Time
|
||||
}
|
||||
item.App = app
|
||||
}
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
// Applications Queries
|
||||
func (db *DB) CreateOrUpdateApplication(rawJobID int64, status, notes string) (*JobApplication, error) {
|
||||
now := time.Now()
|
||||
var appliedAt *time.Time
|
||||
if status == "APPLIED" {
|
||||
appliedAt = &now
|
||||
}
|
||||
|
||||
query := `
|
||||
INSERT INTO job_applications (raw_job_id, status, notes, applied_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT(raw_job_id) DO UPDATE SET
|
||||
status=excluded.status,
|
||||
notes=COALESCE(excluded.notes, job_applications.notes),
|
||||
applied_at=COALESCE(excluded.applied_at, job_applications.applied_at),
|
||||
updated_at=CURRENT_TIMESTAMP
|
||||
RETURNING id, raw_job_id, status, notes, applied_at, updated_at
|
||||
`
|
||||
var app JobApplication
|
||||
var appAt sql.NullTime
|
||||
err := db.QueryRow(query, rawJobID, status, notes, appliedAt).Scan(
|
||||
&app.ID, &app.RawJobID, &app.Status, &app.Notes, &appAt, &app.UpdatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if appAt.Valid {
|
||||
app.AppliedAt = &appAt.Time
|
||||
}
|
||||
return &app, nil
|
||||
}
|
||||
|
||||
func (db *DB) GetApplications() ([]JobWithAnalysis, error) {
|
||||
query := `
|
||||
SELECT r.id, r.canonical_id, r.target_id, r.title, r.company, r.location, r.apply_url, r.raw_content, r.status, r.crawled_at,
|
||||
a.id, a.score, a.is_qualified, COALESCE(a.matched_highlights, '[]'), COALESCE(a.missing_highlights, '[]'), COALESCE(a.tailoring_suggestions, '[]'), COALESCE(a.status, 'PENDING'), a.analyzed_at,
|
||||
app.id, app.status, COALESCE(app.notes, ''), app.applied_at, app.updated_at
|
||||
FROM job_applications app
|
||||
JOIN raw_jobs r ON app.raw_job_id = r.id
|
||||
LEFT JOIN job_analyses a ON r.id = a.raw_job_id
|
||||
ORDER BY app.updated_at DESC
|
||||
`
|
||||
rows, err := db.Query(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var results []JobWithAnalysis
|
||||
for rows.Next() {
|
||||
var item JobWithAnalysis
|
||||
var tID sql.NullInt64
|
||||
var aID, appID sql.NullInt64
|
||||
var score sql.NullFloat64
|
||||
var isQualified sql.NullBool
|
||||
var matched, missing, suggestions, aStatus sql.NullString
|
||||
var analyzedAt sql.NullTime
|
||||
var appStatus, notes sql.NullString
|
||||
var appliedAt sql.NullTime
|
||||
var updatedAt time.Time
|
||||
|
||||
err := rows.Scan(
|
||||
&item.ID, &item.CanonicalID, &tID, &item.Title, &item.Company, &item.Location, &item.ApplyURL, &item.RawContent, &item.Status, &item.CrawledAt,
|
||||
&aID, &score, &isQualified, &matched, &missing, &suggestions, &aStatus, &analyzedAt,
|
||||
&appID, &appStatus, ¬es, &appliedAt, &updatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tID.Valid {
|
||||
item.TargetID = &tID.Int64
|
||||
}
|
||||
if aID.Valid {
|
||||
analysis := &JobAnalysis{
|
||||
ID: aID.Int64,
|
||||
RawJobID: item.ID,
|
||||
Score: score.Float64,
|
||||
IsQualified: isQualified.Bool,
|
||||
MatchedHighlights: matched.String,
|
||||
MissingHighlights: missing.String,
|
||||
TailoringSuggestions: suggestions.String,
|
||||
Status: aStatus.String,
|
||||
}
|
||||
if analyzedAt.Valid {
|
||||
analysis.AnalyzedAt = &analyzedAt.Time
|
||||
}
|
||||
item.Analysis = analysis
|
||||
}
|
||||
app := &JobApplication{
|
||||
ID: appID.Int64,
|
||||
RawJobID: item.ID,
|
||||
Status: appStatus.String,
|
||||
Notes: notes.String,
|
||||
UpdatedAt: updatedAt,
|
||||
}
|
||||
if appliedAt.Valid {
|
||||
app.AppliedAt = &appliedAt.Time
|
||||
}
|
||||
item.App = app
|
||||
results = append(results, item)
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestVectorSimilarityDeduplication(t *testing.T) {
|
||||
tempDir := t.TempDir()
|
||||
dbPath := filepath.Join(tempDir, "test_dedup.db")
|
||||
|
||||
database, err := InitDB(dbPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to init db: %v", err)
|
||||
}
|
||||
defer database.Close()
|
||||
|
||||
contentA := "Senior Software Engineer - Go Distributed Systems. Experience with microservices, gRPC, and SQLite database pipelines."
|
||||
contentSimilar := "Senior Software Engineer - Go Distributed Systems. Experience with microservices, gRPC, and SQLite database pipelines."
|
||||
contentDifferent := "Marketing Specialist - Social Media Campaign Manager. Experience with SEO, Google Ads, and brand strategy."
|
||||
|
||||
// 1. Insert job A
|
||||
jobA, inserted1, err := database.InsertRawJob("canonical-1", nil, "Software Engineer", "Google", "Seattle", "https://google.com/job/1", contentA)
|
||||
if err != nil || !inserted1 {
|
||||
t.Fatalf("expected job A to be inserted: err=%v, inserted=%v", err, inserted1)
|
||||
}
|
||||
|
||||
// 2. Insert job with identical/99%+ similar content -> should be deduplicated!
|
||||
_, inserted2, err := database.InsertRawJob("canonical-2-different-id", nil, "Learn more about Software Engineer", "Google", "Seattle", "https://google.com/job/1-dup", contentSimilar)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if inserted2 {
|
||||
t.Errorf("expected 99%% similar content job to be deduplicated (inserted=false), got inserted=true")
|
||||
}
|
||||
|
||||
// 3. Insert job with distinct content -> should insert normally!
|
||||
_, inserted3, err := database.InsertRawJob("canonical-3", nil, "Marketing Specialist", "Google", "Seattle", "https://google.com/job/3", contentDifferent)
|
||||
if err != nil || !inserted3 {
|
||||
t.Fatalf("expected distinct job to insert normally: err=%v, inserted=%v", err, inserted3)
|
||||
}
|
||||
|
||||
// Verify only 2 raw jobs exist in DB
|
||||
allJobs, err := database.GetPendingRawJobs()
|
||||
if err != nil {
|
||||
t.Fatalf("GetPendingRawJobs failed: %v", err)
|
||||
}
|
||||
if len(allJobs) != 2 {
|
||||
t.Errorf("expected 2 raw jobs in DB after deduplication, got %d", len(allJobs))
|
||||
}
|
||||
|
||||
if allJobs[0].ID != jobA.ID {
|
||||
t.Errorf("first job mismatch")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user