56 lines
2.1 KiB
Go
56 lines
2.1 KiB
Go
package db
|
|
|
|
import (
|
|
"path/filepath"
|
|
"testing"
|
|
)
|
|
|
|
func TestVectorSimilarityDeduplication(t *testing.T) {
|
|
tempDir := t.TempDir()
|
|
dbPath := filepath.Join(tempDir, "test_dedup.db")
|
|
|
|
database, err := InitDB(dbPath)
|
|
if err != nil {
|
|
t.Fatalf("failed to init db: %v", err)
|
|
}
|
|
defer func() { _ = database.Close() }()
|
|
|
|
contentA := "Senior Software Engineer - Go Distributed Systems. Experience with microservices, gRPC, and SQLite database pipelines."
|
|
contentSimilar := "Senior Software Engineer - Go Distributed Systems. Experience with microservices, gRPC, and SQLite database pipelines."
|
|
contentDifferent := "Marketing Specialist - Social Media Campaign Manager. Experience with SEO, Google Ads, and brand strategy."
|
|
|
|
// 1. Insert job A
|
|
jobA, inserted1, err := database.InsertRawJob("canonical-1", nil, "Software Engineer", "Google", "Seattle", "https://google.com/job/1", contentA)
|
|
if err != nil || !inserted1 {
|
|
t.Fatalf("expected job A to be inserted: err=%v, inserted=%v", err, inserted1)
|
|
}
|
|
|
|
// 2. Insert job with identical/99%+ similar content -> should be deduplicated!
|
|
_, inserted2, err := database.InsertRawJob("canonical-2-different-id", nil, "Learn more about Software Engineer", "Google", "Seattle", "https://google.com/job/1-dup", contentSimilar)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if inserted2 {
|
|
t.Errorf("expected 99%% similar content job to be deduplicated (inserted=false), got inserted=true")
|
|
}
|
|
|
|
// 3. Insert job with distinct content -> should insert normally!
|
|
_, inserted3, err := database.InsertRawJob("canonical-3", nil, "Marketing Specialist", "Google", "Seattle", "https://google.com/job/3", contentDifferent)
|
|
if err != nil || !inserted3 {
|
|
t.Fatalf("expected distinct job to insert normally: err=%v, inserted=%v", err, inserted3)
|
|
}
|
|
|
|
// Verify only 2 raw jobs exist in DB
|
|
allJobs, err := database.getAllRawJobsForDeduplication()
|
|
if err != nil {
|
|
t.Fatalf("getAllRawJobsForDeduplication failed: %v", err)
|
|
}
|
|
if len(allJobs) != 2 {
|
|
t.Errorf("expected 2 raw jobs in DB after deduplication, got %d", len(allJobs))
|
|
}
|
|
|
|
if allJobs[0].ID != jobA.ID {
|
|
t.Errorf("first job mismatch")
|
|
}
|
|
}
|