package db import ( "path/filepath" "testing" ) func TestVectorSimilarityDeduplication(t *testing.T) { tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test_dedup.db") database, err := InitDB(dbPath) if err != nil { t.Fatalf("failed to init db: %v", err) } defer func() { _ = database.Close() }() contentA := "Senior Software Engineer - Go Distributed Systems. Experience with microservices, gRPC, and SQLite database pipelines." contentSimilar := "Senior Software Engineer - Go Distributed Systems. Experience with microservices, gRPC, and SQLite database pipelines." contentDifferent := "Marketing Specialist - Social Media Campaign Manager. Experience with SEO, Google Ads, and brand strategy." // 1. Insert job A jobA, inserted1, err := database.InsertRawJob("canonical-1", nil, "Software Engineer", "Google", "Seattle", "https://google.com/job/1", contentA) if err != nil || !inserted1 { t.Fatalf("expected job A to be inserted: err=%v, inserted=%v", err, inserted1) } // 2. Insert job with identical/99%+ similar content -> should be deduplicated! _, inserted2, err := database.InsertRawJob("canonical-2-different-id", nil, "Learn more about Software Engineer", "Google", "Seattle", "https://google.com/job/1-dup", contentSimilar) if err != nil { t.Fatalf("unexpected error: %v", err) } if inserted2 { t.Errorf("expected 99%% similar content job to be deduplicated (inserted=false), got inserted=true") } // 3. Insert job with distinct content -> should insert normally! _, inserted3, err := database.InsertRawJob("canonical-3", nil, "Marketing Specialist", "Google", "Seattle", "https://google.com/job/3", contentDifferent) if err != nil || !inserted3 { t.Fatalf("expected distinct job to insert normally: err=%v, inserted=%v", err, inserted3) } // Verify only 2 raw jobs exist in DB allJobs, err := database.getAllRawJobsForDeduplication() if err != nil { t.Fatalf("getAllRawJobsForDeduplication failed: %v", err) } if len(allJobs) != 2 { t.Errorf("expected 2 raw jobs in DB after deduplication, got %d", len(allJobs)) } if allJobs[0].ID != jobA.ID { t.Errorf("first job mismatch") } }