package db import ( "os" "path/filepath" "strings" "testing" ) func TestVectorSimilarityDeduplication(t *testing.T) { tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test_dedup.db") database, err := InitDB(dbPath) if err != nil { t.Fatalf("failed to init db: %v", err) } defer func() { _ = database.Close() }() contentA := "Senior Software Engineer - Go Distributed Systems. Experience with microservices, gRPC, and SQLite database pipelines." contentSimilar := "Senior Software Engineer - Go Distributed Systems. Experience with microservices, gRPC, and SQLite database pipelines." contentDifferent := "Marketing Specialist - Social Media Campaign Manager. Experience with SEO, Google Ads, and brand strategy." // 1. Insert job A jobA, inserted1, err := database.InsertRawJob("canonical-1", nil, "Software Engineer", "Google", "Seattle", "https://google.com/job/1", contentA) if err != nil || !inserted1 { t.Fatalf("expected job A to be inserted: err=%v, inserted=%v", err, inserted1) } // 2. Insert job with identical/99%+ similar content -> should be deduplicated! _, inserted2, err := database.InsertRawJob("canonical-2-different-id", nil, "Learn more about Software Engineer", "Google", "Seattle", "https://google.com/job/1-dup", contentSimilar) if err != nil { t.Fatalf("unexpected error: %v", err) } if inserted2 { t.Errorf("expected 99%% similar content job to be deduplicated (inserted=false), got inserted=true") } // 3. Insert job with distinct content -> should insert normally! _, inserted3, err := database.InsertRawJob("canonical-3", nil, "Marketing Specialist", "Google", "Seattle", "https://google.com/job/3", contentDifferent) if err != nil || !inserted3 { t.Fatalf("expected distinct job to insert normally: err=%v, inserted=%v", err, inserted3) } // Verify only 2 raw jobs exist in DB allJobs, err := database.getAllRawJobsForDeduplication() if err != nil { t.Fatalf("getAllRawJobsForDeduplication failed: %v", err) } if len(allJobs) != 2 { t.Errorf("expected 2 raw jobs in DB after deduplication, got %d", len(allJobs)) } if allJobs[0].ID != jobA.ID { t.Errorf("first job mismatch") } } func TestDeleteTargetCascade(t *testing.T) { tempDir := t.TempDir() dbPath := filepath.Join(tempDir, "test_delete_target.db") database, err := InitDB(dbPath) if err != nil { t.Fatalf("failed to init db: %v", err) } defer func() { _ = database.Close() }() // 1. Create two search targets target1, err := database.CreateTarget("Target 1", "https://example.com/target1", "google_jobs") if err != nil { t.Fatalf("failed to create target 1: %v", err) } target2, err := database.CreateTarget("Target 2", "https://example.com/target2", "apple") if err != nil { t.Fatalf("failed to create target 2: %v", err) } // 2. Insert raw jobs for target 1 and target 2 job1, _, err := database.InsertRawJob("canon-t1-j1", &target1.ID, "Job 1", "Google", "Seattle", "https://google.com/1", "Software Engineer Go") if err != nil { t.Fatalf("failed to insert job 1: %v", err) } job2, _, err := database.InsertRawJob("canon-t1-j2", &target1.ID, "Job 2", "Google", "Seattle", "https://google.com/2", "Software Engineer Python") if err != nil { t.Fatalf("failed to insert job 2: %v", err) } job3, _, err := database.InsertRawJob("canon-t2-j1", &target2.ID, "Job 3", "Apple", "Seattle", "https://apple.com/1", "Software Engineer iOS") if err != nil { t.Fatalf("failed to insert job 3: %v", err) } // 3. Create analyses and applications for target 1 jobs and target 2 jobs _ = database.CreateJobAnalysis(job1.ID, 85.0, true, "DONE") _ = database.CreateJobAnalysis(job2.ID, 40.0, false, "DONE") _ = database.CreateJobAnalysis(job3.ID, 90.0, true, "DONE") _, _ = database.CreateOrUpdateApplication(job1.ID, "APPLIED", "Sent resume") _, _ = database.CreateOrUpdateApplication(job3.ID, "SAVED", "Review later") // 4. Enqueue tasks _ = database.EnqueueTask("DISCOVER", map[string]interface{}{"target_id": target1.ID, "target_url": target1.TargetURL}) _ = database.EnqueueTask("DISCOVER", map[string]interface{}{"target_id": target2.ID, "target_url": target2.TargetURL}) _ = database.EnqueueTask("PROCESS", map[string]interface{}{"raw_job_id": job1.ID}) _ = database.EnqueueTask("PROCESS", map[string]interface{}{"raw_job_id": job3.ID}) // 5. Delete target 1 deletedTarget, deletedJobsCount, err := database.DeleteTarget(target1.ID) if err != nil { t.Fatalf("DeleteTarget failed: %v", err) } if deletedTarget.ID != target1.ID { t.Errorf("expected deleted target ID %d, got %d", target1.ID, deletedTarget.ID) } if deletedJobsCount != 2 { t.Errorf("expected 2 deleted jobs, got %d", deletedJobsCount) } // 6. Verify Target 1 is gone, Target 2 remains allTargets, err := database.GetAllTargets() if err != nil { t.Fatalf("GetAllTargets failed: %v", err) } if len(allTargets) != 1 || allTargets[0].ID != target2.ID { t.Errorf("expected 1 remaining target (target 2), got %v", allTargets) } // 7. Verify Target 1's jobs are gone, Target 2's job remains jobsWithAnalysis, err := database.GetJobsWithAnalysis() if err != nil { t.Fatalf("GetJobsWithAnalysis failed: %v", err) } if len(jobsWithAnalysis) != 1 || jobsWithAnalysis[0].ID != job3.ID { t.Errorf("expected only job 3 to remain, got %v", jobsWithAnalysis) } // 8. Verify applications for target 1 are gone, target 2 application remains apps, err := database.GetApplications() if err != nil { t.Fatalf("GetApplications failed: %v", err) } if len(apps) != 1 || apps[0].ID != job3.ID { t.Errorf("expected only job 3 application to remain, got %v", apps) } } func TestRemoveTargetFromYAML(t *testing.T) { tempDir := t.TempDir() yamlPath := filepath.Join(tempDir, "targets.yaml") initialYAML := `targets: - name: "Google Seattle" target_url: "https://google.com/jobs" platform: "google_jobs" is_active: true - name: "Apple Seattle" target_url: "https://apple.com/jobs" platform: "apple" is_active: true ` if err := os.WriteFile(yamlPath, []byte(initialYAML), 0644); err != nil { t.Fatalf("failed to write initial yaml: %v", err) } // Remove Google Seattle if err := RemoveTargetFromYAML(yamlPath, "https://google.com/jobs"); err != nil { t.Fatalf("RemoveTargetFromYAML failed: %v", err) } // Read back and verify data, err := os.ReadFile(yamlPath) if err != nil { t.Fatalf("failed to read yaml back: %v", err) } content := string(data) if strings.Contains(content, "https://google.com/jobs") { t.Errorf("expected google target to be removed, but found in file:\n%s", content) } if !strings.Contains(content, "https://apple.com/jobs") { t.Errorf("expected apple target to remain, but not found in file:\n%s", content) } }