refactor: implement worker pool pattern, add persistent task queue, and decouple target loading logic

internal/db/queries_test.go

@@ -13,7 +13,7 @@ func TestVectorSimilarityDeduplication(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to init db: %v", err)
 	}
-	defer database.Close()
+	defer func() { _ = database.Close() }()
 
 	contentA := "Senior Software Engineer - Go Distributed Systems. Experience with microservices, gRPC, and SQLite database pipelines."
 	contentSimilar := "Senior Software Engineer - Go Distributed Systems. Experience with microservices, gRPC, and SQLite database pipelines."
@@ -41,9 +41,9 @@ func TestVectorSimilarityDeduplication(t *testing.T) {
 	}
 
 	// Verify only 2 raw jobs exist in DB
-	allJobs, err := database.GetPendingRawJobs()
+	allJobs, err := database.getAllRawJobsForDeduplication()
 	if err != nil {
-		t.Fatalf("GetPendingRawJobs failed: %v", err)
+		t.Fatalf("getAllRawJobsForDeduplication failed: %v", err)
 	}
 	if len(allJobs) != 2 {
 		t.Errorf("expected 2 raw jobs in DB after deduplication, got %d", len(allJobs))