refactor: implement worker pool pattern, add persistent task queue, and decouple target loading logic

2026-08-12 23:28:31 -07:00
parent 51b9468737
commit 890469fed8
18 changed files with 694 additions and 442 deletions
@@ -22,10 +22,21 @@ type RawJob struct {
 	Location    string    `json:"location"`
 	ApplyURL    string    `json:"apply_url"`
 	RawContent  string    `json:"raw_content"`
-	Status      string    `json:"status"` // 'PENDING', 'PROCESSED', 'FAILED', 'DUPLICATE'
 	CrawledAt   time.Time `json:"crawled_at"`
 }

+type Task struct {
+	ID           int64      `json:"id"`
+	TaskType     string     `json:"task_type"` // 'DISCOVER', 'FETCH', 'PROCESS', 'ANALYZE'
+	Payload      string     `json:"payload"`   // JSON string
+	Status       string     `json:"status"`    // 'PENDING', 'RUNNING', 'COMPLETED', 'FAILED'
+	ErrorMessage string     `json:"error_message"`
+	RetryCount   int        `json:"retry_count"`
+	NextRetryAt  time.Time  `json:"next_retry_at"`
+	CreatedAt    time.Time  `json:"created_at"`
+	UpdatedAt    time.Time  `json:"updated_at"`
+}
+
 type JobAnalysis struct {
 	ID                   int64      `json:"id"`
 	RawJobID             int64      `json:"raw_job_id"`