refactor: implement worker pool pattern, add persistent task queue, and decouple target loading logic

2026-08-12 23:28:31 -07:00
parent 51b9468737
commit 890469fed8
18 changed files with 694 additions and 442 deletions
@@ -50,7 +50,7 @@ func (c *GenericCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*
 	if err != nil {
 		return nil, fmt.Errorf("HTTP request failed: %w", err)
 	}
-	defer resp.Body.Close()
+	defer func() { _ = resp.Body.Close() }()

 	if resp.StatusCode != http.StatusOK {
 		return nil, fmt.Errorf("HTTP status error %d for URL %s", resp.StatusCode, applyURL)