refactor: implement worker pool pattern, add persistent task queue, and decouple target loading logic

This commit is contained in:
2026-08-12 23:28:31 -07:00
parent 51b9468737
commit 890469fed8
18 changed files with 694 additions and 442 deletions
+4 -3
View File
@@ -278,9 +278,10 @@ func extractBalancedBracket(s string, startIdx int) string {
continue
}
if ch == '[' || ch == '{' {
switch ch {
case '[', '{':
depth++
} else if ch == ']' || ch == '}' {
case ']', '}':
depth--
if depth == 0 {
return s[i : j+1]
@@ -328,7 +329,7 @@ func (g *GoogleCrawler) fetchURL(ctx context.Context, urlStr string) (string, er
if err != nil {
return "", fmt.Errorf("HTTP fetch failed for %s: %w", urlStr, err)
}
defer resp.Body.Close()
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr)