haopengzhan/hireme
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: migrate all log statements to structured logger calls for improved observability

Browse Source
This commit is contained in:
haopengzhan
2026-08-15 00:15:27 -07:00
parent 86a092ecd3
commit 150e3dfbd5
6 changed files with 43 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/crawler/google/google.go
+8 -8
View File
@@ -7,7 +7,6 @@ import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"regexp"
"strings"
@@ -15,6 +14,7 @@ import (
md "github.com/JohannesKaufmann/html-to-markdown/v2"
"github.com/hireme/hireme/internal/crawler"
"github.com/hireme/hireme/internal/logger"
"golang.org/x/net/html"
)
@@ -183,7 +183,7 @@ func extractFromAFData(htmlContent string) (title, company, location string) {
marker := `AF_initDataCallback({key: 'ds:0'`
idx := strings.Index(htmlContent, marker)
if idx == -1 {
log.Printf("[GoogleCrawler] AF_initDataCallback ds:0 not found")
logger.Log.Debugw("AF_initDataCallback ds:0 not found", "component", "GoogleCrawler")
return "", "", ""
}
@@ -191,7 +191,7 @@ func extractFromAFData(htmlContent string) (title, company, location string) {
dataMarker := "data:"
dataIdx := strings.Index(htmlContent[idx:], dataMarker)
if dataIdx == -1 {
log.Printf("[GoogleCrawler] 'data:' not found after ds:0 marker")
logger.Log.Debugw("'data:' not found after ds:0 marker", "component", "GoogleCrawler")
return "", "", ""
}
dataStart := idx + dataIdx + len(dataMarker)
@@ -199,14 +199,14 @@ func extractFromAFData(htmlContent string) (title, company, location string) {
// Use bracket counting to find the matching ] for the outer [
jsonStr := extractBalancedBracket(htmlContent, dataStart)
if jsonStr == "" {
log.Printf("[GoogleCrawler] Failed to extract balanced bracket for ds:0 data")
logger.Log.Debugw("Failed to extract balanced bracket for ds:0 data", "component", "GoogleCrawler")
return "", "", ""
}
// Parse the outer array: data = [[...job record...]]
var outerArray []json.RawMessage
if err := json.Unmarshal([]byte(jsonStr), &outerArray); err != nil {
log.Printf("[GoogleCrawler] JSON parse error for ds:0 outer: %v", err)
logger.Log.Debugw("JSON parse error for ds:0 outer", "component", "GoogleCrawler", "error", err)
return "", "", ""
}
if len(outerArray) == 0 {
@@ -216,7 +216,7 @@ func extractFromAFData(htmlContent string) (title, company, location string) {
// Parse inner job record array
var record []json.RawMessage
if err := json.Unmarshal(outerArray[0], &record); err != nil {
log.Printf("[GoogleCrawler] JSON parse error for job record: %v", err)
logger.Log.Debugw("JSON parse error for job record", "component", "GoogleCrawler", "error", err)
return "", "", ""
}
@@ -247,7 +247,7 @@ func extractFromAFData(htmlContent string) (title, company, location string) {
}
}
log.Printf("[GoogleCrawler] AF_initDataCallback parsed: title=%q company=%q location=%q", title, company, location)
logger.Log.Debugw("AF_initDataCallback parsed job", "component", "GoogleCrawler", "title", title, "company", company, "location", location)
return title, company, location
}
@@ -309,7 +309,7 @@ func (g *GoogleCrawler) fetchWithRetry(ctx context.Context, urlStr string) (stri
for attempt := 0; attempt < 3; attempt++ {
if attempt > 0 {
delay := time.Duration(attempt) * 2 * time.Second
log.Printf("[GoogleCrawler] Retry #%d for %s after %v", attempt, urlStr, delay)
logger.Log.Warnw("Retrying request", "component", "GoogleCrawler", "attempt", attempt, "url", urlStr, "delay", delay)
time.Sleep(delay)
}