package snowflake import ( "context" "crypto/sha256" "encoding/json" "fmt" "io" "net/http" "regexp" "strings" "time" "git.pengzhan.dev/haopengzhan/hireme/internal/crawler" "git.pengzhan.dev/haopengzhan/hireme/internal/logger" ) type SnowflakeCrawler struct { client *http.Client } func NewSnowflakeCrawler() *SnowflakeCrawler { return NewSnowflakeCrawlerWithClient(&http.Client{ Timeout: 30 * time.Second, }) } func NewSnowflakeCrawlerWithClient(client *http.Client) *SnowflakeCrawler { return &SnowflakeCrawler{ client: client, } } func (c *SnowflakeCrawler) CanHandle(platform string) bool { p := strings.ToLower(platform) return p == "snowflake" || strings.Contains(p, "careers.snowflake.com") || strings.Contains(p, "boards.greenhouse.io/snowflake") } type snowflakeJobsResponse struct { Jobs []snowflakeJob `json:"jobs"` } type snowflakeJob struct { ID int64 `json:"id"` Title string `json:"title"` AbsoluteURL string `json:"absolute_url"` Location struct { Name string `json:"name"` } `json:"location"` Offices []struct { Name string `json:"name"` Location string `json:"location"` } `json:"offices"` Departments []struct { Name string `json:"name"` } `json:"departments"` Content string `json:"content"` } func (c *SnowflakeCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { seen := make(map[string]bool) var discovered []crawler.DiscoveredJob apiURL := "https://boards-api.greenhouse.io/v1/boards/snowflake/jobs" if targetURL != "" && strings.Contains(targetURL, "boards-api.greenhouse.io") { apiURL = targetURL } body, err := c.fetchURL(ctx, apiURL) if err != nil { return nil, err } var resp snowflakeJobsResponse if err := json.Unmarshal([]byte(body), &resp); err != nil { return nil, fmt.Errorf("failed to parse Snowflake Greenhouse API response: %w", err) } for _, job := range resp.Jobs { if job.ID == 0 { continue } jobIDStr := fmt.Sprintf("%d", job.ID) applyURL := job.AbsoluteURL if applyURL == "" { applyURL = fmt.Sprintf("https://boards.greenhouse.io/snowflake/jobs/%s", jobIDStr) } if !seen[applyURL] { seen[applyURL] = true discovered = append(discovered, crawler.DiscoveredJob{ CanonicalID: fmt.Sprintf("snowflake:%s", jobIDStr), ApplyURL: applyURL, }) } } logger.Log.Infow("Snowflake discovery finished", "found_count", len(discovered)) return discovered, nil } func (c *SnowflakeCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { jobID := parseSnowflakeJobID(applyURL) if jobID == "" { jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16] } detailAPI := fmt.Sprintf("https://boards-api.greenhouse.io/v1/boards/snowflake/jobs/%s", jobID) body, err := c.fetchURL(ctx, detailAPI) if err == nil && json.Valid([]byte(body)) { var job snowflakeJob if err := json.Unmarshal([]byte(body), &job); err == nil && job.Title != "" { loc := job.Location.Name if loc == "" && len(job.Offices) > 0 { var locs []string for _, o := range job.Offices { if o.Name != "" { locs = append(locs, o.Name) } } loc = strings.Join(locs, "; ") } if loc == "" { loc = "Bellevue / Seattle, WA" } return &crawler.ExtractedJob{ CanonicalID: fmt.Sprintf("snowflake:%s", jobID), Title: job.Title, Company: "Snowflake", Location: loc, ApplyURL: applyURL, RawContent: cleanHTML(job.Content), }, nil } } // Fallback to HTML htmlBody, err := c.fetchURL(ctx, applyURL) if err != nil { return nil, err } title := extractRegex(htmlBody, `]*class="app-title"[^>]*>(.*?)`) if title == "" { title = extractRegex(htmlBody, `]*>(.*?)`) } if title == "" { title = "Software Engineer - Snowflake" } return &crawler.ExtractedJob{ CanonicalID: fmt.Sprintf("snowflake:%s", jobID), Title: cleanHTML(title), Company: "Snowflake", Location: "Bellevue / Seattle, WA", ApplyURL: applyURL, RawContent: cleanHTML(htmlBody), }, nil } func (c *SnowflakeCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) { req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) if err != nil { return "", fmt.Errorf("failed to create request: %w", err) } req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") req.Header.Set("Accept", "application/json, text/html, */*") resp, err := c.client.Do(req) if err != nil { return "", fmt.Errorf("HTTP GET failed: %w", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr) } body, err := io.ReadAll(resp.Body) if err != nil { return "", fmt.Errorf("failed to read response: %w", err) } return string(body), nil } func parseSnowflakeJobID(u string) string { re := regexp.MustCompile(`/jobs/(\d+)`) m := re.FindStringSubmatch(u) if len(m) >= 2 { return m[1] } return "" } func extractRegex(s, pattern string) string { re := regexp.MustCompile(pattern) m := re.FindStringSubmatch(s) if len(m) >= 2 { return m[1] } return "" } func cleanHTML(s string) string { s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "
", "\n") s = strings.ReplaceAll(s, "
", "\n") s = strings.ReplaceAll(s, "
", "\n") s = strings.ReplaceAll(s, "
  • ", "- ") s = strings.ReplaceAll(s, "
  • ", "\n") s = strings.ReplaceAll(s, "", "\n") s = strings.ReplaceAll(s, "

    ", "\n") s = strings.ReplaceAll(s, "

    ", "\n") reTags := regexp.MustCompile(`<[^>]*>`) s = reTags.ReplaceAllString(s, "") return strings.TrimSpace(s) }