hireme/pkg/crawler/databricks/databricks.go

package databricks

import (
	"context"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"regexp"
	"strings"
	"time"

	"github.com/hireme/hireme/internal/crawler"
	"github.com/hireme/hireme/internal/logger"
)

type DatabricksCrawler struct {
	client *http.Client
}

func NewDatabricksCrawler() *DatabricksCrawler {
	return NewDatabricksCrawlerWithClient(&http.Client{
		Timeout: 30 * time.Second,
	})
}

func NewDatabricksCrawlerWithClient(client *http.Client) *DatabricksCrawler {
	return &DatabricksCrawler{
		client: client,
	}
}

func (c *DatabricksCrawler) CanHandle(platform string) bool {
	p := strings.ToLower(platform)
	return p == "databricks" || strings.Contains(p, "databricks.com/company/careers") || strings.Contains(p, "boards.greenhouse.io/databricks")
}

type databricksJobsResponse struct {
	Jobs []databricksJob `json:"jobs"`
}

type databricksJob struct {
	ID          int64  `json:"id"`
	Title       string `json:"title"`
	AbsoluteURL string `json:"absolute_url"`
	Location    struct {
		Name string `json:"name"`
	} `json:"location"`
	Offices []struct {
		Name     string `json:"name"`
		Location string `json:"location"`
	} `json:"offices"`
	Departments []struct {
		Name string `json:"name"`
	} `json:"departments"`
	Content string `json:"content"`
}

func (c *DatabricksCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
	seen := make(map[string]bool)
	var discovered []crawler.DiscoveredJob

	apiURL := "https://boards-api.greenhouse.io/v1/boards/databricks/jobs"
	if targetURL != "" && strings.Contains(targetURL, "boards-api.greenhouse.io") {
		apiURL = targetURL
	}

	body, err := c.fetchURL(ctx, apiURL)
	if err != nil {
		return nil, err
	}

	var resp databricksJobsResponse
	if err := json.Unmarshal([]byte(body), &resp); err != nil {
		return nil, fmt.Errorf("failed to parse Databricks Greenhouse API response: %w", err)
	}

	for _, job := range resp.Jobs {
		if job.ID == 0 {
			continue
		}

		jobIDStr := fmt.Sprintf("%d", job.ID)
		applyURL := job.AbsoluteURL
		if applyURL == "" {
			applyURL = fmt.Sprintf("https://boards.greenhouse.io/databricks/jobs/%s", jobIDStr)
		}

		if !seen[applyURL] {
			seen[applyURL] = true
			discovered = append(discovered, crawler.DiscoveredJob{
				CanonicalID: fmt.Sprintf("databricks:%s", jobIDStr),
				ApplyURL:    applyURL,
			})
		}
	}

	logger.Log.Infow("Databricks discovery finished", "found_count", len(discovered))
	return discovered, nil
}

func (c *DatabricksCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
	jobID := parseDatabricksJobID(applyURL)
	if jobID == "" {
		jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16]
	}

	detailAPI := fmt.Sprintf("https://boards-api.greenhouse.io/v1/boards/databricks/jobs/%s", jobID)
	body, err := c.fetchURL(ctx, detailAPI)
	if err == nil && json.Valid([]byte(body)) {
		var job databricksJob
		if err := json.Unmarshal([]byte(body), &job); err == nil && job.Title != "" {
			loc := job.Location.Name
			if loc == "" && len(job.Offices) > 0 {
				var locs []string
				for _, o := range job.Offices {
					if o.Name != "" {
						locs = append(locs, o.Name)
					}
				}
				loc = strings.Join(locs, "; ")
			}
			if loc == "" {
				loc = "Bellevue / Seattle, WA"
			}

			return &crawler.ExtractedJob{
				CanonicalID: fmt.Sprintf("databricks:%s", jobID),
				Title:       job.Title,
				Company:     "Databricks",
				Location:    loc,
				ApplyURL:    applyURL,
				RawContent:  cleanHTML(job.Content),
			}, nil
		}
	}

	// Fallback to HTML
	htmlBody, err := c.fetchURL(ctx, applyURL)
	if err != nil {
		return nil, err
	}

	title := extractRegex(htmlBody, `<h1[^>]*class="app-title"[^>]*>(.*?)</h1>`)
	if title == "" {
		title = extractRegex(htmlBody, `<h1[^>]*>(.*?)</h1>`)
	}
	if title == "" {
		title = "Software Engineer - Databricks"
	}

	return &crawler.ExtractedJob{
		CanonicalID: fmt.Sprintf("databricks:%s", jobID),
		Title:       cleanHTML(title),
		Company:     "Databricks",
		Location:    "Bellevue / Seattle, WA",
		ApplyURL:    applyURL,
		RawContent:  cleanHTML(htmlBody),
	}, nil
}

func (c *DatabricksCrawler) fetchURL(ctx context.Context, urlStr string) (string, error) {
	req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
	if err != nil {
		return "", fmt.Errorf("failed to create request: %w", err)
	}

	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
	req.Header.Set("Accept", "application/json, text/html, */*")

	resp, err := c.client.Do(req)
	if err != nil {
		return "", fmt.Errorf("HTTP GET failed: %w", err)
	}
	defer func() { _ = resp.Body.Close() }()

	if resp.StatusCode != http.StatusOK {
		return "", fmt.Errorf("HTTP GET returned status %d for %s", resp.StatusCode, urlStr)
	}

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", fmt.Errorf("failed to read response: %w", err)
	}

	return string(body), nil
}

func parseDatabricksJobID(u string) string {
	re := regexp.MustCompile(`/jobs/(\d+)`)
	m := re.FindStringSubmatch(u)
	if len(m) >= 2 {
		return m[1]
	}
	return ""
}

func extractRegex(s, pattern string) string {
	re := regexp.MustCompile(pattern)
	m := re.FindStringSubmatch(s)
	if len(m) >= 2 {
		return m[1]
	}
	return ""
}

func cleanHTML(s string) string {
	s = strings.ReplaceAll(s, "<b>", "**")
	s = strings.ReplaceAll(s, "</b>", "**")
	s = strings.ReplaceAll(s, "<strong>", "**")
	s = strings.ReplaceAll(s, "</strong>", "**")
	s = strings.ReplaceAll(s, "<br>", "\n")
	s = strings.ReplaceAll(s, "<br/>", "\n")
	s = strings.ReplaceAll(s, "<br />", "\n")
	s = strings.ReplaceAll(s, "<li>", "- ")
	s = strings.ReplaceAll(s, "</li>", "\n")
	s = strings.ReplaceAll(s, "<ul>", "\n")
	s = strings.ReplaceAll(s, "</ul>", "\n")
	s = strings.ReplaceAll(s, "<p>", "\n")
	s = strings.ReplaceAll(s, "</p>", "\n")
	reTags := regexp.MustCompile(`<[^>]*>`)
	s = reTags.ReplaceAllString(s, "")
	return strings.TrimSpace(s)
}