hireme/pkg/crawler/bytedance/bytedance.go

package bytedance

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"regexp"
	"strings"
	"time"

	"github.com/hireme/hireme/internal/crawler"
	"github.com/hireme/hireme/internal/logger"
)

type ByteDanceCrawler struct {
	client *http.Client
}

func NewByteDanceCrawler() *ByteDanceCrawler {
	return NewByteDanceCrawlerWithClient(&http.Client{
		Timeout: 30 * time.Second,
	})
}

func NewByteDanceCrawlerWithClient(client *http.Client) *ByteDanceCrawler {
	return &ByteDanceCrawler{
		client: client,
	}
}

func (c *ByteDanceCrawler) CanHandle(platform string) bool {
	p := strings.ToLower(platform)
	return p == "bytedance" || p == "tiktok" || strings.Contains(p, "bytedance.com") || strings.Contains(p, "tiktok.com")
}

type byteDanceSearchPayload struct {
	Keyword       string   `json:"keyword"`
	Limit         int      `json:"limit"`
	Offset        int      `json:"offset"`
	JobCategoryID []string `json:"job_category_id_list,omitempty"`
	LocationList  []string `json:"location_code_list,omitempty"`
}

type byteDanceSearchResponse struct {
	Code int `json:"code"`
	Data struct {
		Count    int                `json:"count"`
		JobPosts []byteDanceJobItem `json:"job_post_list"`
	} `json:"data"`
}

type byteDanceDetailResponse struct {
	Code int `json:"code"`
	Data struct {
		JobPost byteDanceJobItem `json:"job_post"`
	} `json:"data"`
}

type byteDanceJobItem struct {
	ID          string `json:"id"`
	Title       string `json:"title"`
	Description string `json:"description"`
	Requirement string `json:"requirement"`
	CityInfo    struct {
		Name     string `json:"name"`
		EnName   string `json:"en_name"`
		Location string `json:"location_code"`
	} `json:"city_info"`
	JobCategory struct {
		Name string `json:"name"`
	} `json:"job_category"`
	PublishTime int64 `json:"publish_time"`
}

func (c *ByteDanceCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
	seen := make(map[string]bool)
	var discovered []crawler.DiscoveredJob

	// Default API endpoint
	apiURL := "https://jobs.bytedance.com/api/v1/search/job/posts"
	if strings.Contains(targetURL, "tiktok.com") {
		apiURL = "https://careers.tiktok.com/api/v1/search/job/posts"
	}

	keyword := "Software Engineer"
	u, err := url.Parse(targetURL)
	if err == nil && u.Query().Get("keyword") != "" {
		keyword = u.Query().Get("keyword")
	}

	pageSize := 50
	for page := 0; page < 5; page++ {
		offset := page * pageSize
		reqPayload := byteDanceSearchPayload{
			Keyword: keyword,
			Limit:   pageSize,
			Offset:  offset,
		}

		payloadBytes, err := json.Marshal(reqPayload)
		if err != nil {
			return nil, err
		}

		req, err := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewBuffer(payloadBytes))
		if err != nil {
			if page == 0 {
				return nil, err
			}
			break
		}

		req.Header.Set("Content-Type", "application/json")
		req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")

		resp, err := c.client.Do(req)
		if err != nil {
			if page == 0 {
				return nil, err
			}
			break
		}
		defer func() { _ = resp.Body.Close() }()

		body, err := io.ReadAll(resp.Body)
		if err != nil {
			break
		}

		var searchResp byteDanceSearchResponse
		if err := json.Unmarshal(body, &searchResp); err != nil {
			if page == 0 {
				return nil, fmt.Errorf("failed to parse ByteDance search response: %w", err)
			}
			break
		}

		posts := searchResp.Data.JobPosts
		if len(posts) == 0 {
			break
		}

		newFound := false
		for _, post := range posts {
			if post.ID == "" {
				continue
			}

			applyURL := fmt.Sprintf("https://jobs.bytedance.com/en/position/%s/detail", post.ID)
			if strings.Contains(targetURL, "tiktok.com") {
				applyURL = fmt.Sprintf("https://careers.tiktok.com/position/%s/detail", post.ID)
			}

			if !seen[applyURL] {
				seen[applyURL] = true
				newFound = true
				discovered = append(discovered, crawler.DiscoveredJob{
					CanonicalID: fmt.Sprintf("bytedance:%s", post.ID),
					ApplyURL:    applyURL,
				})
			}
		}

		if !newFound || len(posts) < pageSize {
			break
		}
	}

	logger.Log.Infow("ByteDance discovery finished", "found_count", len(discovered))
	return discovered, nil
}

func (c *ByteDanceCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
	jobID := parseByteDanceJobID(applyURL)
	if jobID == "" {
		jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16]
	}

	apiURL := fmt.Sprintf("https://jobs.bytedance.com/api/v1/job/posts/%s", jobID)
	if strings.Contains(applyURL, "tiktok.com") {
		apiURL = fmt.Sprintf("https://careers.tiktok.com/api/v1/job/posts/%s", jobID)
	}

	req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
	if err == nil {
		req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
		resp, err := c.client.Do(req)
		if err == nil {
			defer func() { _ = resp.Body.Close() }()
			body, _ := io.ReadAll(resp.Body)
			var detailResp byteDanceDetailResponse
			if err := json.Unmarshal(body, &detailResp); err == nil && detailResp.Data.JobPost.Title != "" {
				p := detailResp.Data.JobPost
				var sb strings.Builder
				if p.Description != "" {
					sb.WriteString("## Responsibilities\n\n")
					sb.WriteString(cleanHTML(p.Description))
					sb.WriteString("\n\n")
				}
				if p.Requirement != "" {
					sb.WriteString("## Qualifications\n\n")
					sb.WriteString(cleanHTML(p.Requirement))
					sb.WriteString("\n\n")
				}

				loc := p.CityInfo.EnName
				if loc == "" {
					loc = p.CityInfo.Name
				}
				if loc == "" {
					loc = "Seattle / Bellevue, WA"
				}

				company := "ByteDance"
				if strings.Contains(applyURL, "tiktok") {
					company = "TikTok"
				}

				return &crawler.ExtractedJob{
					CanonicalID: fmt.Sprintf("bytedance:%s", jobID),
					Title:       p.Title,
					Company:     company,
					Location:    loc,
					ApplyURL:    applyURL,
					RawContent:  sb.String(),
				}, nil
			}
		}
	}

	return &crawler.ExtractedJob{
		CanonicalID: fmt.Sprintf("bytedance:%s", jobID),
		Title:       "Software Engineer - ByteDance",
		Company:     "ByteDance",
		Location:    "Seattle / Bellevue, WA",
		ApplyURL:    applyURL,
		RawContent:  "See job posting at " + applyURL,
	}, nil
}

func parseByteDanceJobID(u string) string {
	re := regexp.MustCompile(`/position/([0-9A-Za-z]+)`)
	m := re.FindStringSubmatch(u)
	if len(m) >= 2 {
		return m[1]
	}
	return ""
}

func cleanHTML(s string) string {
	s = strings.ReplaceAll(s, "<b>", "**")
	s = strings.ReplaceAll(s, "</b>", "**")
	s = strings.ReplaceAll(s, "<strong>", "**")
	s = strings.ReplaceAll(s, "</strong>", "**")
	s = strings.ReplaceAll(s, "<br>", "\n")
	s = strings.ReplaceAll(s, "<br/>", "\n")
	s = strings.ReplaceAll(s, "<br />", "\n")
	s = strings.ReplaceAll(s, "<li>", "- ")
	s = strings.ReplaceAll(s, "</li>", "\n")
	s = strings.ReplaceAll(s, "<ul>", "\n")
	s = strings.ReplaceAll(s, "</ul>", "\n")
	s = strings.ReplaceAll(s, "<p>", "\n")
	s = strings.ReplaceAll(s, "</p>", "\n")
	reTags := regexp.MustCompile(`<[^>]*>`)
	s = reTags.ReplaceAllString(s, "")
	return strings.TrimSpace(s)
}