package bytedance import ( "bytes" "context" "crypto/sha256" "encoding/json" "fmt" "io" "net/http" "net/url" "regexp" "strings" "time" "git.pengzhan.dev/haopengzhan/hireme/internal/crawler" "git.pengzhan.dev/haopengzhan/hireme/internal/logger" ) type ByteDanceCrawler struct { client *http.Client } func NewByteDanceCrawler() *ByteDanceCrawler { return NewByteDanceCrawlerWithClient(&http.Client{ Timeout: 30 * time.Second, }) } func NewByteDanceCrawlerWithClient(client *http.Client) *ByteDanceCrawler { return &ByteDanceCrawler{ client: client, } } func (c *ByteDanceCrawler) CanHandle(platform string) bool { p := strings.ToLower(platform) return p == "bytedance" || p == "tiktok" || strings.Contains(p, "bytedance.com") || strings.Contains(p, "tiktok.com") } type byteDanceSearchPayload struct { Keyword string `json:"keyword"` Limit int `json:"limit"` Offset int `json:"offset"` JobCategoryID []string `json:"job_category_id_list,omitempty"` LocationList []string `json:"location_code_list,omitempty"` } type byteDanceSearchResponse struct { Code int `json:"code"` Data struct { Count int `json:"count"` JobPosts []byteDanceJobItem `json:"job_post_list"` } `json:"data"` } type byteDanceDetailResponse struct { Code int `json:"code"` Data struct { JobPost byteDanceJobItem `json:"job_post"` } `json:"data"` } type byteDanceJobItem struct { ID string `json:"id"` Title string `json:"title"` Description string `json:"description"` Requirement string `json:"requirement"` CityInfo struct { Name string `json:"name"` EnName string `json:"en_name"` Location string `json:"location_code"` } `json:"city_info"` JobCategory struct { Name string `json:"name"` } `json:"job_category"` PublishTime int64 `json:"publish_time"` } func (c *ByteDanceCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) { seen := make(map[string]bool) var discovered []crawler.DiscoveredJob // Default API endpoint apiURL := "https://jobs.bytedance.com/api/v1/search/job/posts" if strings.Contains(targetURL, "tiktok.com") { apiURL = "https://careers.tiktok.com/api/v1/search/job/posts" } keyword := "Software Engineer" u, err := url.Parse(targetURL) if err == nil && u.Query().Get("keyword") != "" { keyword = u.Query().Get("keyword") } pageSize := 50 for page := 0; page < 5; page++ { offset := page * pageSize reqPayload := byteDanceSearchPayload{ Keyword: keyword, Limit: pageSize, Offset: offset, } payloadBytes, err := json.Marshal(reqPayload) if err != nil { return nil, err } req, err := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewBuffer(payloadBytes)) if err != nil { if page == 0 { return nil, err } break } req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") resp, err := c.client.Do(req) if err != nil { if page == 0 { return nil, err } break } defer func() { _ = resp.Body.Close() }() body, err := io.ReadAll(resp.Body) if err != nil { break } var searchResp byteDanceSearchResponse if err := json.Unmarshal(body, &searchResp); err != nil { if page == 0 { return nil, fmt.Errorf("failed to parse ByteDance search response: %w", err) } break } posts := searchResp.Data.JobPosts if len(posts) == 0 { break } newFound := false for _, post := range posts { if post.ID == "" { continue } applyURL := fmt.Sprintf("https://jobs.bytedance.com/en/position/%s/detail", post.ID) if strings.Contains(targetURL, "tiktok.com") { applyURL = fmt.Sprintf("https://careers.tiktok.com/position/%s/detail", post.ID) } if !seen[applyURL] { seen[applyURL] = true newFound = true discovered = append(discovered, crawler.DiscoveredJob{ CanonicalID: fmt.Sprintf("bytedance:%s", post.ID), ApplyURL: applyURL, }) } } if !newFound || len(posts) < pageSize { break } } logger.Log.Infow("ByteDance discovery finished", "found_count", len(discovered)) return discovered, nil } func (c *ByteDanceCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) { jobID := parseByteDanceJobID(applyURL) if jobID == "" { jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16] } apiURL := fmt.Sprintf("https://jobs.bytedance.com/api/v1/job/posts/%s", jobID) if strings.Contains(applyURL, "tiktok.com") { apiURL = fmt.Sprintf("https://careers.tiktok.com/api/v1/job/posts/%s", jobID) } req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil) if err == nil { req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") resp, err := c.client.Do(req) if err == nil { defer func() { _ = resp.Body.Close() }() body, _ := io.ReadAll(resp.Body) var detailResp byteDanceDetailResponse if err := json.Unmarshal(body, &detailResp); err == nil && detailResp.Data.JobPost.Title != "" { p := detailResp.Data.JobPost var sb strings.Builder if p.Description != "" { sb.WriteString("## Responsibilities\n\n") sb.WriteString(cleanHTML(p.Description)) sb.WriteString("\n\n") } if p.Requirement != "" { sb.WriteString("## Qualifications\n\n") sb.WriteString(cleanHTML(p.Requirement)) sb.WriteString("\n\n") } loc := p.CityInfo.EnName if loc == "" { loc = p.CityInfo.Name } if loc == "" { loc = "Seattle / Bellevue, WA" } company := "ByteDance" if strings.Contains(applyURL, "tiktok") { company = "TikTok" } return &crawler.ExtractedJob{ CanonicalID: fmt.Sprintf("bytedance:%s", jobID), Title: p.Title, Company: company, Location: loc, ApplyURL: applyURL, RawContent: sb.String(), }, nil } } } return &crawler.ExtractedJob{ CanonicalID: fmt.Sprintf("bytedance:%s", jobID), Title: "Software Engineer - ByteDance", Company: "ByteDance", Location: "Seattle / Bellevue, WA", ApplyURL: applyURL, RawContent: "See job posting at " + applyURL, }, nil } func parseByteDanceJobID(u string) string { re := regexp.MustCompile(`/position/([0-9A-Za-z]+)`) m := re.FindStringSubmatch(u) if len(m) >= 2 { return m[1] } return "" } func cleanHTML(s string) string { s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "", "**") s = strings.ReplaceAll(s, "
", "\n") s = strings.ReplaceAll(s, "
", "\n") s = strings.ReplaceAll(s, "
", "\n") s = strings.ReplaceAll(s, "
  • ", "- ") s = strings.ReplaceAll(s, "
  • ", "\n") s = strings.ReplaceAll(s, "", "\n") s = strings.ReplaceAll(s, "

    ", "\n") s = strings.ReplaceAll(s, "

    ", "\n") reTags := regexp.MustCompile(`<[^>]*>`) s = reTags.ReplaceAllString(s, "") return strings.TrimSpace(s) }