feat: implement multiple new job crawlers and add corresponding test suites
This commit is contained in:
@@ -0,0 +1,272 @@
|
||||
package bytedance
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hireme/hireme/internal/crawler"
|
||||
"github.com/hireme/hireme/internal/logger"
|
||||
)
|
||||
|
||||
type ByteDanceCrawler struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func NewByteDanceCrawler() *ByteDanceCrawler {
|
||||
return NewByteDanceCrawlerWithClient(&http.Client{
|
||||
Timeout: 30 * time.Second,
|
||||
})
|
||||
}
|
||||
|
||||
func NewByteDanceCrawlerWithClient(client *http.Client) *ByteDanceCrawler {
|
||||
return &ByteDanceCrawler{
|
||||
client: client,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ByteDanceCrawler) CanHandle(platform string) bool {
|
||||
p := strings.ToLower(platform)
|
||||
return p == "bytedance" || p == "tiktok" || strings.Contains(p, "bytedance.com") || strings.Contains(p, "tiktok.com")
|
||||
}
|
||||
|
||||
type byteDanceSearchPayload struct {
|
||||
Keyword string `json:"keyword"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
JobCategoryID []string `json:"job_category_id_list,omitempty"`
|
||||
LocationList []string `json:"location_code_list,omitempty"`
|
||||
}
|
||||
|
||||
type byteDanceSearchResponse struct {
|
||||
Code int `json:"code"`
|
||||
Data struct {
|
||||
Count int `json:"count"`
|
||||
JobPosts []byteDanceJobItem `json:"job_post_list"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type byteDanceDetailResponse struct {
|
||||
Code int `json:"code"`
|
||||
Data struct {
|
||||
JobPost byteDanceJobItem `json:"job_post"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type byteDanceJobItem struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Description string `json:"description"`
|
||||
Requirement string `json:"requirement"`
|
||||
CityInfo struct {
|
||||
Name string `json:"name"`
|
||||
EnName string `json:"en_name"`
|
||||
Location string `json:"location_code"`
|
||||
} `json:"city_info"`
|
||||
JobCategory struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"job_category"`
|
||||
PublishTime int64 `json:"publish_time"`
|
||||
}
|
||||
|
||||
func (c *ByteDanceCrawler) DiscoverJobs(ctx context.Context, targetURL string) ([]crawler.DiscoveredJob, error) {
|
||||
seen := make(map[string]bool)
|
||||
var discovered []crawler.DiscoveredJob
|
||||
|
||||
// Default API endpoint
|
||||
apiURL := "https://jobs.bytedance.com/api/v1/search/job/posts"
|
||||
if strings.Contains(targetURL, "tiktok.com") {
|
||||
apiURL = "https://careers.tiktok.com/api/v1/search/job/posts"
|
||||
}
|
||||
|
||||
keyword := "Software Engineer"
|
||||
u, err := url.Parse(targetURL)
|
||||
if err == nil && u.Query().Get("keyword") != "" {
|
||||
keyword = u.Query().Get("keyword")
|
||||
}
|
||||
|
||||
pageSize := 50
|
||||
for page := 0; page < 5; page++ {
|
||||
offset := page * pageSize
|
||||
reqPayload := byteDanceSearchPayload{
|
||||
Keyword: keyword,
|
||||
Limit: pageSize,
|
||||
Offset: offset,
|
||||
}
|
||||
|
||||
payloadBytes, err := json.Marshal(reqPayload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewBuffer(payloadBytes))
|
||||
if err != nil {
|
||||
if page == 0 {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
if page == 0 {
|
||||
return nil, err
|
||||
}
|
||||
break
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
|
||||
var searchResp byteDanceSearchResponse
|
||||
if err := json.Unmarshal(body, &searchResp); err != nil {
|
||||
if page == 0 {
|
||||
return nil, fmt.Errorf("failed to parse ByteDance search response: %w", err)
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
posts := searchResp.Data.JobPosts
|
||||
if len(posts) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
newFound := false
|
||||
for _, post := range posts {
|
||||
if post.ID == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
applyURL := fmt.Sprintf("https://jobs.bytedance.com/en/position/%s/detail", post.ID)
|
||||
if strings.Contains(targetURL, "tiktok.com") {
|
||||
applyURL = fmt.Sprintf("https://careers.tiktok.com/position/%s/detail", post.ID)
|
||||
}
|
||||
|
||||
if !seen[applyURL] {
|
||||
seen[applyURL] = true
|
||||
newFound = true
|
||||
discovered = append(discovered, crawler.DiscoveredJob{
|
||||
CanonicalID: fmt.Sprintf("bytedance:%s", post.ID),
|
||||
ApplyURL: applyURL,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if !newFound || len(posts) < pageSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
logger.Log.Infow("ByteDance discovery finished", "found_count", len(discovered))
|
||||
return discovered, nil
|
||||
}
|
||||
|
||||
func (c *ByteDanceCrawler) FetchJobDetail(ctx context.Context, applyURL string) (*crawler.ExtractedJob, error) {
|
||||
jobID := parseByteDanceJobID(applyURL)
|
||||
if jobID == "" {
|
||||
jobID = fmt.Sprintf("%x", sha256.Sum256([]byte(applyURL)))[:16]
|
||||
}
|
||||
|
||||
apiURL := fmt.Sprintf("https://jobs.bytedance.com/api/v1/job/posts/%s", jobID)
|
||||
if strings.Contains(applyURL, "tiktok.com") {
|
||||
apiURL = fmt.Sprintf("https://careers.tiktok.com/api/v1/job/posts/%s", jobID)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
|
||||
if err == nil {
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||
resp, err := c.client.Do(req)
|
||||
if err == nil {
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
var detailResp byteDanceDetailResponse
|
||||
if err := json.Unmarshal(body, &detailResp); err == nil && detailResp.Data.JobPost.Title != "" {
|
||||
p := detailResp.Data.JobPost
|
||||
var sb strings.Builder
|
||||
if p.Description != "" {
|
||||
sb.WriteString("## Responsibilities\n\n")
|
||||
sb.WriteString(cleanHTML(p.Description))
|
||||
sb.WriteString("\n\n")
|
||||
}
|
||||
if p.Requirement != "" {
|
||||
sb.WriteString("## Qualifications\n\n")
|
||||
sb.WriteString(cleanHTML(p.Requirement))
|
||||
sb.WriteString("\n\n")
|
||||
}
|
||||
|
||||
loc := p.CityInfo.EnName
|
||||
if loc == "" {
|
||||
loc = p.CityInfo.Name
|
||||
}
|
||||
if loc == "" {
|
||||
loc = "Seattle / Bellevue, WA"
|
||||
}
|
||||
|
||||
company := "ByteDance"
|
||||
if strings.Contains(applyURL, "tiktok") {
|
||||
company = "TikTok"
|
||||
}
|
||||
|
||||
return &crawler.ExtractedJob{
|
||||
CanonicalID: fmt.Sprintf("bytedance:%s", jobID),
|
||||
Title: p.Title,
|
||||
Company: company,
|
||||
Location: loc,
|
||||
ApplyURL: applyURL,
|
||||
RawContent: sb.String(),
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return &crawler.ExtractedJob{
|
||||
CanonicalID: fmt.Sprintf("bytedance:%s", jobID),
|
||||
Title: "Software Engineer - ByteDance",
|
||||
Company: "ByteDance",
|
||||
Location: "Seattle / Bellevue, WA",
|
||||
ApplyURL: applyURL,
|
||||
RawContent: "See job posting at " + applyURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseByteDanceJobID(u string) string {
|
||||
re := regexp.MustCompile(`/position/([0-9A-Za-z]+)`)
|
||||
m := re.FindStringSubmatch(u)
|
||||
if len(m) >= 2 {
|
||||
return m[1]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func cleanHTML(s string) string {
|
||||
s = strings.ReplaceAll(s, "<b>", "**")
|
||||
s = strings.ReplaceAll(s, "</b>", "**")
|
||||
s = strings.ReplaceAll(s, "<strong>", "**")
|
||||
s = strings.ReplaceAll(s, "</strong>", "**")
|
||||
s = strings.ReplaceAll(s, "<br>", "\n")
|
||||
s = strings.ReplaceAll(s, "<br/>", "\n")
|
||||
s = strings.ReplaceAll(s, "<br />", "\n")
|
||||
s = strings.ReplaceAll(s, "<li>", "- ")
|
||||
s = strings.ReplaceAll(s, "</li>", "\n")
|
||||
s = strings.ReplaceAll(s, "<ul>", "\n")
|
||||
s = strings.ReplaceAll(s, "</ul>", "\n")
|
||||
s = strings.ReplaceAll(s, "<p>", "\n")
|
||||
s = strings.ReplaceAll(s, "</p>", "\n")
|
||||
reTags := regexp.MustCompile(`<[^>]*>`)
|
||||
s = reTags.ReplaceAllString(s, "")
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
Reference in New Issue
Block a user