33 lines
688 B
Go
33 lines
688 B
Go
package crawler
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"fmt"
|
|
)
|
|
|
|
type DiscoveredJob struct {
|
|
CanonicalID string
|
|
ApplyURL string
|
|
}
|
|
|
|
type ExtractedJob struct {
|
|
CanonicalID string
|
|
Title string
|
|
Company string
|
|
Location string
|
|
ApplyURL string
|
|
RawContent string
|
|
}
|
|
|
|
func GenerateCanonicalID(company, title, applyURL string) string {
|
|
data := fmt.Sprintf("%s|%s|%s", company, title, applyURL)
|
|
return fmt.Sprintf("%x", sha256.Sum256([]byte(data)))
|
|
}
|
|
|
|
type SiteCrawler interface {
|
|
CanHandle(platform string) bool
|
|
DiscoverJobs(ctx context.Context, targetURL string) ([]DiscoveredJob, error)
|
|
FetchJobDetail(ctx context.Context, applyURL string) (*ExtractedJob, error)
|
|
}
|