124 lines
3.7 KiB
Go
124 lines
3.7 KiB
Go
package readability
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
|
|
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
|
|
"git.pengzhan.dev/haopengzhan/rsspipe/internal/worker"
|
|
readability "github.com/go-shiori/go-readability"
|
|
"github.com/microcosm-cc/bluemonday"
|
|
"go.uber.org/zap"
|
|
)
|
|
|
|
type Scraper struct {
|
|
client *http.Client
|
|
sanitizer *bluemonday.Policy
|
|
log *zap.Logger
|
|
}
|
|
|
|
func NewScraper(timeout time.Duration) *Scraper {
|
|
if timeout <= 0 {
|
|
timeout = 15 * time.Second
|
|
}
|
|
|
|
p := bluemonday.UGCPolicy()
|
|
// Allow rich media & responsive elements
|
|
p.AllowElements("video", "audio", "source", "track", "iframe", "picture", "figure", "figcaption", "article", "section")
|
|
p.AllowAttrs("src", "poster", "controls", "autoplay", "loop", "muted", "preload").OnElements("video", "audio")
|
|
p.AllowAttrs("src", "type", "srcset", "media").OnElements("source")
|
|
p.AllowAttrs("src", "width", "height", "frameborder", "allow", "allowfullscreen", "sandbox").OnElements("iframe")
|
|
p.AllowAttrs("data-src", "data-original", "loading", "srcset", "sizes").OnElements("img")
|
|
p.AllowAttrs("class", "id").Globally()
|
|
|
|
return &Scraper{
|
|
client: &http.Client{
|
|
Timeout: timeout,
|
|
},
|
|
sanitizer: p,
|
|
log: logger.Named("readability-scraper"),
|
|
}
|
|
}
|
|
|
|
// Handler returns the worker.ActionHandler function for readability scraping.
|
|
func (s *Scraper) Handler() worker.ActionHandler {
|
|
return func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*worker.ActionResult, error) {
|
|
if strings.TrimSpace(entry.URL) == "" {
|
|
return nil, fmt.Errorf("entry URL is empty")
|
|
}
|
|
|
|
parsedURL, err := url.Parse(entry.URL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid URL: %w", err)
|
|
}
|
|
|
|
s.log.Info("Starting readability scrape",
|
|
zap.String("entry_id", entry.ID),
|
|
zap.String("url", entry.URL),
|
|
)
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "GET", entry.URL, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create request: %w", err)
|
|
}
|
|
|
|
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
|
|
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
|
req.Header.Set("Accept-Language", "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7")
|
|
|
|
resp, err := s.client.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to fetch article page: %w", err)
|
|
}
|
|
defer func() {
|
|
_ = resp.Body.Close()
|
|
}()
|
|
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return nil, fmt.Errorf("server returned HTTP status %d", resp.StatusCode)
|
|
}
|
|
|
|
article, err := readability.FromReader(resp.Body, parsedURL)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse readability article: %w", err)
|
|
}
|
|
|
|
cleanHTML := s.sanitizer.Sanitize(article.Content)
|
|
if strings.TrimSpace(cleanHTML) == "" && strings.TrimSpace(article.TextContent) != "" {
|
|
cleanHTML = "<p>" + s.sanitizer.Sanitize(article.TextContent) + "</p>"
|
|
}
|
|
|
|
s.log.Info("Readability scrape successful",
|
|
zap.String("entry_id", entry.ID),
|
|
zap.String("title", article.Title),
|
|
zap.Int("content_length", len(cleanHTML)),
|
|
)
|
|
|
|
now := time.Now().UTC()
|
|
variant := models.EntryVariant{
|
|
EntryID: entry.ID,
|
|
VariantKey: "readability_full_text",
|
|
ContentType: "html",
|
|
TextValue: cleanHTML,
|
|
Metadata: map[string]interface{}{
|
|
"title": article.Title,
|
|
"byline": article.Byline,
|
|
"excerpt": article.Excerpt,
|
|
"length": len(article.TextContent),
|
|
"scraped_at": now.Format(time.RFC3339),
|
|
},
|
|
CreatedAt: now,
|
|
UpdatedAt: now,
|
|
}
|
|
|
|
return &worker.ActionResult{
|
|
ResultVariant: &variant,
|
|
}, nil
|
|
}
|
|
}
|