feat: implement rule engine, worker pool, event bus, and readability scraping with enhanced logging and content sanitization.

internal/actions/readability/readability.go

@@ -0,0 +1,123 @@
+package readability
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
+	"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
+	"git.pengzhan.dev/haopengzhan/rsspipe/internal/worker"
+	readability "github.com/go-shiori/go-readability"
+	"github.com/microcosm-cc/bluemonday"
+	"go.uber.org/zap"
+)
+
+type Scraper struct {
+	client    *http.Client
+	sanitizer *bluemonday.Policy
+	log       *zap.Logger
+}
+
+func NewScraper(timeout time.Duration) *Scraper {
+	if timeout <= 0 {
+		timeout = 15 * time.Second
+	}
+
+	p := bluemonday.UGCPolicy()
+	// Allow rich media & responsive elements
+	p.AllowElements("video", "audio", "source", "track", "iframe", "picture", "figure", "figcaption", "article", "section")
+	p.AllowAttrs("src", "poster", "controls", "autoplay", "loop", "muted", "preload").OnElements("video", "audio")
+	p.AllowAttrs("src", "type", "srcset", "media").OnElements("source")
+	p.AllowAttrs("src", "width", "height", "frameborder", "allow", "allowfullscreen", "sandbox").OnElements("iframe")
+	p.AllowAttrs("data-src", "data-original", "loading", "srcset", "sizes").OnElements("img")
+	p.AllowAttrs("class", "id").Globally()
+
+	return &Scraper{
+		client: &http.Client{
+			Timeout: timeout,
+		},
+		sanitizer: p,
+		log:       logger.Named("readability-scraper"),
+	}
+}
+
+// Handler returns the worker.ActionHandler function for readability scraping.
+func (s *Scraper) Handler() worker.ActionHandler {
+	return func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*worker.ActionResult, error) {
+		if strings.TrimSpace(entry.URL) == "" {
+			return nil, fmt.Errorf("entry URL is empty")
+		}
+
+		parsedURL, err := url.Parse(entry.URL)
+		if err != nil {
+			return nil, fmt.Errorf("invalid URL: %w", err)
+		}
+
+		s.log.Info("Starting readability scrape",
+			zap.String("entry_id", entry.ID),
+			zap.String("url", entry.URL),
+		)
+
+		req, err := http.NewRequestWithContext(ctx, "GET", entry.URL, nil)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create request: %w", err)
+		}
+
+		req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
+		req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+		req.Header.Set("Accept-Language", "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7")
+
+		resp, err := s.client.Do(req)
+		if err != nil {
+			return nil, fmt.Errorf("failed to fetch article page: %w", err)
+		}
+		defer func() {
+			_ = resp.Body.Close()
+		}()
+
+		if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+			return nil, fmt.Errorf("server returned HTTP status %d", resp.StatusCode)
+		}
+
+		article, err := readability.FromReader(resp.Body, parsedURL)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse readability article: %w", err)
+		}
+
+		cleanHTML := s.sanitizer.Sanitize(article.Content)
+		if strings.TrimSpace(cleanHTML) == "" && strings.TrimSpace(article.TextContent) != "" {
+			cleanHTML = "<p>" + s.sanitizer.Sanitize(article.TextContent) + "</p>"
+		}
+
+		s.log.Info("Readability scrape successful",
+			zap.String("entry_id", entry.ID),
+			zap.String("title", article.Title),
+			zap.Int("content_length", len(cleanHTML)),
+		)
+
+		now := time.Now().UTC()
+		variant := models.EntryVariant{
+			EntryID:     entry.ID,
+			VariantKey:  "readability_full_text",
+			ContentType: "html",
+			TextValue:   cleanHTML,
+			Metadata: map[string]interface{}{
+				"title":      article.Title,
+				"byline":     article.Byline,
+				"excerpt":    article.Excerpt,
+				"length":     len(article.TextContent),
+				"scraped_at": now.Format(time.RFC3339),
+			},
+			CreatedAt: now,
+			UpdatedAt: now,
+		}
+
+		return &worker.ActionResult{
+			ResultVariant: &variant,
+		}, nil
+	}
+}