haopengzhan/rsspipe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: implement rule engine, worker pool, event bus, and readability scraping with enhanced logging and content sanitization.

Browse Source
This commit is contained in:
haopengzhan
2026-08-25 00:31:32 -07:00
parent 5ea176d5ad
commit d440728ca5
29 changed files with 2788 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/crawler/crawler.go
+43 -14
View File
@@ -3,35 +3,41 @@ package crawler
import (
"context"
"fmt"
"log"
"sync"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/events"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/parser"
"go.uber.org/zap"
)
type Crawler struct {
db *db.DB
parser *parser.Parser
bus *events.EventBus
intervalMinutes int
workerCount int
isRefreshing bool
mu sync.Mutex
log *zap.Logger
}
func NewCrawler(database *db.DB, intervalMinutes, workers int) *Crawler {
func NewCrawler(database *db.DB, bus *events.EventBus, intervalMinutes, workers int) *Crawler {
return &Crawler{
db: database,
parser: parser.NewParser(),
bus: bus,
intervalMinutes: intervalMinutes,
workerCount: workers,
log: logger.Named("crawler"),
}
}
func (c *Crawler) Start(ctx context.Context) {
log.Printf("==> Starting RSS background crawler scheduler (Workers: %d)\n", c.workerCount)
c.log.Info("Starting RSS background crawler scheduler", zap.Int("worker_count", c.workerCount))
// Run initial fetch asynchronously after 2 seconds startup
go func() {
@@ -72,7 +78,7 @@ func (c *Crawler) RefreshDue(ctx context.Context) {
feeds, err := c.db.ListFeeds()
if err != nil {
log.Printf("Crawler error listing feeds: %v\n", err)
c.log.Error("Crawler error listing feeds", zap.Error(err))
return
}
@@ -93,7 +99,7 @@ func (c *Crawler) RefreshDue(ctx context.Context) {
return
}
log.Printf("Crawler: Refreshing %d due feeds (out of %d total)...\n", len(dueFeeds), len(feeds))
c.log.Info("Crawler refreshing due feeds", zap.Int("due_count", len(dueFeeds)), zap.Int("total_count", len(feeds)))
c.runWorkerPool(ctx, dueFeeds)
}
@@ -102,7 +108,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) {
c.mu.Lock()
if c.isRefreshing {
c.mu.Unlock()
log.Println("Crawler: RefreshAll already in progress, skipping")
c.log.Debug("Crawler RefreshAll already in progress, skipping")
return
}
c.isRefreshing = true
@@ -116,7 +122,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) {
feeds, err := c.db.ListFeeds()
if err != nil {
log.Printf("Crawler error listing feeds: %v\n", err)
c.log.Error("Crawler error listing feeds", zap.Error(err))
return
}
@@ -124,7 +130,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) {
return
}
log.Printf("Crawler: Refreshing all %d feeds...\n", len(feeds))
c.log.Info("Crawler refreshing all feeds", zap.Int("count", len(feeds)))
c.runWorkerPool(ctx, feeds)
}
@@ -148,7 +154,7 @@ func (c *Crawler) runWorkerPool(ctx context.Context, feeds []models.Feed) {
close(jobs)
wg.Wait()
log.Printf("Crawler: Completed refresh of %d feeds.\n", len(feeds))
c.log.Info("Crawler completed refresh batch", zap.Int("feed_count", len(feeds)))
}
func (c *Crawler) RefreshSingle(ctx context.Context, feedID string) error {
@@ -169,7 +175,11 @@ func (c *Crawler) fetchSingleFeed(ctx context.Context, f models.Feed) error {
res, err := c.parser.FetchFeed(feedCtx, f.FeedURL, f.ETag, f.LastModified)
if err != nil {
log.Printf("Crawler error fetching [%s]: %v\n", f.Title, err)
c.log.Warn("Crawler error fetching feed",
zap.String("feed_title", f.Title),
zap.String("feed_url", f.FeedURL),
zap.Error(err),
)
_ = c.db.UpdateFeedFetchStatus(f.ID, f.ETag, f.LastModified, err.Error())
return err
}
@@ -180,11 +190,30 @@ func (c *Crawler) fetchSingleFeed(ctx context.Context, f models.Feed) error {
}
// Upsert entries
inserted, err := c.db.UpsertFeedEntries(f.ID, res.Entries)
newEntries, err := c.db.UpsertFeedEntries(f.ID, res.Entries)
if err != nil {
log.Printf("Crawler error upserting entries for [%s]: %v\n", f.Title, err)
} else if inserted > 0 {
log.Printf("Crawler [%s]: inserted %d new entries\n", f.Title, inserted)
c.log.Error("Crawler error upserting entries",
zap.String("feed_title", f.Title),
zap.Error(err),
)
} else if len(newEntries) > 0 {
c.log.Info("Crawler inserted new entries",
zap.String("feed_title", f.Title),
zap.Int("new_entries", len(newEntries)),
)
// Publish on_post_created events for newly inserted entries
if c.bus != nil {
for _, entry := range newEntries {
entry.FeedTitle = f.Title
entry.FeedSiteURL = f.SiteURL
c.bus.Publish(ctx, events.Event{
Type: events.EventPostCreated,
Entry: entry,
Depth: 0,
})
}
}
}
_ = c.db.UpdateFeedFetchStatus(f.ID, res.ETag, res.LastModified, "")