feat: implement rule engine, worker pool, event bus, and readability scraping with enhanced logging and content sanitization.

internal/db/db.go

@@ -114,15 +114,28 @@ func (d *DB) migrate() error {
 	CREATE INDEX IF NOT EXISTS idx_entries_is_starred ON feed_entries(is_starred);
 	CREATE INDEX IF NOT EXISTS idx_entries_share_uuid ON feed_entries(share_uuid);
 
+	CREATE TABLE IF NOT EXISTS entry_variants (
+		entry_id TEXT NOT NULL,
+		variant_key TEXT NOT NULL,
+		content_type TEXT NOT NULL,
+		text_value TEXT,
+		blob_path TEXT,
+		metadata TEXT DEFAULT '{}',
+		created_at DATETIME NOT NULL,
+		updated_at DATETIME NOT NULL,
+		PRIMARY KEY (entry_id, variant_key),
+		FOREIGN KEY(entry_id) REFERENCES feed_entries(id) ON DELETE CASCADE
+	);
+	CREATE INDEX IF NOT EXISTS idx_entry_variants_key ON entry_variants(variant_key);
+
 	CREATE TABLE IF NOT EXISTS rules (
 		id TEXT PRIMARY KEY,
 		name TEXT NOT NULL,
-		rule_type TEXT NOT NULL,
-		is_active BOOLEAN DEFAULT 1,
-		trigger_type TEXT NOT NULL,
-		trigger_config TEXT DEFAULT '{}',
+		trigger TEXT NOT NULL,
 		conditions TEXT DEFAULT '[]',
+		match_mode TEXT DEFAULT 'all',
 		actions TEXT DEFAULT '[]',
+		is_active BOOLEAN DEFAULT 1,
 		created_at DATETIME NOT NULL,
 		updated_at DATETIME NOT NULL
 	);
@@ -132,6 +145,26 @@ func (d *DB) migrate() error {
 	}
 	// Migration for timeout_seconds column if older DB exists
 	_, _ = d.Exec("ALTER TABLE feeds ADD COLUMN timeout_seconds INTEGER DEFAULT 10")
+
+	// Migration for rules table schema if older table exists
+	var testCol string
+	err := d.QueryRow("SELECT trigger FROM rules LIMIT 1").Scan(&testCol)
+	if err != nil && err != sql.ErrNoRows {
+		_, _ = d.Exec("DROP TABLE IF EXISTS rules")
+		_, _ = d.Exec(`
+			CREATE TABLE IF NOT EXISTS rules (
+				id TEXT PRIMARY KEY,
+				name TEXT NOT NULL,
+				trigger TEXT NOT NULL,
+				conditions TEXT DEFAULT '[]',
+				match_mode TEXT DEFAULT 'all',
+				actions TEXT DEFAULT '[]',
+				is_active BOOLEAN DEFAULT 1,
+				created_at DATETIME NOT NULL,
+				updated_at DATETIME NOT NULL
+			);
+		`)
+	}
 	return nil
 }
 
@@ -474,51 +507,67 @@ func (d *DB) DeleteFeed(id string) error {
 }
 
 // Feed Entries operations
-func (d *DB) UpsertFeedEntries(feedID string, entries []models.FeedEntry) (int, error) {
+func (d *DB) UpsertFeedEntries(feedID string, entries []models.FeedEntry) ([]models.FeedEntry, error) {
 	if len(entries) == 0 {
-		return 0, nil
+		return nil, nil
 	}
 
-	stmt, err := d.Prepare(`
+	var newEntries []models.FeedEntry
+	now := time.Now().UTC()
+
+	insertStmt, err := d.Prepare(`
 		INSERT INTO feed_entries (
 			id, feed_id, guid, url, title, author, content, raw_content, published_at, is_read, is_starred, metadata, created_at
 		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?, ?)
-		ON CONFLICT(feed_id, guid) DO UPDATE SET
-			title = excluded.title,
-			content = CASE WHEN feed_entries.content = '' THEN excluded.content ELSE feed_entries.content END,
-			url = excluded.url,
-			author = excluded.author
 	`)
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
 	defer func() {
-		_ = stmt.Close()
+		_ = insertStmt.Close()
 	}()
 
-	insertedCount := 0
-	for _, e := range entries {
-		id := uuid.New().String()
-		now := time.Now().UTC()
-		pubAt := e.PublishedAt
-		if pubAt.IsZero() {
-			pubAt = now
-		}
-		metaBytes, _ := json.Marshal(e.Metadata)
-		if len(metaBytes) == 0 {
-			metaBytes = []byte("{}")
-		}
+	updateStmt, err := d.Prepare(`
+		UPDATE feed_entries 
+		SET title = ?, url = ?, author = ?, content = ?, raw_content = ?
+		WHERE feed_id = ? AND guid = ?
+	`)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = updateStmt.Close()
+	}()
 
-		res, err := stmt.Exec(
-			id, feedID, e.GUID, e.URL, e.Title, e.Author, e.Content, e.RawContent, pubAt, string(metaBytes), now,
-		)
-		if err == nil {
-			if n, _ := res.RowsAffected(); n > 0 {
-				insertedCount++
+	for _, e := range entries {
+		var existingID string
+		err := d.QueryRow("SELECT id FROM feed_entries WHERE feed_id = ? AND guid = ?", feedID, e.GUID).Scan(&existingID)
+		switch err {
+		case sql.ErrNoRows:
+			e.ID = uuid.New().String()
+			e.FeedID = feedID
+			pubAt := e.PublishedAt
+			if pubAt.IsZero() {
+				pubAt = now
 			}
+			metaBytes, _ := json.Marshal(e.Metadata)
+			if len(metaBytes) == 0 {
+				metaBytes = []byte("{}")
+			}
+
+			_, err := insertStmt.Exec(
+				e.ID, feedID, e.GUID, e.URL, e.Title, e.Author, e.Content, e.RawContent, pubAt, string(metaBytes), now,
+			)
+			if err == nil {
+				e.CreatedAt = now
+				newEntries = append(newEntries, e)
+			}
+		case nil:
+			_, _ = updateStmt.Exec(e.Title, e.URL, e.Author, e.Content, e.RawContent, feedID, e.GUID)
 		}
 	}
-	return insertedCount, nil
+
+	return newEntries, nil
 }
 
 type EntryFilter struct {
@@ -530,67 +579,98 @@ type EntryFilter struct {
 	Offset     int
 }
 
-func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) {
-	where := []string{"1=1"}
-	args := []interface{}{}
+func (d *DB) GetEntry(id string) (*models.FeedEntry, error) {
+	query := `
+	SELECT 
+		e.id, e.feed_id, f.title as feed_title, COALESCE(f.site_url, '') as feed_site_url,
+		e.guid, e.url, e.title, e.author, e.content, e.raw_content,
+		e.published_at, e.is_read, e.is_starred, e.share_uuid, e.metadata, e.created_at
+	FROM feed_entries e
+	JOIN feeds f ON f.id = e.feed_id
+	WHERE e.id = ?
+	`
+	var e models.FeedEntry
+	var author, rawContent, shareUUID, metaStr sql.NullString
+	err := d.QueryRow(query, id).Scan(
+		&e.ID, &e.FeedID, &e.FeedTitle, &e.FeedSiteURL,
+		&e.GUID, &e.URL, &e.Title, &author, &e.Content, &rawContent,
+		&e.PublishedAt, &e.IsRead, &e.IsStarred, &shareUUID, &metaStr, &e.CreatedAt,
+	)
+	if err != nil {
+		return nil, err
+	}
+	if author.Valid {
+		e.Author = author.String
+	}
+	if rawContent.Valid {
+		e.RawContent = rawContent.String
+	}
+	if shareUUID.Valid {
+		s := shareUUID.String
+		e.ShareUUID = &s
+	}
+	e.Metadata = make(map[string]interface{})
+	if metaStr.Valid && metaStr.String != "" {
+		_ = json.Unmarshal([]byte(metaStr.String), &e.Metadata)
+	}
 
-	switch f.View {
+	// Load variants
+	variants, _ := d.ListVariantsForEntry(e.ID)
+	e.Variants = variants
+
+	return &e, nil
+}
+
+func (d *DB) ListEntries(filter EntryFilter) ([]models.FeedEntry, int, error) {
+	var whereClauses []string
+	var args []interface{}
+
+	switch filter.View {
 	case "inbox":
-		where = append(where, "e.is_read = 0")
+		whereClauses = append(whereClauses, "e.is_read = 0")
 	case "starred":
-		where = append(where, "e.is_starred = 1")
+		whereClauses = append(whereClauses, "e.is_starred = 1")
 	}
 
-	if f.FeedID != "" {
-		where = append(where, "e.feed_id = ?")
-		args = append(args, f.FeedID)
+	if filter.FeedID != "" {
+		whereClauses = append(whereClauses, "e.feed_id = ?")
+		args = append(args, filter.FeedID)
 	}
 
-	if f.CategoryID != "" {
-		where = append(where, "f.category_id = ?")
-		args = append(args, f.CategoryID)
+	if filter.CategoryID != "" {
+		whereClauses = append(whereClauses, "f.category_id = ?")
+		args = append(args, filter.CategoryID)
 	}
 
-	if f.Search != "" {
-		searchTerms := strings.Fields(f.Search)
-		for _, term := range searchTerms {
-			lower := strings.ToLower(term)
-			if strings.HasPrefix(lower, "is:unread") {
-				where = append(where, "e.is_read = 0")
-			} else if strings.HasPrefix(lower, "is:read") {
-				where = append(where, "e.is_read = 1")
-			} else if strings.HasPrefix(lower, "is:starred") {
-				where = append(where, "e.is_starred = 1")
-			} else if strings.HasPrefix(lower, "feed:") {
-				feedVal := strings.TrimPrefix(term, "feed:")
-				where = append(where, "(f.title LIKE ? OR f.feed_url LIKE ?)")
-				args = append(args, "%"+feedVal+"%", "%"+feedVal+"%")
-			} else {
-				where = append(where, "(e.title LIKE ? OR e.content LIKE ? OR e.author LIKE ?)")
-				args = append(args, "%"+term+"%", "%"+term+"%", "%"+term+"%")
-			}
-		}
+	if filter.Search != "" {
+		whereClauses = append(whereClauses, "(e.title LIKE ? OR e.content LIKE ? OR e.author LIKE ?)")
+		pattern := "%" + filter.Search + "%"
+		args = append(args, pattern, pattern, pattern)
 	}
 
-	whereClause := strings.Join(where, " AND ")
+	whereSQL := ""
+	if len(whereClauses) > 0 {
+		whereSQL = "WHERE " + strings.Join(whereClauses, " AND ")
+	}
 
-	// Count total
 	countQuery := fmt.Sprintf(`
-		SELECT COUNT(*) 
-		FROM feed_entries e 
-		JOIN feeds f ON f.id = e.feed_id 
-		WHERE %s
-	`, whereClause)
+		SELECT COUNT(e.id)
+		FROM feed_entries e
+		JOIN feeds f ON f.id = e.feed_id
+		%s
+	`, whereSQL)
+
 	var total int
-	if err := d.QueryRow(countQuery, args...).Scan(&total); err != nil {
+	err := d.QueryRow(countQuery, args...).Scan(&total)
+	if err != nil {
 		return nil, 0, err
 	}
 
-	limit := f.Limit
-	if limit <= 0 || limit > 100 {
-		limit = 50
+	limit := filter.Limit
+	if limit <= 0 || limit > 500 {
+		limit = 100
 	}
-	offset := f.Offset
+	offset := filter.Offset
 	if offset < 0 {
 		offset = 0
 	}
@@ -602,10 +682,10 @@ func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) {
 			e.published_at, e.is_read, e.is_starred, e.share_uuid, e.metadata, e.created_at
 		FROM feed_entries e
 		JOIN feeds f ON f.id = e.feed_id
-		WHERE %s
-		ORDER BY e.published_at DESC
+		%s
+		ORDER BY e.published_at DESC, e.created_at DESC
 		LIMIT ? OFFSET ?
-	`, whereClause)
+	`, whereSQL)
 
 	queryArgs := append(args, limit, offset)
 	rows, err := d.Query(query, queryArgs...)
@@ -619,13 +699,13 @@ func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) {
 	var entries []models.FeedEntry
 	for rows.Next() {
 		var e models.FeedEntry
-		var author, rawContent, metaStr sql.NullString
-		var shareUUID sql.NullString
-		if err := rows.Scan(
+		var author, rawContent, shareUUID, metaStr sql.NullString
+		err := rows.Scan(
 			&e.ID, &e.FeedID, &e.FeedTitle, &e.FeedSiteURL,
 			&e.GUID, &e.URL, &e.Title, &author, &e.Content, &rawContent,
 			&e.PublishedAt, &e.IsRead, &e.IsStarred, &shareUUID, &metaStr, &e.CreatedAt,
-		); err != nil {
+		)
+		if err != nil {
 			return nil, 0, err
 		}
 		if author.Valid {
@@ -675,6 +755,232 @@ func (d *DB) MarkAllRead(feedID, categoryID *string) error {
 	return err
 }
 
+// Entry Variants CRUD
+func (d *DB) GetVariant(entryID, key string) (*models.EntryVariant, error) {
+	query := `
+		SELECT entry_id, variant_key, content_type, COALESCE(text_value, ''), COALESCE(blob_path, ''), metadata, created_at, updated_at
+		FROM entry_variants
+		WHERE entry_id = ? AND variant_key = ?
+	`
+	var v models.EntryVariant
+	var metaStr sql.NullString
+	err := d.QueryRow(query, entryID, key).Scan(
+		&v.EntryID, &v.VariantKey, &v.ContentType, &v.TextValue, &v.BlobPath, &metaStr, &v.CreatedAt, &v.UpdatedAt,
+	)
+	if err != nil {
+		return nil, err
+	}
+	v.Metadata = make(map[string]interface{})
+	if metaStr.Valid && metaStr.String != "" {
+		_ = json.Unmarshal([]byte(metaStr.String), &v.Metadata)
+	}
+	return &v, nil
+}
+
+func (d *DB) ListVariantsForEntry(entryID string) (map[string]models.EntryVariant, error) {
+	query := `
+		SELECT entry_id, variant_key, content_type, COALESCE(text_value, ''), COALESCE(blob_path, ''), metadata, created_at, updated_at
+		FROM entry_variants
+		WHERE entry_id = ?
+	`
+	rows, err := d.Query(query, entryID)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = rows.Close()
+	}()
+
+	variants := make(map[string]models.EntryVariant)
+	for rows.Next() {
+		var v models.EntryVariant
+		var metaStr sql.NullString
+		err := rows.Scan(
+			&v.EntryID, &v.VariantKey, &v.ContentType, &v.TextValue, &v.BlobPath, &metaStr, &v.CreatedAt, &v.UpdatedAt,
+		)
+		if err != nil {
+			return nil, err
+		}
+		v.Metadata = make(map[string]interface{})
+		if metaStr.Valid && metaStr.String != "" {
+			_ = json.Unmarshal([]byte(metaStr.String), &v.Metadata)
+		}
+		variants[v.VariantKey] = v
+	}
+	return variants, nil
+}
+
+func (d *DB) SetVariant(v models.EntryVariant) error {
+	now := time.Now().UTC()
+	metaBytes, _ := json.Marshal(v.Metadata)
+	if len(metaBytes) == 0 {
+		metaBytes = []byte("{}")
+	}
+
+	query := `
+		INSERT INTO entry_variants (
+			entry_id, variant_key, content_type, text_value, blob_path, metadata, created_at, updated_at
+		) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+		ON CONFLICT(entry_id, variant_key) DO UPDATE SET
+			content_type = excluded.content_type,
+			text_value = excluded.text_value,
+			blob_path = excluded.blob_path,
+			metadata = excluded.metadata,
+			updated_at = excluded.updated_at
+	`
+	_, err := d.Exec(query, v.EntryID, v.VariantKey, v.ContentType, v.TextValue, v.BlobPath, string(metaBytes), now, now)
+	return err
+}
+
+func (d *DB) DeleteVariant(entryID, key string) error {
+	_, err := d.Exec("DELETE FROM entry_variants WHERE entry_id = ? AND variant_key = ?", entryID, key)
+	return err
+}
+
+// Rules CRUD
+func (d *DB) ListRules() ([]models.Rule, error) {
+	query := `
+		SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at
+		FROM rules
+		ORDER BY created_at ASC
+	`
+	rows, err := d.Query(query)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = rows.Close()
+	}()
+
+	var rules []models.Rule
+	for rows.Next() {
+		var r models.Rule
+		var condsStr, actionsStr string
+		err := rows.Scan(
+			&r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt,
+		)
+		if err != nil {
+			return nil, err
+		}
+		_ = json.Unmarshal([]byte(condsStr), &r.Conditions)
+		_ = json.Unmarshal([]byte(actionsStr), &r.Actions)
+		rules = append(rules, r)
+	}
+	return rules, nil
+}
+
+func (d *DB) GetActiveRules(trigger string) ([]models.Rule, error) {
+	query := `
+		SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at
+		FROM rules
+		WHERE is_active = 1 AND (trigger = ? OR trigger = '*')
+		ORDER BY created_at ASC
+	`
+	rows, err := d.Query(query, trigger)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = rows.Close()
+	}()
+
+	var rules []models.Rule
+	for rows.Next() {
+		var r models.Rule
+		var condsStr, actionsStr string
+		err := rows.Scan(
+			&r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt,
+		)
+		if err != nil {
+			return nil, err
+		}
+		_ = json.Unmarshal([]byte(condsStr), &r.Conditions)
+		_ = json.Unmarshal([]byte(actionsStr), &r.Actions)
+		rules = append(rules, r)
+	}
+	return rules, nil
+}
+
+func (d *DB) GetRule(id string) (*models.Rule, error) {
+	query := `
+		SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at
+		FROM rules
+		WHERE id = ?
+	`
+	var r models.Rule
+	var condsStr, actionsStr string
+	err := d.QueryRow(query, id).Scan(
+		&r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt,
+	)
+	if err != nil {
+		return nil, err
+	}
+	_ = json.Unmarshal([]byte(condsStr), &r.Conditions)
+	_ = json.Unmarshal([]byte(actionsStr), &r.Actions)
+	return &r, nil
+}
+
+func (d *DB) CreateRule(name, trigger, matchMode string, conditions []models.RuleCondition, actions []models.RuleAction) (*models.Rule, error) {
+	id := uuid.New().String()
+	now := time.Now().UTC()
+	if matchMode == "" {
+		matchMode = "all"
+	}
+	condsBytes, _ := json.Marshal(conditions)
+	actionsBytes, _ := json.Marshal(actions)
+
+	query := `
+		INSERT INTO rules (
+			id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at
+		) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?)
+	`
+	_, err := d.Exec(query, id, name, trigger, string(condsBytes), matchMode, string(actionsBytes), now, now)
+	if err != nil {
+		return nil, err
+	}
+
+	return &models.Rule{
+		ID:         id,
+		Name:       name,
+		Trigger:    trigger,
+		Conditions: conditions,
+		MatchMode:  matchMode,
+		Actions:    actions,
+		IsActive:   true,
+		CreatedAt:  now,
+		UpdatedAt:  now,
+	}, nil
+}
+
+func (d *DB) UpdateRule(id string, name, trigger, matchMode string, conditions []models.RuleCondition, actions []models.RuleAction, isActive bool) (*models.Rule, error) {
+	now := time.Now().UTC()
+	if matchMode == "" {
+		matchMode = "all"
+	}
+	condsBytes, _ := json.Marshal(conditions)
+	actionsBytes, _ := json.Marshal(actions)
+
+	query := `
+		UPDATE rules
+		SET name = ?, trigger = ?, conditions = ?, match_mode = ?, actions = ?, is_active = ?, updated_at = ?
+		WHERE id = ?
+	`
+	res, err := d.Exec(query, name, trigger, string(condsBytes), matchMode, string(actionsBytes), isActive, now, id)
+	if err != nil {
+		return nil, err
+	}
+	if n, _ := res.RowsAffected(); n == 0 {
+		return nil, fmt.Errorf("rule not found")
+	}
+
+	return d.GetRule(id)
+}
+
+func (d *DB) DeleteRule(id string) error {
+	_, err := d.Exec("DELETE FROM rules WHERE id = ?", id)
+	return err
+}
+
 func (d *DB) GetStats() (*models.Stats, error) {
 	var s models.Stats
 	err := d.QueryRow("SELECT COUNT(*) FROM feeds").Scan(&s.TotalFeeds)
@@ -693,5 +999,6 @@ func (d *DB) GetStats() (*models.Stats, error) {
 	if err != nil {
 		return nil, err
 	}
+	_ = d.QueryRow("SELECT COUNT(*) FROM rules").Scan(&s.TotalRules)
 	return &s, nil
 }