feat: implement rule engine, worker pool, event bus, and readability scraping with enhanced logging and content sanitization.

This commit is contained in:
2026-08-25 00:31:32 -07:00
parent 5ea176d5ad
commit d440728ca5
29 changed files with 2788 additions and 204 deletions
+4 -1
View File
@@ -8,9 +8,12 @@ description = "Global allowlist"
paths = [ paths = [
'''sample_subscriptions\.opml''', '''sample_subscriptions\.opml''',
'''docs/''', '''docs/''',
'''web/node_modules/''' '''web/node_modules/''',
'''web/dist/''',
'''data/'''
] ]
regexes = [ regexes = [
'''admin123456''', '''admin123456''',
'''example_jwt_secret_key''' '''example_jwt_secret_key'''
] ]
+35 -17
View File
@@ -2,52 +2,69 @@ package main
import ( import (
"context" "context"
"log"
"net/http" "net/http"
"os" "os"
"os/signal" "os/signal"
"syscall" "syscall"
"time" "time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/actions/readability"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/api" "git.pengzhan.dev/haopengzhan/rsspipe/internal/api"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/auth" "git.pengzhan.dev/haopengzhan/rsspipe/internal/auth"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/config" "git.pengzhan.dev/haopengzhan/rsspipe/internal/config"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/crawler" "git.pengzhan.dev/haopengzhan/rsspipe/internal/crawler"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db" "git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/events"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/rules"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/worker"
"go.uber.org/zap"
) )
func main() { func main() {
log.Println("==================================================") log := logger.InitLogger(os.Getenv("LOG_LEVEL"))
log.Println(" Starting RSSPipe Server ") log.Info("Starting RSSPipe Server")
log.Println("==================================================")
cfg := config.LoadConfig() cfg := config.LoadConfig()
// 1. Initialize SQLite Database // 1. Initialize SQLite Database
database, err := db.InitDB(cfg.DBPath) database, err := db.InitDB(cfg.DBPath)
if err != nil { if err != nil {
log.Fatalf("Fatal: Database initialization failed: %v", err) log.Fatal("Database initialization failed", zap.Error(err))
} }
defer func() { defer func() {
_ = database.Close() _ = database.Close()
}() }()
log.Printf("==> Database connected: %s (WAL mode enabled)\n", cfg.DBPath) log.Info("Database connected", zap.String("path", cfg.DBPath))
// 2. Initialize Auth Service & Bootstrap Admin // 2. Initialize Auth Service & Bootstrap Admin
authService := auth.NewAuthService(database, cfg) authService := auth.NewAuthService(database, cfg)
if err := authService.BootstrapAdmin(); err != nil { if err := authService.BootstrapAdmin(); err != nil {
log.Fatalf("Fatal: Admin bootstrap failed: %v", err) log.Fatal("Admin bootstrap failed", zap.Error(err))
} }
// 3. Start Background Crawler Pool
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
cr := crawler.NewCrawler(database, cfg.CrawlerIntervalMinutes, cfg.CrawlerWorkers) // 3. Initialize EventBus, WorkerPool, and Action Handlers
bus := events.NewEventBus()
pool := worker.NewPool(database, bus, 5, 100)
// Register Scraper Action
scraper := readability.NewScraper(15 * time.Second)
pool.RegisterHandler("readability_scrape", scraper.Handler())
pool.Start(ctx)
// 4. Initialize Rules Engine
engine := rules.NewEngine(database, bus, pool)
engine.Start(ctx)
// 5. Start Background Crawler Pool
cr := crawler.NewCrawler(database, bus, cfg.CrawlerIntervalMinutes, cfg.CrawlerWorkers)
cr.Start(ctx) cr.Start(ctx)
// 4. Mount API Routes // 6. Mount API Routes
handler := api.NewHandler(database, authService, cr) handler := api.NewHandler(database, authService, cr, pool)
router := api.NewRouter(handler, authService, "./web/dist") router := api.NewRouter(handler, authService, "./web/dist")
server := &http.Server{ server := &http.Server{
@@ -58,26 +75,27 @@ func main() {
IdleTimeout: 120 * time.Second, IdleTimeout: 120 * time.Second,
} }
// 5. Graceful shutdown handler // 7. Graceful shutdown handler
go func() { go func() {
sigChan := make(chan os.Signal, 1) sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
<-sigChan <-sigChan
log.Println("\n==> Shutting down RSSPipe gracefully...") log.Info("Shutting down RSSPipe gracefully...")
cancel() cancel()
pool.Stop()
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer shutdownCancel() defer shutdownCancel()
if err := server.Shutdown(shutdownCtx); err != nil { if err := server.Shutdown(shutdownCtx); err != nil {
log.Printf("Error during server shutdown: %v\n", err) log.Error("Error during server shutdown", zap.Error(err))
} }
}() }()
log.Printf("==> RSSPipe HTTP Server listening at http://localhost:%s\n", cfg.Port) log.Info("RSSPipe HTTP Server listening", zap.String("url", "http://localhost:"+cfg.Port))
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("Fatal: Server error: %v", err) log.Fatal("Server error", zap.Error(err))
} }
log.Println("==> RSSPipe Server stopped.") log.Info("RSSPipe Server stopped")
} }
+39
View File
@@ -93,3 +93,42 @@ This document records planned features, UX enhancements, and architectural exten
- Staggered dynamic aspect ratios with prominent media cards. - Staggered dynamic aspect ratios with prominent media cards.
- Smooth infinite scroll pagination. - Smooth infinite scroll pagination.
- Interactive gesture swipe to dismiss the full-screen reader. - Interactive gesture swipe to dismiss the full-screen reader.
---
## 10. Beautiful 404 Error Page
- **Goal**: Provide a polished, modern, brand-consistent 404 Not Found experience.
- **Scope**:
- Frontend 404 route component with sleek modern aesthetics (glassmorphism/minimal illustration, clear error description, and a "Return to Gateway" action button).
- Server-side fallback handler returning a clean, branded HTML response for unmatched routes.
---
## 11. Multi-Variant Pipeline Enrichments (AI Summary, TTS, Keywords, Audio)
- **Goal**: Expand the event-driven rules engine and variant storage system with automated enrichment plugins.
- **Scope**:
- **AI Summary Hook**: Integrate LLM actions (e.g. OpenAI / Anthropic / Gemini / Ollama) to automatically generate and store concise summaries in `entry_variants(key='ai_summary')`.
- **Keyword & Tag Extraction**: Automated extractor storing structured YAML/JSON keyword lists in `entry_variants(key='keywords')` and `entry_variants(key='tags')` to enable rule-based chaining.
- **Text-to-Speech (TTS)**: Synthesize article text to voice audio blobs (`.mp3` stored under `./data/blobs/{entry_id}/tts.mp3`) with HTTP 206 partial streaming.
- **Podcast TTS & Summary TTS**: Convert articles into dual-host conversational podcasts or audio-narrated summaries streamed directly within the reader.
---
## 12. Aggregated Shared Posts Dashboard (`/shares`)
- **Goal**: Provide a dedicated gallery view aggregating all publicly shared articles.
- **Scope**:
- Centralized dashboard listing all articles with an active `share_uuid`.
- Copy public link, preview shared HTML canvas, revoke share link, and view access stats.
---
## 13. Data Retention & Cleanup Policies (Auto-Deletion)
- **Goal**: Provide configurable retention rules to prune older or noisy articles and keep SQLite storage compact.
- **Scope**:
- Retention triggers (e.g. delete unstarred articles older than 90 days, or limit feeds to latest 500 posts).
- Background garbage collection cleaning associated `entry_variants` and on-disk audio blobs (`./data/blobs/{entry_id}/`).
+116
View File
@@ -0,0 +1,116 @@
# RSSPipe Variant Data & Extensible Processing Pipeline Architecture
## 1. Overview & Vision
RSSPipe is designed as an **RSS-Driven Information Gateway**. Subscriptions come in many formats (full-text blogs, brief news tickers, media podcasts, academic papers). Consequently, a one-size-fits-all processing approach fails:
- Some feeds require automatic full-text extraction via Readability.
- Some feeds require LLM keyword and topic tagging.
- Specific high-value posts (filtered by keywords or feeds) require AI summarization, translation, or Text-to-Speech (TTS) audio synthesis.
To support this cleanly without database corruption, performance degradation, or rigid schemas, RSSPipe adopts a **Variant Data Architecture** paired with a **Reactive Chained Trigger Pipeline**.
---
## 2. Core Data Model
### 2.1 Separation of Immutable Source vs. Enriched Variants
1. **`feed_entries` (Immutable Source of Truth)**:
- Stores the exact metadata and content parsed from the RSS XML feed.
- Core fields: `id`, `feed_id`, `guid`, `url`, `title`, `author`, `rss_content`, `published_at`, `is_read`, `is_starred`, `created_at`.
- **Rule**: `rss_content` is NEVER overwritten by scrapers, LLMs, or TTS engines.
2. **`entry_variants` (Extensible Key-Value Store)**:
- Stores all downstream enrichments as strongly typed variant entries.
- Schema:
```sql
CREATE TABLE IF NOT EXISTS entry_variants (
entry_id TEXT NOT NULL,
variant_key TEXT NOT NULL,
content_type TEXT NOT NULL, -- 'text', 'html', 'json', 'yaml', 'audio/mp3', 'binary'
text_value TEXT, -- UTF-8 text, HTML, JSON array/object, YAML string
blob_path TEXT, -- Relative filesystem path for large media (e.g. data/blobs/{entry_id}/{key}.mp3)
metadata JSON DEFAULT '{}', -- Stats: model, tokens, duration_ms, byte_size, executed_at
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (entry_id, variant_key),
FOREIGN KEY (entry_id) REFERENCES feed_entries(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_entry_variants_key ON entry_variants(variant_key);
```
---
## 3. Supported Variant Keys & Types
| Variant Key | Content Type | Storage Location | Description |
| :--- | :--- | :--- | :--- |
| `readability_full_text` | `html` | `text_value` | Sanitized readability full text extracted from the source web page. |
| `keywords` | `json` / `yaml` | `text_value` | List of extracted keywords (e.g., `["AI", "Go", "Compiler"]`). |
| `tags` | `json` | `text_value` | Category and topic tags. |
| `ai_summary` | `text` / `markdown` | `text_value` | Bulleted or narrative AI executive summary. |
| `translation_{lang}` | `html` / `text` | `text_value` | Translated content (e.g. `translation_zh`). |
| `tts` | `audio/mp3` | `blob_path` | Synthesized speech audio file of the article. |
| `podcast_tts` | `audio/mp3` | `blob_path` | Multi-speaker podcast discussion audio file. |
| `ai_summary_tts` | `audio/mp3` | `blob_path` | Audio narration of the AI summary. |
---
## 4. Large Binary Blob Strategy
Storing multi-megabyte audio files or binary blobs directly inside SQLite columns degrades query cache efficiency and slows down table scans.
- **Text & JSON Variants** are stored inline in `entry_variants.text_value`.
- **Audio & Media Blobs** are stored on disk under `./data/blobs/{entry_id}/{variant_key}.{ext}`.
- `entry_variants.blob_path` stores the relative path.
- The API provides a streaming endpoint with HTTP Range support: `GET /api/entries/{id}/variants/{key}/stream`.
---
## 5. Reactive Chained Trigger & Watch Pipeline
The pipeline operates on typed pub/sub events with depth guards to prevent infinite loops:
```mermaid
graph TD
A[Feed Ingestion / Crawler] -->|Event: on_post_created| Bus[Typed Event Bus]
Bus --> Rules[Rules Engine Evaluator]
Rules -->|Matches Rule 1: Scrape Tech| W1[Worker: Readability Scraper]
W1 -->|Writes variant: readability_full_text| DB[(entry_variants)]
W1 -->|Event: on_variant_updated| Bus
Rules -->|Matches Rule 2: Tag AI Keywords| W2[Worker: Keyword Extractor]
W2 -->|Writes variant: keywords| DB
W2 -->|Event: on_variant_updated| Bus
Bus -->|Matches Rule 3: Watch keywords contains 'AI'| W3[Worker: AI Summary & TTS]
W3 -->|Writes variant: ai_summary & tts blob| DB
```
### 5.1 Concrete Chained Pipeline Example
1. **Trigger 1**: New post ingested from Hacker News -> Emits `on_post_created`.
2. **Rule 1 (Auto-Enrich Keywords)**:
- Condition: `feed_id == "hn-feed-id"`
- Action: `extract_keywords` -> Saves `entry_variants(key='keywords', text_value='["OpenAI", "LLM", "Agent"]')`.
- Emits: `on_variant_updated(key='keywords')`.
3. **Rule 2 (Watch & Synthesize Audio)**:
- Condition: `variant.keywords CONTAINS "OpenAI"`
- Action: `generate_tts` -> Synthesizes MP3, saves to `./data/blobs/...`, saves variant `tts`.
- Emits: `on_variant_updated(key='tts')`.
---
## 6. Reader UX & Content Switcher
In the 3-pane reader interface:
- **Default View**: Renders `rss_content` (original source).
- **Enrichment Pills**: If variants exist on the entry, dynamic tabs/pills appear in the reader header:
- `[📄 Original RSS]`
- `[⚡ Readability Full-Text]` (if `readability_full_text` exists)
- `[💡 AI Summary]` (if `ai_summary` exists)
- `[🎧 Listen Audio]` (if `tts` or `podcast_tts` exists, plays with embedded HTML5 player)
- Switching variants never mutates the underlying article or overwrites database fields.
+10
View File
@@ -3,16 +3,26 @@ module git.pengzhan.dev/haopengzhan/rsspipe
go 1.26.5 go 1.26.5
require ( require (
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-chi/chi/v5 v5.3.2 // indirect github.com/go-chi/chi/v5 v5.3.2 // indirect
github.com/go-chi/cors v1.2.2 // indirect github.com/go-chi/cors v1.2.2 // indirect
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect
github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0 // indirect
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/google/uuid v1.6.0 // indirect github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-isatty v0.0.24 // indirect
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
github.com/mmcdole/gofeed v1.4.2 // indirect github.com/mmcdole/gofeed v1.4.2 // indirect
github.com/mmcdole/goxpp/v2 v2.0.0 // indirect github.com/mmcdole/goxpp/v2 v2.0.0 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
go.uber.org/multierr v1.10.0 // indirect
go.uber.org/zap v1.28.0 // indirect
golang.org/x/crypto v0.55.0 // indirect golang.org/x/crypto v0.55.0 // indirect
golang.org/x/net v0.57.0 // indirect golang.org/x/net v0.57.0 // indirect
golang.org/x/sys v0.47.0 // indirect golang.org/x/sys v0.47.0 // indirect
+94
View File
@@ -1,31 +1,125 @@
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY= github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY=
github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE= github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0 h1:A3B75Yp163FAIf9nLlFMl4pwIj+T3uKxfI7mbvvY2Ls=
github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0/go.mod h1:suxK0Wpz4BM3/2+z1mnOVTIWHDiMCIOGoKDCRumSsk0=
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
github.com/mmcdole/gofeed v1.4.2 h1:XFFOtsNNZg+zudjtMXb8BI0J/YdnSIGfjEpPgPnZib0= github.com/mmcdole/gofeed v1.4.2 h1:XFFOtsNNZg+zudjtMXb8BI0J/YdnSIGfjEpPgPnZib0=
github.com/mmcdole/gofeed v1.4.2/go.mod h1:X5x1PyeibJi152VEya0AsV+PW4daYmCD4LJaJbeFkcs= github.com/mmcdole/gofeed v1.4.2/go.mod h1:X5x1PyeibJi152VEya0AsV+PW4daYmCD4LJaJbeFkcs=
github.com/mmcdole/goxpp/v2 v2.0.0 h1:HrSCflxerUEqZQNq3u7ldtmE/XkwnTx4Zpq2DW4i5rQ= github.com/mmcdole/goxpp/v2 v2.0.0 h1:HrSCflxerUEqZQNq3u7ldtmE/XkwnTx4Zpq2DW4i5rQ=
github.com/mmcdole/goxpp/v2 v2.0.0/go.mod h1:CUduYMnO9JB6Z/uqDn9Ormk/r8E9BsLQxHPWDZ961Os= github.com/mmcdole/goxpp/v2 v2.0.0/go.mod h1:CUduYMnO9JB6Z/uqDn9Ormk/r8E9BsLQxHPWDZ961Os=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k=
modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+123
View File
@@ -0,0 +1,123 @@
package readability
import (
"context"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/worker"
readability "github.com/go-shiori/go-readability"
"github.com/microcosm-cc/bluemonday"
"go.uber.org/zap"
)
type Scraper struct {
client *http.Client
sanitizer *bluemonday.Policy
log *zap.Logger
}
func NewScraper(timeout time.Duration) *Scraper {
if timeout <= 0 {
timeout = 15 * time.Second
}
p := bluemonday.UGCPolicy()
// Allow rich media & responsive elements
p.AllowElements("video", "audio", "source", "track", "iframe", "picture", "figure", "figcaption", "article", "section")
p.AllowAttrs("src", "poster", "controls", "autoplay", "loop", "muted", "preload").OnElements("video", "audio")
p.AllowAttrs("src", "type", "srcset", "media").OnElements("source")
p.AllowAttrs("src", "width", "height", "frameborder", "allow", "allowfullscreen", "sandbox").OnElements("iframe")
p.AllowAttrs("data-src", "data-original", "loading", "srcset", "sizes").OnElements("img")
p.AllowAttrs("class", "id").Globally()
return &Scraper{
client: &http.Client{
Timeout: timeout,
},
sanitizer: p,
log: logger.Named("readability-scraper"),
}
}
// Handler returns the worker.ActionHandler function for readability scraping.
func (s *Scraper) Handler() worker.ActionHandler {
return func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*worker.ActionResult, error) {
if strings.TrimSpace(entry.URL) == "" {
return nil, fmt.Errorf("entry URL is empty")
}
parsedURL, err := url.Parse(entry.URL)
if err != nil {
return nil, fmt.Errorf("invalid URL: %w", err)
}
s.log.Info("Starting readability scrape",
zap.String("entry_id", entry.ID),
zap.String("url", entry.URL),
)
req, err := http.NewRequestWithContext(ctx, "GET", entry.URL, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7")
resp, err := s.client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to fetch article page: %w", err)
}
defer func() {
_ = resp.Body.Close()
}()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("server returned HTTP status %d", resp.StatusCode)
}
article, err := readability.FromReader(resp.Body, parsedURL)
if err != nil {
return nil, fmt.Errorf("failed to parse readability article: %w", err)
}
cleanHTML := s.sanitizer.Sanitize(article.Content)
if strings.TrimSpace(cleanHTML) == "" && strings.TrimSpace(article.TextContent) != "" {
cleanHTML = "<p>" + s.sanitizer.Sanitize(article.TextContent) + "</p>"
}
s.log.Info("Readability scrape successful",
zap.String("entry_id", entry.ID),
zap.String("title", article.Title),
zap.Int("content_length", len(cleanHTML)),
)
now := time.Now().UTC()
variant := models.EntryVariant{
EntryID: entry.ID,
VariantKey: "readability_full_text",
ContentType: "html",
TextValue: cleanHTML,
Metadata: map[string]interface{}{
"title": article.Title,
"byline": article.Byline,
"excerpt": article.Excerpt,
"length": len(article.TextContent),
"scraped_at": now.Format(time.RFC3339),
},
CreatedAt: now,
UpdatedAt: now,
}
return &worker.ActionResult{
ResultVariant: &variant,
}, nil
}
}
+161 -1
View File
@@ -17,6 +17,7 @@ import (
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models" "git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/opml" "git.pengzhan.dev/haopengzhan/rsspipe/internal/opml"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/parser" "git.pengzhan.dev/haopengzhan/rsspipe/internal/parser"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/worker"
"github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5"
) )
@@ -25,14 +26,16 @@ type Handler struct {
auth *auth.AuthService auth *auth.AuthService
crawler *crawler.Crawler crawler *crawler.Crawler
parser *parser.Parser parser *parser.Parser
pool *worker.Pool
} }
func NewHandler(database *db.DB, authService *auth.AuthService, cr *crawler.Crawler) *Handler { func NewHandler(database *db.DB, authService *auth.AuthService, cr *crawler.Crawler, pool *worker.Pool) *Handler {
return &Handler{ return &Handler{
db: database, db: database,
auth: authService, auth: authService,
crawler: cr, crawler: cr,
parser: parser.NewParser(), parser: parser.NewParser(),
pool: pool,
} }
} }
@@ -547,3 +550,160 @@ func (h *Handler) ViewSharedEntry(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8") w.Header().Set("Content-Type", "text/html; charset=utf-8")
_, _ = w.Write([]byte(html)) _, _ = w.Write([]byte(html))
} }
// GetEntry returns a single entry by ID with its loaded variants
func (h *Handler) GetEntry(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
entry, err := h.db.GetEntry(id)
if err != nil {
if err == sql.ErrNoRows {
h.writeError(w, http.StatusNotFound, "entry not found")
return
}
h.writeError(w, http.StatusInternalServerError, err.Error())
return
}
h.writeJSON(w, http.StatusOK, entry)
}
// ListEntryVariants returns all variants for a given entry ID
func (h *Handler) ListEntryVariants(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
variants, err := h.db.ListVariantsForEntry(id)
if err != nil {
h.writeError(w, http.StatusInternalServerError, err.Error())
return
}
h.writeJSON(w, http.StatusOK, variants)
}
// GetVariantBlobStream streams the audio/binary file for a variant with HTTP 206 Partial Content support
func (h *Handler) GetVariantBlobStream(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
key := chi.URLParam(r, "key")
variant, err := h.db.GetVariant(id, key)
if err != nil {
h.writeError(w, http.StatusNotFound, "variant not found")
return
}
if variant.BlobPath == "" {
h.writeError(w, http.StatusNotFound, "variant has no binary media blob")
return
}
http.ServeFile(w, r, variant.BlobPath)
}
// ScrapeEntry manually triggers readability full text scraping and saves to entry_variants
func (h *Handler) ScrapeEntry(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
entry, err := h.db.GetEntry(id)
if err != nil {
h.writeError(w, http.StatusNotFound, "entry not found")
return
}
if h.pool == nil {
h.writeError(w, http.StatusInternalServerError, "worker pool not configured")
return
}
res, err := h.pool.ExecuteDirect(r.Context(), "readability_scrape", *entry, nil)
if err != nil {
h.writeError(w, http.StatusInternalServerError, fmt.Sprintf("scraping failed: %v", err))
return
}
if res != nil && res.ResultVariant != nil {
_ = h.db.SetVariant(*res.ResultVariant)
}
// Reload entry with newly saved variant
updatedEntry, err := h.db.GetEntry(id)
if err != nil {
h.writeError(w, http.StatusInternalServerError, err.Error())
return
}
h.writeJSON(w, http.StatusOK, updatedEntry)
}
// Rules API Handlers
func (h *Handler) ListRules(w http.ResponseWriter, r *http.Request) {
rules, err := h.db.ListRules()
if err != nil {
h.writeError(w, http.StatusInternalServerError, err.Error())
return
}
if rules == nil {
rules = []models.Rule{}
}
h.writeJSON(w, http.StatusOK, rules)
}
type CreateRuleRequest struct {
Name string `json:"name"`
Trigger string `json:"trigger"`
MatchMode string `json:"match_mode"`
Conditions []models.RuleCondition `json:"conditions"`
Actions []models.RuleAction `json:"actions"`
}
func (h *Handler) CreateRule(w http.ResponseWriter, r *http.Request) {
var body CreateRuleRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
h.writeError(w, http.StatusBadRequest, "invalid request body")
return
}
if strings.TrimSpace(body.Name) == "" {
h.writeError(w, http.StatusBadRequest, "rule name is required")
return
}
if strings.TrimSpace(body.Trigger) == "" {
body.Trigger = "on_post_created"
}
if strings.TrimSpace(body.MatchMode) == "" {
body.MatchMode = "all"
}
rule, err := h.db.CreateRule(body.Name, body.Trigger, body.MatchMode, body.Conditions, body.Actions)
if err != nil {
h.writeError(w, http.StatusInternalServerError, err.Error())
return
}
h.writeJSON(w, http.StatusCreated, rule)
}
type UpdateRuleRequest struct {
Name string `json:"name"`
Trigger string `json:"trigger"`
MatchMode string `json:"match_mode"`
Conditions []models.RuleCondition `json:"conditions"`
Actions []models.RuleAction `json:"actions"`
IsActive bool `json:"is_active"`
}
func (h *Handler) UpdateRule(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
var body UpdateRuleRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
h.writeError(w, http.StatusBadRequest, "invalid request body")
return
}
rule, err := h.db.UpdateRule(id, body.Name, body.Trigger, body.MatchMode, body.Conditions, body.Actions, body.IsActive)
if err != nil {
h.writeError(w, http.StatusInternalServerError, err.Error())
return
}
h.writeJSON(w, http.StatusOK, rule)
}
func (h *Handler) DeleteRule(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
if err := h.db.DeleteRule(id); err != nil {
h.writeError(w, http.StatusInternalServerError, err.Error())
return
}
h.writeJSON(w, http.StatusOK, map[string]string{"message": "rule deleted"})
}
+10
View File
@@ -62,9 +62,19 @@ func NewRouter(h *Handler, authService *auth.AuthService, staticDir string) http
// Entries // Entries
r.Get("/entries", h.ListEntries) r.Get("/entries", h.ListEntries)
r.Get("/entries/{id}", h.GetEntry)
r.Put("/entries/{id}/read", h.UpdateEntryRead) r.Put("/entries/{id}/read", h.UpdateEntryRead)
r.Put("/entries/{id}/star", h.UpdateEntryStar) r.Put("/entries/{id}/star", h.UpdateEntryStar)
r.Post("/entries/mark-all-read", h.MarkAllRead) r.Post("/entries/mark-all-read", h.MarkAllRead)
r.Post("/entries/{id}/scrape", h.ScrapeEntry)
r.Get("/entries/{id}/variants", h.ListEntryVariants)
r.Get("/entries/{id}/variants/{key}/stream", h.GetVariantBlobStream)
// Rules
r.Get("/rules", h.ListRules)
r.Post("/rules", h.CreateRule)
r.Put("/rules/{id}", h.UpdateRule)
r.Delete("/rules/{id}", h.DeleteRule)
// OPML // OPML
r.Post("/opml/import", h.ImportOPML) r.Post("/opml/import", h.ImportOPML)
+43 -14
View File
@@ -3,35 +3,41 @@ package crawler
import ( import (
"context" "context"
"fmt" "fmt"
"log"
"sync" "sync"
"time" "time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db" "git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/events"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models" "git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/parser" "git.pengzhan.dev/haopengzhan/rsspipe/internal/parser"
"go.uber.org/zap"
) )
type Crawler struct { type Crawler struct {
db *db.DB db *db.DB
parser *parser.Parser parser *parser.Parser
bus *events.EventBus
intervalMinutes int intervalMinutes int
workerCount int workerCount int
isRefreshing bool isRefreshing bool
mu sync.Mutex mu sync.Mutex
log *zap.Logger
} }
func NewCrawler(database *db.DB, intervalMinutes, workers int) *Crawler { func NewCrawler(database *db.DB, bus *events.EventBus, intervalMinutes, workers int) *Crawler {
return &Crawler{ return &Crawler{
db: database, db: database,
parser: parser.NewParser(), parser: parser.NewParser(),
bus: bus,
intervalMinutes: intervalMinutes, intervalMinutes: intervalMinutes,
workerCount: workers, workerCount: workers,
log: logger.Named("crawler"),
} }
} }
func (c *Crawler) Start(ctx context.Context) { func (c *Crawler) Start(ctx context.Context) {
log.Printf("==> Starting RSS background crawler scheduler (Workers: %d)\n", c.workerCount) c.log.Info("Starting RSS background crawler scheduler", zap.Int("worker_count", c.workerCount))
// Run initial fetch asynchronously after 2 seconds startup // Run initial fetch asynchronously after 2 seconds startup
go func() { go func() {
@@ -72,7 +78,7 @@ func (c *Crawler) RefreshDue(ctx context.Context) {
feeds, err := c.db.ListFeeds() feeds, err := c.db.ListFeeds()
if err != nil { if err != nil {
log.Printf("Crawler error listing feeds: %v\n", err) c.log.Error("Crawler error listing feeds", zap.Error(err))
return return
} }
@@ -93,7 +99,7 @@ func (c *Crawler) RefreshDue(ctx context.Context) {
return return
} }
log.Printf("Crawler: Refreshing %d due feeds (out of %d total)...\n", len(dueFeeds), len(feeds)) c.log.Info("Crawler refreshing due feeds", zap.Int("due_count", len(dueFeeds)), zap.Int("total_count", len(feeds)))
c.runWorkerPool(ctx, dueFeeds) c.runWorkerPool(ctx, dueFeeds)
} }
@@ -102,7 +108,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) {
c.mu.Lock() c.mu.Lock()
if c.isRefreshing { if c.isRefreshing {
c.mu.Unlock() c.mu.Unlock()
log.Println("Crawler: RefreshAll already in progress, skipping") c.log.Debug("Crawler RefreshAll already in progress, skipping")
return return
} }
c.isRefreshing = true c.isRefreshing = true
@@ -116,7 +122,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) {
feeds, err := c.db.ListFeeds() feeds, err := c.db.ListFeeds()
if err != nil { if err != nil {
log.Printf("Crawler error listing feeds: %v\n", err) c.log.Error("Crawler error listing feeds", zap.Error(err))
return return
} }
@@ -124,7 +130,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) {
return return
} }
log.Printf("Crawler: Refreshing all %d feeds...\n", len(feeds)) c.log.Info("Crawler refreshing all feeds", zap.Int("count", len(feeds)))
c.runWorkerPool(ctx, feeds) c.runWorkerPool(ctx, feeds)
} }
@@ -148,7 +154,7 @@ func (c *Crawler) runWorkerPool(ctx context.Context, feeds []models.Feed) {
close(jobs) close(jobs)
wg.Wait() wg.Wait()
log.Printf("Crawler: Completed refresh of %d feeds.\n", len(feeds)) c.log.Info("Crawler completed refresh batch", zap.Int("feed_count", len(feeds)))
} }
func (c *Crawler) RefreshSingle(ctx context.Context, feedID string) error { func (c *Crawler) RefreshSingle(ctx context.Context, feedID string) error {
@@ -169,7 +175,11 @@ func (c *Crawler) fetchSingleFeed(ctx context.Context, f models.Feed) error {
res, err := c.parser.FetchFeed(feedCtx, f.FeedURL, f.ETag, f.LastModified) res, err := c.parser.FetchFeed(feedCtx, f.FeedURL, f.ETag, f.LastModified)
if err != nil { if err != nil {
log.Printf("Crawler error fetching [%s]: %v\n", f.Title, err) c.log.Warn("Crawler error fetching feed",
zap.String("feed_title", f.Title),
zap.String("feed_url", f.FeedURL),
zap.Error(err),
)
_ = c.db.UpdateFeedFetchStatus(f.ID, f.ETag, f.LastModified, err.Error()) _ = c.db.UpdateFeedFetchStatus(f.ID, f.ETag, f.LastModified, err.Error())
return err return err
} }
@@ -180,11 +190,30 @@ func (c *Crawler) fetchSingleFeed(ctx context.Context, f models.Feed) error {
} }
// Upsert entries // Upsert entries
inserted, err := c.db.UpsertFeedEntries(f.ID, res.Entries) newEntries, err := c.db.UpsertFeedEntries(f.ID, res.Entries)
if err != nil { if err != nil {
log.Printf("Crawler error upserting entries for [%s]: %v\n", f.Title, err) c.log.Error("Crawler error upserting entries",
} else if inserted > 0 { zap.String("feed_title", f.Title),
log.Printf("Crawler [%s]: inserted %d new entries\n", f.Title, inserted) zap.Error(err),
)
} else if len(newEntries) > 0 {
c.log.Info("Crawler inserted new entries",
zap.String("feed_title", f.Title),
zap.Int("new_entries", len(newEntries)),
)
// Publish on_post_created events for newly inserted entries
if c.bus != nil {
for _, entry := range newEntries {
entry.FeedTitle = f.Title
entry.FeedSiteURL = f.SiteURL
c.bus.Publish(ctx, events.Event{
Type: events.EventPostCreated,
Entry: entry,
Depth: 0,
})
}
}
} }
_ = c.db.UpdateFeedFetchStatus(f.ID, res.ETag, res.LastModified, "") _ = c.db.UpdateFeedFetchStatus(f.ID, res.ETag, res.LastModified, "")
+391 -84
View File
@@ -114,15 +114,28 @@ func (d *DB) migrate() error {
CREATE INDEX IF NOT EXISTS idx_entries_is_starred ON feed_entries(is_starred); CREATE INDEX IF NOT EXISTS idx_entries_is_starred ON feed_entries(is_starred);
CREATE INDEX IF NOT EXISTS idx_entries_share_uuid ON feed_entries(share_uuid); CREATE INDEX IF NOT EXISTS idx_entries_share_uuid ON feed_entries(share_uuid);
CREATE TABLE IF NOT EXISTS entry_variants (
entry_id TEXT NOT NULL,
variant_key TEXT NOT NULL,
content_type TEXT NOT NULL,
text_value TEXT,
blob_path TEXT,
metadata TEXT DEFAULT '{}',
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL,
PRIMARY KEY (entry_id, variant_key),
FOREIGN KEY(entry_id) REFERENCES feed_entries(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_entry_variants_key ON entry_variants(variant_key);
CREATE TABLE IF NOT EXISTS rules ( CREATE TABLE IF NOT EXISTS rules (
id TEXT PRIMARY KEY, id TEXT PRIMARY KEY,
name TEXT NOT NULL, name TEXT NOT NULL,
rule_type TEXT NOT NULL, trigger TEXT NOT NULL,
is_active BOOLEAN DEFAULT 1,
trigger_type TEXT NOT NULL,
trigger_config TEXT DEFAULT '{}',
conditions TEXT DEFAULT '[]', conditions TEXT DEFAULT '[]',
match_mode TEXT DEFAULT 'all',
actions TEXT DEFAULT '[]', actions TEXT DEFAULT '[]',
is_active BOOLEAN DEFAULT 1,
created_at DATETIME NOT NULL, created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL updated_at DATETIME NOT NULL
); );
@@ -132,6 +145,26 @@ func (d *DB) migrate() error {
} }
// Migration for timeout_seconds column if older DB exists // Migration for timeout_seconds column if older DB exists
_, _ = d.Exec("ALTER TABLE feeds ADD COLUMN timeout_seconds INTEGER DEFAULT 10") _, _ = d.Exec("ALTER TABLE feeds ADD COLUMN timeout_seconds INTEGER DEFAULT 10")
// Migration for rules table schema if older table exists
var testCol string
err := d.QueryRow("SELECT trigger FROM rules LIMIT 1").Scan(&testCol)
if err != nil && err != sql.ErrNoRows {
_, _ = d.Exec("DROP TABLE IF EXISTS rules")
_, _ = d.Exec(`
CREATE TABLE IF NOT EXISTS rules (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
trigger TEXT NOT NULL,
conditions TEXT DEFAULT '[]',
match_mode TEXT DEFAULT 'all',
actions TEXT DEFAULT '[]',
is_active BOOLEAN DEFAULT 1,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL
);
`)
}
return nil return nil
} }
@@ -474,32 +507,45 @@ func (d *DB) DeleteFeed(id string) error {
} }
// Feed Entries operations // Feed Entries operations
func (d *DB) UpsertFeedEntries(feedID string, entries []models.FeedEntry) (int, error) { func (d *DB) UpsertFeedEntries(feedID string, entries []models.FeedEntry) ([]models.FeedEntry, error) {
if len(entries) == 0 { if len(entries) == 0 {
return 0, nil return nil, nil
} }
stmt, err := d.Prepare(` var newEntries []models.FeedEntry
now := time.Now().UTC()
insertStmt, err := d.Prepare(`
INSERT INTO feed_entries ( INSERT INTO feed_entries (
id, feed_id, guid, url, title, author, content, raw_content, published_at, is_read, is_starred, metadata, created_at id, feed_id, guid, url, title, author, content, raw_content, published_at, is_read, is_starred, metadata, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?, ?) ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?, ?)
ON CONFLICT(feed_id, guid) DO UPDATE SET
title = excluded.title,
content = CASE WHEN feed_entries.content = '' THEN excluded.content ELSE feed_entries.content END,
url = excluded.url,
author = excluded.author
`) `)
if err != nil { if err != nil {
return 0, err return nil, err
} }
defer func() { defer func() {
_ = stmt.Close() _ = insertStmt.Close()
}()
updateStmt, err := d.Prepare(`
UPDATE feed_entries
SET title = ?, url = ?, author = ?, content = ?, raw_content = ?
WHERE feed_id = ? AND guid = ?
`)
if err != nil {
return nil, err
}
defer func() {
_ = updateStmt.Close()
}() }()
insertedCount := 0
for _, e := range entries { for _, e := range entries {
id := uuid.New().String() var existingID string
now := time.Now().UTC() err := d.QueryRow("SELECT id FROM feed_entries WHERE feed_id = ? AND guid = ?", feedID, e.GUID).Scan(&existingID)
switch err {
case sql.ErrNoRows:
e.ID = uuid.New().String()
e.FeedID = feedID
pubAt := e.PublishedAt pubAt := e.PublishedAt
if pubAt.IsZero() { if pubAt.IsZero() {
pubAt = now pubAt = now
@@ -509,16 +555,19 @@ func (d *DB) UpsertFeedEntries(feedID string, entries []models.FeedEntry) (int,
metaBytes = []byte("{}") metaBytes = []byte("{}")
} }
res, err := stmt.Exec( _, err := insertStmt.Exec(
id, feedID, e.GUID, e.URL, e.Title, e.Author, e.Content, e.RawContent, pubAt, string(metaBytes), now, e.ID, feedID, e.GUID, e.URL, e.Title, e.Author, e.Content, e.RawContent, pubAt, string(metaBytes), now,
) )
if err == nil { if err == nil {
if n, _ := res.RowsAffected(); n > 0 { e.CreatedAt = now
insertedCount++ newEntries = append(newEntries, e)
}
case nil:
_, _ = updateStmt.Exec(e.Title, e.URL, e.Author, e.Content, e.RawContent, feedID, e.GUID)
} }
} }
}
return insertedCount, nil return newEntries, nil
} }
type EntryFilter struct { type EntryFilter struct {
@@ -530,67 +579,98 @@ type EntryFilter struct {
Offset int Offset int
} }
func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) { func (d *DB) GetEntry(id string) (*models.FeedEntry, error) {
where := []string{"1=1"} query := `
args := []interface{}{} SELECT
e.id, e.feed_id, f.title as feed_title, COALESCE(f.site_url, '') as feed_site_url,
switch f.View { e.guid, e.url, e.title, e.author, e.content, e.raw_content,
case "inbox": e.published_at, e.is_read, e.is_starred, e.share_uuid, e.metadata, e.created_at
where = append(where, "e.is_read = 0")
case "starred":
where = append(where, "e.is_starred = 1")
}
if f.FeedID != "" {
where = append(where, "e.feed_id = ?")
args = append(args, f.FeedID)
}
if f.CategoryID != "" {
where = append(where, "f.category_id = ?")
args = append(args, f.CategoryID)
}
if f.Search != "" {
searchTerms := strings.Fields(f.Search)
for _, term := range searchTerms {
lower := strings.ToLower(term)
if strings.HasPrefix(lower, "is:unread") {
where = append(where, "e.is_read = 0")
} else if strings.HasPrefix(lower, "is:read") {
where = append(where, "e.is_read = 1")
} else if strings.HasPrefix(lower, "is:starred") {
where = append(where, "e.is_starred = 1")
} else if strings.HasPrefix(lower, "feed:") {
feedVal := strings.TrimPrefix(term, "feed:")
where = append(where, "(f.title LIKE ? OR f.feed_url LIKE ?)")
args = append(args, "%"+feedVal+"%", "%"+feedVal+"%")
} else {
where = append(where, "(e.title LIKE ? OR e.content LIKE ? OR e.author LIKE ?)")
args = append(args, "%"+term+"%", "%"+term+"%", "%"+term+"%")
}
}
}
whereClause := strings.Join(where, " AND ")
// Count total
countQuery := fmt.Sprintf(`
SELECT COUNT(*)
FROM feed_entries e FROM feed_entries e
JOIN feeds f ON f.id = e.feed_id JOIN feeds f ON f.id = e.feed_id
WHERE %s WHERE e.id = ?
`, whereClause) `
var e models.FeedEntry
var author, rawContent, shareUUID, metaStr sql.NullString
err := d.QueryRow(query, id).Scan(
&e.ID, &e.FeedID, &e.FeedTitle, &e.FeedSiteURL,
&e.GUID, &e.URL, &e.Title, &author, &e.Content, &rawContent,
&e.PublishedAt, &e.IsRead, &e.IsStarred, &shareUUID, &metaStr, &e.CreatedAt,
)
if err != nil {
return nil, err
}
if author.Valid {
e.Author = author.String
}
if rawContent.Valid {
e.RawContent = rawContent.String
}
if shareUUID.Valid {
s := shareUUID.String
e.ShareUUID = &s
}
e.Metadata = make(map[string]interface{})
if metaStr.Valid && metaStr.String != "" {
_ = json.Unmarshal([]byte(metaStr.String), &e.Metadata)
}
// Load variants
variants, _ := d.ListVariantsForEntry(e.ID)
e.Variants = variants
return &e, nil
}
func (d *DB) ListEntries(filter EntryFilter) ([]models.FeedEntry, int, error) {
var whereClauses []string
var args []interface{}
switch filter.View {
case "inbox":
whereClauses = append(whereClauses, "e.is_read = 0")
case "starred":
whereClauses = append(whereClauses, "e.is_starred = 1")
}
if filter.FeedID != "" {
whereClauses = append(whereClauses, "e.feed_id = ?")
args = append(args, filter.FeedID)
}
if filter.CategoryID != "" {
whereClauses = append(whereClauses, "f.category_id = ?")
args = append(args, filter.CategoryID)
}
if filter.Search != "" {
whereClauses = append(whereClauses, "(e.title LIKE ? OR e.content LIKE ? OR e.author LIKE ?)")
pattern := "%" + filter.Search + "%"
args = append(args, pattern, pattern, pattern)
}
whereSQL := ""
if len(whereClauses) > 0 {
whereSQL = "WHERE " + strings.Join(whereClauses, " AND ")
}
countQuery := fmt.Sprintf(`
SELECT COUNT(e.id)
FROM feed_entries e
JOIN feeds f ON f.id = e.feed_id
%s
`, whereSQL)
var total int var total int
if err := d.QueryRow(countQuery, args...).Scan(&total); err != nil { err := d.QueryRow(countQuery, args...).Scan(&total)
if err != nil {
return nil, 0, err return nil, 0, err
} }
limit := f.Limit limit := filter.Limit
if limit <= 0 || limit > 100 { if limit <= 0 || limit > 500 {
limit = 50 limit = 100
} }
offset := f.Offset offset := filter.Offset
if offset < 0 { if offset < 0 {
offset = 0 offset = 0
} }
@@ -602,10 +682,10 @@ func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) {
e.published_at, e.is_read, e.is_starred, e.share_uuid, e.metadata, e.created_at e.published_at, e.is_read, e.is_starred, e.share_uuid, e.metadata, e.created_at
FROM feed_entries e FROM feed_entries e
JOIN feeds f ON f.id = e.feed_id JOIN feeds f ON f.id = e.feed_id
WHERE %s %s
ORDER BY e.published_at DESC ORDER BY e.published_at DESC, e.created_at DESC
LIMIT ? OFFSET ? LIMIT ? OFFSET ?
`, whereClause) `, whereSQL)
queryArgs := append(args, limit, offset) queryArgs := append(args, limit, offset)
rows, err := d.Query(query, queryArgs...) rows, err := d.Query(query, queryArgs...)
@@ -619,13 +699,13 @@ func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) {
var entries []models.FeedEntry var entries []models.FeedEntry
for rows.Next() { for rows.Next() {
var e models.FeedEntry var e models.FeedEntry
var author, rawContent, metaStr sql.NullString var author, rawContent, shareUUID, metaStr sql.NullString
var shareUUID sql.NullString err := rows.Scan(
if err := rows.Scan(
&e.ID, &e.FeedID, &e.FeedTitle, &e.FeedSiteURL, &e.ID, &e.FeedID, &e.FeedTitle, &e.FeedSiteURL,
&e.GUID, &e.URL, &e.Title, &author, &e.Content, &rawContent, &e.GUID, &e.URL, &e.Title, &author, &e.Content, &rawContent,
&e.PublishedAt, &e.IsRead, &e.IsStarred, &shareUUID, &metaStr, &e.CreatedAt, &e.PublishedAt, &e.IsRead, &e.IsStarred, &shareUUID, &metaStr, &e.CreatedAt,
); err != nil { )
if err != nil {
return nil, 0, err return nil, 0, err
} }
if author.Valid { if author.Valid {
@@ -675,6 +755,232 @@ func (d *DB) MarkAllRead(feedID, categoryID *string) error {
return err return err
} }
// Entry Variants CRUD
func (d *DB) GetVariant(entryID, key string) (*models.EntryVariant, error) {
query := `
SELECT entry_id, variant_key, content_type, COALESCE(text_value, ''), COALESCE(blob_path, ''), metadata, created_at, updated_at
FROM entry_variants
WHERE entry_id = ? AND variant_key = ?
`
var v models.EntryVariant
var metaStr sql.NullString
err := d.QueryRow(query, entryID, key).Scan(
&v.EntryID, &v.VariantKey, &v.ContentType, &v.TextValue, &v.BlobPath, &metaStr, &v.CreatedAt, &v.UpdatedAt,
)
if err != nil {
return nil, err
}
v.Metadata = make(map[string]interface{})
if metaStr.Valid && metaStr.String != "" {
_ = json.Unmarshal([]byte(metaStr.String), &v.Metadata)
}
return &v, nil
}
func (d *DB) ListVariantsForEntry(entryID string) (map[string]models.EntryVariant, error) {
query := `
SELECT entry_id, variant_key, content_type, COALESCE(text_value, ''), COALESCE(blob_path, ''), metadata, created_at, updated_at
FROM entry_variants
WHERE entry_id = ?
`
rows, err := d.Query(query, entryID)
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
variants := make(map[string]models.EntryVariant)
for rows.Next() {
var v models.EntryVariant
var metaStr sql.NullString
err := rows.Scan(
&v.EntryID, &v.VariantKey, &v.ContentType, &v.TextValue, &v.BlobPath, &metaStr, &v.CreatedAt, &v.UpdatedAt,
)
if err != nil {
return nil, err
}
v.Metadata = make(map[string]interface{})
if metaStr.Valid && metaStr.String != "" {
_ = json.Unmarshal([]byte(metaStr.String), &v.Metadata)
}
variants[v.VariantKey] = v
}
return variants, nil
}
func (d *DB) SetVariant(v models.EntryVariant) error {
now := time.Now().UTC()
metaBytes, _ := json.Marshal(v.Metadata)
if len(metaBytes) == 0 {
metaBytes = []byte("{}")
}
query := `
INSERT INTO entry_variants (
entry_id, variant_key, content_type, text_value, blob_path, metadata, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(entry_id, variant_key) DO UPDATE SET
content_type = excluded.content_type,
text_value = excluded.text_value,
blob_path = excluded.blob_path,
metadata = excluded.metadata,
updated_at = excluded.updated_at
`
_, err := d.Exec(query, v.EntryID, v.VariantKey, v.ContentType, v.TextValue, v.BlobPath, string(metaBytes), now, now)
return err
}
func (d *DB) DeleteVariant(entryID, key string) error {
_, err := d.Exec("DELETE FROM entry_variants WHERE entry_id = ? AND variant_key = ?", entryID, key)
return err
}
// Rules CRUD
func (d *DB) ListRules() ([]models.Rule, error) {
query := `
SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at
FROM rules
ORDER BY created_at ASC
`
rows, err := d.Query(query)
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
var rules []models.Rule
for rows.Next() {
var r models.Rule
var condsStr, actionsStr string
err := rows.Scan(
&r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt,
)
if err != nil {
return nil, err
}
_ = json.Unmarshal([]byte(condsStr), &r.Conditions)
_ = json.Unmarshal([]byte(actionsStr), &r.Actions)
rules = append(rules, r)
}
return rules, nil
}
func (d *DB) GetActiveRules(trigger string) ([]models.Rule, error) {
query := `
SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at
FROM rules
WHERE is_active = 1 AND (trigger = ? OR trigger = '*')
ORDER BY created_at ASC
`
rows, err := d.Query(query, trigger)
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
var rules []models.Rule
for rows.Next() {
var r models.Rule
var condsStr, actionsStr string
err := rows.Scan(
&r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt,
)
if err != nil {
return nil, err
}
_ = json.Unmarshal([]byte(condsStr), &r.Conditions)
_ = json.Unmarshal([]byte(actionsStr), &r.Actions)
rules = append(rules, r)
}
return rules, nil
}
func (d *DB) GetRule(id string) (*models.Rule, error) {
query := `
SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at
FROM rules
WHERE id = ?
`
var r models.Rule
var condsStr, actionsStr string
err := d.QueryRow(query, id).Scan(
&r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt,
)
if err != nil {
return nil, err
}
_ = json.Unmarshal([]byte(condsStr), &r.Conditions)
_ = json.Unmarshal([]byte(actionsStr), &r.Actions)
return &r, nil
}
func (d *DB) CreateRule(name, trigger, matchMode string, conditions []models.RuleCondition, actions []models.RuleAction) (*models.Rule, error) {
id := uuid.New().String()
now := time.Now().UTC()
if matchMode == "" {
matchMode = "all"
}
condsBytes, _ := json.Marshal(conditions)
actionsBytes, _ := json.Marshal(actions)
query := `
INSERT INTO rules (
id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?)
`
_, err := d.Exec(query, id, name, trigger, string(condsBytes), matchMode, string(actionsBytes), now, now)
if err != nil {
return nil, err
}
return &models.Rule{
ID: id,
Name: name,
Trigger: trigger,
Conditions: conditions,
MatchMode: matchMode,
Actions: actions,
IsActive: true,
CreatedAt: now,
UpdatedAt: now,
}, nil
}
func (d *DB) UpdateRule(id string, name, trigger, matchMode string, conditions []models.RuleCondition, actions []models.RuleAction, isActive bool) (*models.Rule, error) {
now := time.Now().UTC()
if matchMode == "" {
matchMode = "all"
}
condsBytes, _ := json.Marshal(conditions)
actionsBytes, _ := json.Marshal(actions)
query := `
UPDATE rules
SET name = ?, trigger = ?, conditions = ?, match_mode = ?, actions = ?, is_active = ?, updated_at = ?
WHERE id = ?
`
res, err := d.Exec(query, name, trigger, string(condsBytes), matchMode, string(actionsBytes), isActive, now, id)
if err != nil {
return nil, err
}
if n, _ := res.RowsAffected(); n == 0 {
return nil, fmt.Errorf("rule not found")
}
return d.GetRule(id)
}
func (d *DB) DeleteRule(id string) error {
_, err := d.Exec("DELETE FROM rules WHERE id = ?", id)
return err
}
func (d *DB) GetStats() (*models.Stats, error) { func (d *DB) GetStats() (*models.Stats, error) {
var s models.Stats var s models.Stats
err := d.QueryRow("SELECT COUNT(*) FROM feeds").Scan(&s.TotalFeeds) err := d.QueryRow("SELECT COUNT(*) FROM feeds").Scan(&s.TotalFeeds)
@@ -693,5 +999,6 @@ func (d *DB) GetStats() (*models.Stats, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
_ = d.QueryRow("SELECT COUNT(*) FROM rules").Scan(&s.TotalRules)
return &s, nil return &s, nil
} }
+61 -4
View File
@@ -72,8 +72,8 @@ func TestDatabaseOperations(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("UpsertFeedEntries failed: %v", err) t.Fatalf("UpsertFeedEntries failed: %v", err)
} }
if inserted != 2 { if len(inserted) != 2 {
t.Fatalf("expected 2 inserted entries, got %d", inserted) t.Fatalf("expected 2 inserted entries, got %d", len(inserted))
} }
// 5. List entries // 5. List entries
@@ -94,12 +94,69 @@ func TestDatabaseOperations(t *testing.T) {
t.Fatalf("UpdateEntryStarred failed: %v", err) t.Fatalf("UpdateEntryStarred failed: %v", err)
} }
// 7. Verify Stats // 7. Test Variant CRUD
variant := models.EntryVariant{
EntryID: firstID,
VariantKey: "readability_full_text",
ContentType: "html",
TextValue: "<div>Clean full text content</div>",
Metadata: map[string]interface{}{
"scraped_at": time.Now().UTC().Format(time.RFC3339),
"length": 100,
},
}
if err := database.SetVariant(variant); err != nil {
t.Fatalf("SetVariant failed: %v", err)
}
fetchedVariant, err := database.GetVariant(firstID, "readability_full_text")
if err != nil {
t.Fatalf("GetVariant failed: %v", err)
}
if fetchedVariant.TextValue != "<div>Clean full text content</div>" {
t.Fatalf("expected variant text, got %s", fetchedVariant.TextValue)
}
// Test GetEntry with variants
entryWithVariants, err := database.GetEntry(firstID)
if err != nil {
t.Fatalf("GetEntry failed: %v", err)
}
if len(entryWithVariants.Variants) != 1 {
t.Fatalf("expected 1 variant in GetEntry, got %d", len(entryWithVariants.Variants))
}
// 8. Test Rules CRUD
rule, err := database.CreateRule(
"Auto Scrape Rule",
"on_post_created",
"all",
[]models.RuleCondition{
{Field: "feed_id", Operator: "eq", Value: feed.ID},
},
[]models.RuleAction{
{Type: "readability_scrape"},
},
)
if err != nil {
t.Fatalf("CreateRule failed: %v", err)
}
if rule.Name != "Auto Scrape Rule" {
t.Errorf("expected rule name Auto Scrape Rule, got %s", rule.Name)
}
activeRules, err := database.GetActiveRules("on_post_created")
if err != nil {
t.Fatalf("GetActiveRules failed: %v", err)
}
if len(activeRules) != 1 {
t.Fatalf("expected 1 active rule, got %d", len(activeRules))
}
// 9. Verify Stats
stats, err := database.GetStats() stats, err := database.GetStats()
if err != nil { if err != nil {
t.Fatalf("GetStats failed: %v", err) t.Fatalf("GetStats failed: %v", err)
} }
if stats.TotalFeeds != 1 || stats.TotalUnread != 1 || stats.TotalStarred != 1 || stats.TotalCategories != 1 { if stats.TotalFeeds != 1 || stats.TotalUnread != 1 || stats.TotalStarred != 1 || stats.TotalRules != 1 {
t.Errorf("unexpected stats: %+v", stats) t.Errorf("unexpected stats: %+v", stats)
} }
+92
View File
@@ -0,0 +1,92 @@
package events
import (
"context"
"sync"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"go.uber.org/zap"
)
type EventType string
const (
EventPostCreated EventType = "on_post_created"
EventPostUpdated EventType = "on_post_updated"
EventVariantUpdated EventType = "on_variant_updated"
EventManualTrigger EventType = "manual"
)
type Event struct {
Type EventType
Entry models.FeedEntry
VariantKey string // populated when Type == EventVariantUpdated
Depth int
DirectAttrs map[string]interface{}
}
type EventHandler func(ctx context.Context, evt Event) error
type EventBus struct {
mu sync.RWMutex
subscribers map[EventType][]EventHandler
log *zap.Logger
maxDepth int
}
func NewEventBus() *EventBus {
return &EventBus{
subscribers: make(map[EventType][]EventHandler),
log: logger.Named("eventbus"),
maxDepth: 5,
}
}
func (b *EventBus) Subscribe(eventType EventType, handler EventHandler) {
b.mu.Lock()
defer b.mu.Unlock()
b.subscribers[eventType] = append(b.subscribers[eventType], handler)
}
func (b *EventBus) Publish(ctx context.Context, evt Event) {
if evt.Depth > b.maxDepth {
b.log.Warn("Event dropped: recursion depth limit exceeded",
zap.String("type", string(evt.Type)),
zap.String("entry_id", evt.Entry.ID),
zap.Int("depth", evt.Depth),
)
return
}
b.mu.RLock()
handlers := append([]EventHandler{}, b.subscribers[evt.Type]...)
wildcardHandlers := append([]EventHandler{}, b.subscribers["*"]...)
b.mu.RUnlock()
b.log.Debug("Event published",
zap.String("type", string(evt.Type)),
zap.String("entry_id", evt.Entry.ID),
zap.String("variant_key", evt.VariantKey),
zap.Int("depth", evt.Depth),
zap.Int("subscriber_count", len(handlers)+len(wildcardHandlers)),
)
allHandlers := append(handlers, wildcardHandlers...)
for _, h := range allHandlers {
go func(fn EventHandler) {
defer func() {
if r := recover(); r != nil {
b.log.Error("Event handler panic recovered", zap.Any("panic", r))
}
}()
if err := fn(ctx, evt); err != nil {
b.log.Error("Event handler error",
zap.String("type", string(evt.Type)),
zap.String("entry_id", evt.Entry.ID),
zap.Error(err),
)
}
}(h)
}
}
+74
View File
@@ -0,0 +1,74 @@
package logger
import (
"os"
"strings"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
var globalLogger *zap.Logger
var globalSugared *zap.SugaredLogger
func init() {
InitLogger(os.Getenv("LOG_LEVEL"))
}
// InitLogger initializes the global Zap logger with proper formatting and log levels.
func InitLogger(level string) *zap.Logger {
zapLevel := zapcore.InfoLevel
switch strings.ToLower(level) {
case "debug":
zapLevel = zapcore.DebugLevel
case "warn", "warning":
zapLevel = zapcore.WarnLevel
case "error":
zapLevel = zapcore.ErrorLevel
}
encoderConfig := zapcore.EncoderConfig{
TimeKey: "time",
LevelKey: "level",
NameKey: "logger",
CallerKey: "caller",
MessageKey: "msg",
StacktraceKey: "stacktrace",
LineEnding: zapcore.DefaultLineEnding,
EncodeLevel: zapcore.CapitalColorLevelEncoder,
EncodeTime: zapcore.TimeEncoderOfLayout("15:04:05.000"),
EncodeDuration: zapcore.StringDurationEncoder,
EncodeCaller: zapcore.ShortCallerEncoder,
}
core := zapcore.NewCore(
zapcore.NewConsoleEncoder(encoderConfig),
zapcore.AddSync(os.Stdout),
zapLevel,
)
globalLogger = zap.New(core, zap.AddCaller())
globalSugared = globalLogger.Sugar()
return globalLogger
}
// L returns the global zap logger instance.
func L() *zap.Logger {
if globalLogger == nil {
InitLogger("info")
}
return globalLogger
}
// S returns the global sugared logger instance.
func S() *zap.SugaredLogger {
if globalSugared == nil {
InitLogger("info")
}
return globalSugared
}
// Named creates a child logger with a component name tag.
func Named(name string) *zap.Logger {
return L().Named(name)
}
+15
View File
@@ -0,0 +1,15 @@
package logger
import (
"testing"
)
func TestLoggerInit(t *testing.T) {
log := InitLogger("debug")
if log == nil {
t.Fatal("expected non-nil logger")
}
L().Info("Test structured logging message")
S().Infof("Test sugared logging message with %s", "parameter")
Named("test-component").Debug("Test component debug log")
}
+43 -2
View File
@@ -53,22 +53,63 @@ type FeedEntry struct {
URL string `json:"url"` URL string `json:"url"`
Title string `json:"title"` Title string `json:"title"`
Author string `json:"author,omitempty"` Author string `json:"author,omitempty"`
Content string `json:"content"` Content string `json:"content"` // Immutable original RSS content
RawContent string `json:"raw_content,omitempty"` RawContent string `json:"raw_content,omitempty"`
PublishedAt time.Time `json:"published_at"` PublishedAt time.Time `json:"published_at"`
IsRead bool `json:"is_read"` IsRead bool `json:"is_read"`
IsStarred bool `json:"is_starred"` IsStarred bool `json:"is_starred"`
ShareUUID *string `json:"share_uuid,omitempty"` ShareUUID *string `json:"share_uuid,omitempty"`
Metadata map[string]interface{} `json:"metadata"` Variants map[string]EntryVariant `json:"variants,omitempty"`
Metadata map[string]interface{} `json:"metadata,omitempty"`
CreatedAt time.Time `json:"created_at"` CreatedAt time.Time `json:"created_at"`
} }
// EntryVariant represents an extensible enrichment result (readability, keywords, ai_summary, tts)
type EntryVariant struct {
EntryID string `json:"entry_id"`
VariantKey string `json:"variant_key"`
ContentType string `json:"content_type"` // 'text', 'html', 'json', 'yaml', 'audio/mp3', 'binary'
TextValue string `json:"text_value,omitempty"`
BlobPath string `json:"blob_path,omitempty"`
StreamURL string `json:"stream_url,omitempty"`
Metadata map[string]interface{} `json:"metadata,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// Rule represents a filter/automation hook rule
type Rule struct {
ID string `json:"id"`
Name string `json:"name"`
Trigger string `json:"trigger"` // 'on_post_created', 'on_post_updated', 'on_variant_updated', 'manual'
Conditions []RuleCondition `json:"conditions"`
MatchMode string `json:"match_mode"` // 'all' (AND) or 'any' (OR)
Actions []RuleAction `json:"actions"`
IsActive bool `json:"is_active"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// RuleCondition represents a single condition predicate
type RuleCondition struct {
Field string `json:"field"` // 'feed_id', 'title', 'author', 'variants.keywords', 'variants.readability_full_text'
Operator string `json:"operator"` // 'eq', 'neq', 'in', 'contains', 'matches'
Value interface{} `json:"value"`
}
// RuleAction represents an action triggered by a rule
type RuleAction struct {
Type string `json:"type"` // 'readability_scrape', 'mark_read', 'mark_starred', 'extract_keywords', 'ai_summary', 'tts'
Params map[string]interface{} `json:"params,omitempty"`
}
// Stats represents dashboard aggregate counts // Stats represents dashboard aggregate counts
type Stats struct { type Stats struct {
TotalFeeds int `json:"total_feeds"` TotalFeeds int `json:"total_feeds"`
TotalUnread int `json:"total_unread"` TotalUnread int `json:"total_unread"`
TotalStarred int `json:"total_starred"` TotalStarred int `json:"total_starred"`
TotalCategories int `json:"total_categories"` TotalCategories int `json:"total_categories"`
TotalRules int `json:"total_rules"`
} }
// LoginRequest is the body for POST /api/auth/login // LoginRequest is the body for POST /api/auth/login
+97 -5
View File
@@ -2,7 +2,9 @@ package parser
import ( import (
"context" "context"
"encoding/json"
"fmt" "fmt"
"html"
"io" "io"
"net/http" "net/http"
"strings" "strings"
@@ -104,10 +106,7 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri
pubDate = item.UpdatedParsed.UTC() pubDate = item.UpdatedParsed.UTC()
} }
content := item.Content content := sanitizeAndCombineContent(item.Content, item.Description)
if strings.TrimSpace(content) == "" {
content = item.Description
}
author := "" author := ""
if item.Author != nil { if item.Author != nil {
@@ -120,7 +119,7 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri
Title: item.Title, Title: item.Title,
Author: author, Author: author,
Content: content, Content: content,
RawContent: item.Description, RawContent: content,
PublishedAt: pubDate, PublishedAt: pubDate,
Metadata: make(map[string]interface{}), Metadata: make(map[string]interface{}),
} }
@@ -129,3 +128,96 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri
return result, nil return result, nil
} }
type draftBlock struct {
Key string `json:"key"`
Text string `json:"text"`
Type string `json:"type"`
}
type draftEntityData struct {
URL string `json:"url"`
Src string `json:"src"`
Caption string `json:"caption"`
}
type draftEntity struct {
Type string `json:"type"`
Data draftEntityData `json:"data"`
}
type draftPayload struct {
Blocks []draftBlock `json:"blocks"`
EntityMap map[string]draftEntity `json:"entityMap"`
}
func parseDraftJSText(raw string) (string, bool) {
unescaped := html.UnescapeString(raw)
idx := strings.Index(unescaped, `{"entityMap"`)
if idx == -1 {
idx = strings.Index(unescaped, `{"blocks"`)
}
if idx == -1 {
return "", false
}
prefix := strings.TrimSpace(unescaped[:idx])
jsonPart := strings.TrimSpace(unescaped[idx:])
var payload draftPayload
if err := json.Unmarshal([]byte(jsonPart), &payload); err != nil {
return "", false
}
if len(payload.Blocks) == 0 && len(payload.EntityMap) == 0 {
return "", false
}
var sb strings.Builder
if prefix != "" {
lines := strings.Split(prefix, "\n")
for _, line := range lines {
l := strings.TrimSpace(line)
if l != "" {
sb.WriteString("<p>")
sb.WriteString(html.EscapeString(l))
sb.WriteString("</p>\n")
}
}
}
for _, block := range payload.Blocks {
txt := strings.TrimSpace(block.Text)
if txt != "" {
sb.WriteString("<p>")
sb.WriteString(html.EscapeString(txt))
sb.WriteString("</p>\n")
}
}
for _, ent := range payload.EntityMap {
if strings.EqualFold(ent.Type, "IMAGE") && ent.Data.Src != "" {
fmt.Fprintf(&sb, `<p><img src="%s" alt="%s" /></p>`+"\n", html.EscapeString(ent.Data.Src), html.EscapeString(ent.Data.Caption))
}
}
return sb.String(), true
}
func sanitizeAndCombineContent(content, description string) string {
if parsedDesc, ok := parseDraftJSText(description); ok {
if strings.TrimSpace(content) != "" {
return content + "<br/>\n" + parsedDesc
}
return parsedDesc
}
if parsedContent, ok := parseDraftJSText(content); ok {
return parsedContent
}
if strings.TrimSpace(content) != "" {
return content
}
return description
}
+27
View File
@@ -0,0 +1,27 @@
package parser
import (
"strings"
"testing"
)
func TestParseDraftJSText(t *testing.T) {
desc := `就这样吧评论: 奥德赛 (https://movie.douban.com/subject/36808876/)
评价: 推荐
{"entityMap": {}, "blocks": [{"key": "1", "text": "终于看完了奥德赛,非常精彩!", "type": "unstyled"}]}`
content := `<img src="https://img9.doubanio.com/poster.jpg" /><a href="#">就这样吧</a>评论: <a href="#">奥德赛</a><br/>评价: 推荐`
combined := sanitizeAndCombineContent(content, desc)
if !strings.Contains(combined, "终于看完了奥德赛,非常精彩!") {
t.Fatalf("expected parsed text in output, got: %s", combined)
}
if !strings.Contains(combined, "<p>终于看完了奥德赛,非常精彩!</p>") {
t.Fatalf("expected formatted paragraph in output, got: %s", combined)
}
if !strings.Contains(combined, "img9.doubanio.com/poster.jpg") {
t.Fatalf("expected poster in output, got: %s", combined)
}
}
+82
View File
@@ -0,0 +1,82 @@
package rules
import (
"context"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/events"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"go.uber.org/zap"
)
type WorkerDispatcher interface {
Dispatch(ctx context.Context, action models.RuleAction, entry models.FeedEntry, depth int)
}
type Engine struct {
db *db.DB
bus *events.EventBus
evaluator *Evaluator
dispatcher WorkerDispatcher
log *zap.Logger
}
func NewEngine(database *db.DB, bus *events.EventBus, dispatcher WorkerDispatcher) *Engine {
return &Engine{
db: database,
bus: bus,
evaluator: NewEvaluator(),
dispatcher: dispatcher,
log: logger.Named("rules-engine"),
}
}
// Start registers event bus subscriptions to evaluate rules reactively.
func (e *Engine) Start(ctx context.Context) {
e.bus.Subscribe(events.EventPostCreated, e.handleEvent)
e.bus.Subscribe(events.EventPostUpdated, e.handleEvent)
e.bus.Subscribe(events.EventVariantUpdated, e.handleEvent)
e.bus.Subscribe(events.EventManualTrigger, e.handleEvent)
e.log.Info("Rules engine started, listening for events")
}
func (e *Engine) handleEvent(ctx context.Context, evt events.Event) error {
trigger := string(evt.Type)
rules, err := e.db.GetActiveRules(trigger)
if err != nil {
e.log.Error("Failed to fetch active rules", zap.String("trigger", trigger), zap.Error(err))
return err
}
if len(rules) == 0 {
return nil
}
// Fetch full entry with latest variants if entry ID is present
entry := evt.Entry
if entry.ID != "" {
if full, err := e.db.GetEntry(entry.ID); err == nil && full != nil {
entry = *full
}
}
for _, r := range rules {
if e.evaluator.Evaluate(r, entry) {
e.log.Info("Rule matched",
zap.String("rule_id", r.ID),
zap.String("rule_name", r.Name),
zap.String("entry_id", entry.ID),
zap.Int("actions_count", len(r.Actions)),
)
for _, action := range r.Actions {
if e.dispatcher != nil {
e.dispatcher.Dispatch(ctx, action, entry, evt.Depth+1)
}
}
}
}
return nil
}
+133
View File
@@ -0,0 +1,133 @@
package rules
import (
"fmt"
"regexp"
"strings"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
)
type Evaluator struct{}
func NewEvaluator() *Evaluator {
return &Evaluator{}
}
// Evaluate checks whether a FeedEntry satisfies a Rule's conditions.
func (e *Evaluator) Evaluate(rule models.Rule, entry models.FeedEntry) bool {
if len(rule.Conditions) == 0 {
return true
}
matchMode := strings.ToLower(rule.MatchMode)
if matchMode == "" || matchMode == "all" {
for _, cond := range rule.Conditions {
if !e.evaluateCondition(cond, entry) {
return false
}
}
return true
}
// matchMode == "any"
for _, cond := range rule.Conditions {
if e.evaluateCondition(cond, entry) {
return true
}
}
return false
}
func (e *Evaluator) evaluateCondition(cond models.RuleCondition, entry models.FeedEntry) bool {
actualValue := e.resolveField(cond.Field, entry)
return e.compare(actualValue, cond.Operator, cond.Value)
}
func (e *Evaluator) resolveField(field string, entry models.FeedEntry) interface{} {
switch strings.ToLower(field) {
case "feed_id":
return entry.FeedID
case "feed_title":
return entry.FeedTitle
case "title":
return entry.Title
case "author":
return entry.Author
case "url":
return entry.URL
case "content", "rss_content":
return entry.Content
case "is_read":
return entry.IsRead
case "is_starred":
return entry.IsStarred
}
// Variant field resolution: "variants.keywords", "variants.readability_full_text", "variants.tags"
if strings.HasPrefix(field, "variants.") {
variantKey := strings.TrimPrefix(field, "variants.")
if entry.Variants != nil {
if v, ok := entry.Variants[variantKey]; ok {
return v.TextValue
}
}
}
return nil
}
func (e *Evaluator) compare(actual interface{}, op string, expected interface{}) bool {
if actual == nil {
return op == "neq"
}
actualStr := fmt.Sprintf("%v", actual)
expectedStr := fmt.Sprintf("%v", expected)
switch strings.ToLower(op) {
case "eq", "==":
return strings.EqualFold(actualStr, expectedStr)
case "neq", "!=":
return !strings.EqualFold(actualStr, expectedStr)
case "contains":
return strings.Contains(strings.ToLower(actualStr), strings.ToLower(expectedStr))
case "not_contains":
return !strings.Contains(strings.ToLower(actualStr), strings.ToLower(expectedStr))
case "in":
// expected can be []interface{}, []string, or comma-separated string
switch v := expected.(type) {
case []interface{}:
for _, item := range v {
if strings.EqualFold(actualStr, fmt.Sprintf("%v", item)) {
return true
}
}
return false
case []string:
for _, item := range v {
if strings.EqualFold(actualStr, item) {
return true
}
}
return false
case string:
items := strings.Split(v, ",")
for _, item := range items {
if strings.EqualFold(strings.TrimSpace(actualStr), strings.TrimSpace(item)) {
return true
}
}
return false
}
return false
case "matches", "regex":
re, err := regexp.Compile(expectedStr)
if err != nil {
return false
}
return re.MatchString(actualStr)
default:
return false
}
}
+70
View File
@@ -0,0 +1,70 @@
package rules
import (
"testing"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
)
func TestRuleEvaluator(t *testing.T) {
eval := NewEvaluator()
entry := models.FeedEntry{
ID: "entry-1",
FeedID: "feed-hn",
FeedTitle: "Hacker News",
Title: "Announcing Go 1.24 Release",
Author: "Go Team",
Content: "<p>Go 1.24 brings new features...</p>",
Variants: map[string]models.EntryVariant{
"keywords": {
EntryID: "entry-1",
VariantKey: "keywords",
ContentType: "json",
TextValue: `["go", "compiler", "release"]`,
},
},
PublishedAt: time.Now(),
}
// Test 1: Title contains Go AND FeedID == feed-hn
rule1 := models.Rule{
ID: "r1",
Name: "Tech Feeds",
MatchMode: "all",
Conditions: []models.RuleCondition{
{Field: "feed_id", Operator: "eq", Value: "feed-hn"},
{Field: "title", Operator: "contains", Value: "Go 1.24"},
},
}
if !eval.Evaluate(rule1, entry) {
t.Error("expected rule1 to match entry")
}
// Test 2: Negative condition (author != Python Team)
rule2 := models.Rule{
ID: "r2",
Name: "Author check",
MatchMode: "all",
Conditions: []models.RuleCondition{
{Field: "author", Operator: "neq", Value: "Python Team"},
},
}
if !eval.Evaluate(rule2, entry) {
t.Error("expected rule2 to match entry")
}
// Test 3: Variant matching (variants.keywords contains compiler)
rule3 := models.Rule{
ID: "r3",
Name: "Keywords check",
MatchMode: "all",
Conditions: []models.RuleCondition{
{Field: "variants.keywords", Operator: "contains", Value: "compiler"},
},
}
if !eval.Evaluate(rule3, entry) {
t.Error("expected rule3 to match entry variant")
}
}
+179
View File
@@ -0,0 +1,179 @@
package worker
import (
"context"
"fmt"
"sync"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/events"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"go.uber.org/zap"
)
type ActionResult struct {
ResultVariant *models.EntryVariant
MarkRead *bool
MarkStarred *bool
}
type ActionHandler func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*ActionResult, error)
type Task struct {
Action models.RuleAction
Entry models.FeedEntry
Depth int
}
type Pool struct {
db *db.DB
bus *events.EventBus
handlers map[string]ActionHandler
queue chan Task
workers int
wg sync.WaitGroup
log *zap.Logger
mu sync.RWMutex
}
func NewPool(database *db.DB, bus *events.EventBus, workers, queueSize int) *Pool {
if workers <= 0 {
workers = 5
}
if queueSize <= 0 {
queueSize = 100
}
return &Pool{
db: database,
bus: bus,
handlers: make(map[string]ActionHandler),
queue: make(chan Task, queueSize),
workers: workers,
log: logger.Named("worker-pool"),
}
}
func (p *Pool) RegisterHandler(actionType string, handler ActionHandler) {
p.mu.Lock()
defer p.mu.Unlock()
p.handlers[actionType] = handler
p.log.Info("Registered action handler", zap.String("action", actionType))
}
func (p *Pool) Start(ctx context.Context) {
p.log.Info("Starting worker pool", zap.Int("worker_count", p.workers))
for i := 0; i < p.workers; i++ {
p.wg.Add(1)
go p.worker(ctx, i)
}
}
func (p *Pool) Stop() {
close(p.queue)
p.wg.Wait()
p.log.Info("Worker pool stopped")
}
func (p *Pool) Dispatch(ctx context.Context, action models.RuleAction, entry models.FeedEntry, depth int) {
select {
case p.queue <- Task{Action: action, Entry: entry, Depth: depth}:
default:
p.log.Warn("Worker pool queue full, task dropped",
zap.String("action", action.Type),
zap.String("entry_id", entry.ID),
)
}
}
func (p *Pool) ExecuteDirect(ctx context.Context, actionType string, entry models.FeedEntry, params map[string]interface{}) (*ActionResult, error) {
p.mu.RLock()
handler, ok := p.handlers[actionType]
p.mu.RUnlock()
if !ok {
return nil, fmt.Errorf("unknown action handler: %s", actionType)
}
return handler(ctx, entry, params)
}
func (p *Pool) worker(ctx context.Context, id int) {
defer p.wg.Done()
for {
select {
case <-ctx.Done():
return
case task, ok := <-p.queue:
if !ok {
return
}
p.processTask(ctx, task)
}
}
}
func (p *Pool) processTask(ctx context.Context, task Task) {
p.mu.RLock()
handler, ok := p.handlers[task.Action.Type]
p.mu.RUnlock()
if !ok {
p.log.Warn("No handler registered for action", zap.String("type", task.Action.Type))
return
}
start := time.Now()
res, err := handler(ctx, task.Entry, task.Action.Params)
duration := time.Since(start)
if err != nil {
p.log.Error("Action execution failed",
zap.String("action", task.Action.Type),
zap.String("entry_id", task.Entry.ID),
zap.Duration("duration", duration),
zap.Error(err),
)
return
}
if res == nil {
return
}
// 1. Save Variant if returned
if res.ResultVariant != nil {
if err := p.db.SetVariant(*res.ResultVariant); err != nil {
p.log.Error("Failed to save entry variant",
zap.String("entry_id", task.Entry.ID),
zap.String("variant_key", res.ResultVariant.VariantKey),
zap.Error(err),
)
} else {
p.log.Info("Entry variant saved",
zap.String("entry_id", task.Entry.ID),
zap.String("variant_key", res.ResultVariant.VariantKey),
zap.Duration("duration", duration),
)
// Publish EventVariantUpdated to enable downstream chained triggers
if p.bus != nil {
p.bus.Publish(ctx, events.Event{
Type: events.EventVariantUpdated,
Entry: task.Entry,
VariantKey: res.ResultVariant.VariantKey,
Depth: task.Depth,
})
}
}
}
// 2. Apply Read/Star updates
if res.MarkRead != nil {
_ = p.db.UpdateEntryRead(task.Entry.ID, *res.MarkRead)
}
if res.MarkStarred != nil {
_ = p.db.UpdateEntryStarred(task.Entry.ID, *res.MarkStarred)
}
}
+22 -30
View File
@@ -159,6 +159,19 @@ export const App: React.FC = () => {
setFocusedIndex(entries.findIndex((e) => e.id === selectedEntryId)); setFocusedIndex(entries.findIndex((e) => e.id === selectedEntryId));
return; return;
} }
// If not found in current list, load directly so reading pane stays populated
api.getEntry(selectedEntryId).then((entry) => {
setActiveArticle(entry);
}).catch(() => {
// fallback to top
if (entries[0]) {
setFocusedIndex(0);
setActiveArticle(entries[0]);
setSelectedEntryId(entries[0].id);
localStorage.setItem('rsspipe_selected_entry_id', entries[0].id);
}
});
return;
} }
setFocusedIndex(0); setFocusedIndex(0);
@@ -177,36 +190,7 @@ export const App: React.FC = () => {
} }
}; };
// Auto-Refresh & Background Polling (Every 30s + on focus / visibility change) // Refresh data only on initial mount or when user clicks refresh / changes filter
useEffect(() => {
if (!currentUser) return;
const interval = setInterval(() => {
loadSidebarData();
loadEntries();
}, 30000);
const handleWindowFocus = () => {
loadSidebarData();
loadEntries();
};
const handleVisibilityChange = () => {
if (document.visibilityState === 'visible') {
loadSidebarData();
loadEntries();
}
};
window.addEventListener('focus', handleWindowFocus);
document.addEventListener('visibilitychange', handleVisibilityChange);
return () => {
clearInterval(interval);
window.removeEventListener('focus', handleWindowFocus);
document.removeEventListener('visibilitychange', handleVisibilityChange);
};
}, [currentUser, loadSidebarData, loadEntries]);
// Toggle Read Action with optimistic update // Toggle Read Action with optimistic update
const handleToggleRead = async (entry: FeedEntry, e?: React.MouseEvent) => { const handleToggleRead = async (entry: FeedEntry, e?: React.MouseEvent) => {
@@ -516,6 +500,10 @@ export const App: React.FC = () => {
onToggleRead={handleToggleRead} onToggleRead={handleToggleRead}
onToggleStar={handleToggleStar} onToggleStar={handleToggleStar}
onMarkAllRead={handleMarkAllRead} onMarkAllRead={handleMarkAllRead}
onUpdateEntry={(updated) => {
setActiveArticle(updated);
setEntries((prev) => prev.map((e) => (e.id === updated.id ? updated : e)));
}}
sidebarCollapsed={sidebarCollapsed} sidebarCollapsed={sidebarCollapsed}
/> />
) : viewMode === 'waterfall' ? ( ) : viewMode === 'waterfall' ? (
@@ -526,6 +514,10 @@ export const App: React.FC = () => {
entries={entries} entries={entries}
onToggleRead={handleToggleRead} onToggleRead={handleToggleRead}
onToggleStar={handleToggleStar} onToggleStar={handleToggleStar}
onUpdateEntry={(updated) => {
setActiveArticle(updated);
setEntries((prev) => prev.map((e) => (e.id === updated.id ? updated : e)));
}}
/> />
) : ( ) : (
<SettingsView <SettingsView
+364 -20
View File
@@ -1,5 +1,5 @@
import React, { useState } from 'react'; import React, { useState, useEffect } from 'react';
import { Category, Feed, Stats, User } from '../types'; import { Category, Feed, Rule, Stats, User } from '../types';
import { api } from '../services/api'; import { api } from '../services/api';
import { import {
Rss, Rss,
@@ -22,6 +22,8 @@ import {
Eye, Eye,
Lock, Lock,
Key, Key,
Sparkles,
Zap,
} from 'lucide-react'; } from 'lucide-react';
interface SettingsViewProps { interface SettingsViewProps {
@@ -41,7 +43,7 @@ export const SettingsView: React.FC<SettingsViewProps> = ({
onRefreshData, onRefreshData,
onOpenAddFeed, onOpenAddFeed,
}) => { }) => {
const [activeTab, setActiveTab] = useState<'feeds' | 'categories' | 'viewer' | 'opml' | 'account'>('feeds'); const [activeTab, setActiveTab] = useState<'feeds' | 'categories' | 'rules' | 'viewer' | 'opml' | 'account'>('feeds');
const [autoReadBottom, setAutoReadBottom] = useState<boolean>(() => { const [autoReadBottom, setAutoReadBottom] = useState<boolean>(() => {
return localStorage.getItem('rsspipe_auto_read_bottom') !== 'false'; return localStorage.getItem('rsspipe_auto_read_bottom') !== 'false';
}); });
@@ -50,6 +52,18 @@ export const SettingsView: React.FC<SettingsViewProps> = ({
const [loading, setLoading] = useState(false); const [loading, setLoading] = useState(false);
const [message, setMessage] = useState<string | null>(null); const [message, setMessage] = useState<string | null>(null);
// Rules State
const [rules, setRules] = useState<Rule[]>([]);
const [rulesLoading, setRulesLoading] = useState(false);
const [showRuleModal, setShowRuleModal] = useState(false);
const [ruleName, setRuleName] = useState('');
const [ruleTrigger, setRuleTrigger] = useState('on_post_created');
const [ruleMatchMode, setRuleMatchMode] = useState('all');
const [ruleField, setRuleField] = useState('feed_id');
const [ruleOp, setRuleOp] = useState('eq');
const [ruleVal, setRuleVal] = useState('');
const [ruleActionType, setRuleActionType] = useState('readability_scrape');
// Edit Feed Modal State // Edit Feed Modal State
const [editingFeed, setEditingFeed] = useState<Feed | null>(null); const [editingFeed, setEditingFeed] = useState<Feed | null>(null);
const [editTitle, setEditTitle] = useState(''); const [editTitle, setEditTitle] = useState('');
@@ -110,6 +124,78 @@ export const SettingsView: React.FC<SettingsViewProps> = ({
} }
}; };
useEffect(() => {
loadRules();
}, []);
const loadRules = async () => {
setRulesLoading(true);
try {
const data = await api.listRules();
setRules(data || []);
} catch {
// ignore
} finally {
setRulesLoading(false);
}
};
const handleCreateRule = async (e: React.FormEvent) => {
e.preventDefault();
if (!ruleName.trim()) return;
try {
await api.createRule({
name: ruleName.trim(),
trigger: ruleTrigger,
match_mode: ruleMatchMode,
conditions: [
{
field: ruleField,
operator: ruleOp,
value: ruleVal.trim(),
},
],
actions: [
{
type: ruleActionType,
},
],
});
setShowRuleModal(false);
setRuleName('');
setRuleVal('');
loadRules();
} catch (err: any) {
alert(err.message || 'Failed to create rule');
}
};
const handleToggleRuleActive = async (rule: Rule) => {
try {
await api.updateRule(rule.id, {
name: rule.name,
trigger: rule.trigger,
match_mode: rule.match_mode,
conditions: rule.conditions,
actions: rule.actions,
is_active: !rule.is_active,
});
loadRules();
} catch (err: any) {
alert(err.message || 'Failed to toggle rule');
}
};
const handleDeleteRule = async (id: string) => {
if (!confirm('Are you sure you want to delete this rule?')) return;
try {
await api.deleteRule(id);
loadRules();
} catch (err: any) {
alert(err.message || 'Failed to delete rule');
}
};
// Filter feeds // Filter feeds
const filteredFeeds = feeds.filter( const filteredFeeds = feeds.filter(
(f) => (f) =>
@@ -148,14 +234,20 @@ export const SettingsView: React.FC<SettingsViewProps> = ({
} }
}; };
const handleRefreshFeed = async (feedId: string) => { const handleCreateCategory = async (e: React.FormEvent) => {
e.preventDefault();
if (!newCatTitle.trim()) return;
setLoading(true);
try { try {
await api.refreshFeed(feedId); await api.createCategory(newCatTitle.trim());
setNewCatTitle('');
onRefreshData(); onRefreshData();
setMessage('Feed refreshed successfully.'); setMessage(`Category "${newCatTitle}" created.`);
setTimeout(() => setMessage(null), 3000); setTimeout(() => setMessage(null), 3000);
} catch (err: any) { } catch (err: any) {
alert(err.message || 'Failed to refresh feed'); alert(err.message || 'Failed to create category');
} finally {
setLoading(false);
} }
}; };
@@ -170,28 +262,22 @@ export const SettingsView: React.FC<SettingsViewProps> = ({
setMessage(`Deleted category "${deleteConfirm.title}".`); setMessage(`Deleted category "${deleteConfirm.title}".`);
} }
onRefreshData(); onRefreshData();
setTimeout(() => setMessage(null), 3500); setTimeout(() => setMessage(null), 3000);
} catch (err: any) { } catch (err: any) {
alert(err.message || 'Delete operation failed'); alert(err.message || 'Failed to delete');
} finally { } finally {
setDeleteConfirm(null); setDeleteConfirm(null);
} }
}; };
const handleCreateCategory = async (e: React.FormEvent) => { const handleRefreshFeed = async (feedId: string) => {
e.preventDefault();
if (!newCatTitle.trim()) return;
setLoading(true);
try { try {
await api.createCategory(newCatTitle.trim()); await api.refreshFeed(feedId);
setNewCatTitle('');
onRefreshData(); onRefreshData();
setMessage('Category created.'); setMessage('Feed refreshed successfully.');
setTimeout(() => setMessage(null), 3000); setTimeout(() => setMessage(null), 2500);
} catch (err: any) { } catch (err: any) {
alert(err.message || 'Failed to create category'); alert(err.message || 'Failed to refresh feed');
} finally {
setLoading(false);
} }
}; };
@@ -234,6 +320,14 @@ export const SettingsView: React.FC<SettingsViewProps> = ({
<span>Categories ({categories.length})</span> <span>Categories ({categories.length})</span>
</button> </button>
<button
className={`settings-nav-item ${activeTab === 'rules' ? 'active' : ''}`}
onClick={() => setActiveTab('rules')}
>
<Sparkles size={15} />
<span>Rules & Hooks ({rules.length})</span>
</button>
<button <button
className={`settings-nav-item ${activeTab === 'viewer' ? 'active' : ''}`} className={`settings-nav-item ${activeTab === 'viewer' ? 'active' : ''}`}
onClick={() => setActiveTab('viewer')} onClick={() => setActiveTab('viewer')}
@@ -514,6 +608,118 @@ export const SettingsView: React.FC<SettingsViewProps> = ({
</div> </div>
)} )}
{/* Rules & Automation Hooks Tab */}
{activeTab === 'rules' && (
<div>
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', marginBottom: '1.25rem' }}>
<div>
<h2 style={{ fontSize: '1.2rem', fontWeight: 700, color: 'var(--text-main)' }}>
Automation Rules & Pipeline Hooks
</h2>
<p style={{ fontSize: '0.8rem', color: 'var(--text-muted)' }}>
Filter feeds and automatically trigger enrichments (Readability Scrape, Star, Read) on post arrival or variant updates.
</p>
</div>
<button className="btn-primary" onClick={() => setShowRuleModal(true)}>
<Plus size={14} />
<span>Create Rule</span>
</button>
</div>
{rulesLoading ? (
<div style={{ textAlign: 'center', padding: '3rem', color: 'var(--text-muted)' }}>
<Loader2 size={24} className="spin" style={{ margin: '0 auto 0.5rem auto' }} />
<p>Loading rules...</p>
</div>
) : rules.length === 0 ? (
<div className="settings-card" style={{ textAlign: 'center', padding: '3rem 1rem' }}>
<div style={{ fontSize: '2rem', marginBottom: '0.5rem' }}>⚡</div>
<h3 style={{ fontSize: '1rem', fontWeight: 600 }}>No Automation Rules Defined</h3>
<p style={{ fontSize: '0.8rem', color: 'var(--text-muted)', maxWidth: 400, margin: '0.5rem auto 1.5rem auto' }}>
Create rules to automatically scrape full-text content or apply tags whenever new articles arrive from specific feeds.
</p>
<button className="btn-primary" onClick={() => setShowRuleModal(true)}>
<Plus size={14} />
<span>Create Your First Rule</span>
</button>
</div>
) : (
<div style={{ display: 'flex', flexDirection: 'column', gap: '0.75rem' }}>
{rules.map((rule) => (
<div key={rule.id} className="settings-card" style={{ padding: '1rem 1.25rem' }}>
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
<div style={{ display: 'flex', alignItems: 'center', gap: '0.75rem' }}>
<div
style={{
width: 32,
height: 32,
borderRadius: '50%',
background: rule.is_active ? 'rgba(16, 185, 129, 0.1)' : 'var(--bg-tertiary)',
color: rule.is_active ? 'var(--success)' : 'var(--text-muted)',
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
}}
>
<Zap size={16} />
</div>
<div>
<div style={{ display: 'flex', alignItems: 'center', gap: '0.5rem' }}>
<span style={{ fontWeight: 600, fontSize: '0.9rem' }}>{rule.name}</span>
<span className="tag-badge" style={{ fontSize: '0.7rem' }}>
Trigger: {rule.trigger}
</span>
{!rule.is_active && (
<span style={{ fontSize: '0.7rem', color: 'var(--text-muted)' }}>(Inactive)</span>
)}
</div>
<div style={{ fontSize: '0.75rem', color: 'var(--text-muted)', marginTop: '0.25rem', display: 'flex', flexWrap: 'wrap', gap: '0.5rem' }}>
{rule.conditions && rule.conditions.length > 0 ? (
rule.conditions.map((c, i) => (
<span key={i} style={{ background: 'var(--bg-tertiary)', padding: '2px 6px', borderRadius: 4 }}>
If <strong>{c.field}</strong> {c.operator} <em>"{String(c.value)}"</em>
</span>
))
) : (
<span>Always matches</span>
)}
<span>➜</span>
{rule.actions?.map((a, i) => (
<span key={i} style={{ background: 'rgba(56, 189, 248, 0.1)', color: 'var(--accent)', padding: '2px 6px', borderRadius: 4, fontWeight: 500 }}>
Action: {a.type}
</span>
))}
</div>
</div>
</div>
<div style={{ display: 'flex', alignItems: 'center', gap: '0.5rem' }}>
<button
className={`btn-secondary`}
style={{ fontSize: '0.75rem', padding: '0.3rem 0.6rem' }}
onClick={() => handleToggleRuleActive(rule)}
>
{rule.is_active ? 'Disable' : 'Enable'}
</button>
<button
className="btn-icon"
style={{ color: 'var(--danger)' }}
onClick={() => handleDeleteRule(rule.id)}
title="Delete Rule"
>
<Trash2 size={15} />
</button>
</div>
</div>
</div>
))}
</div>
)}
</div>
)}
{/* 3. Viewer Preferences Tab */} {/* 3. Viewer Preferences Tab */}
{activeTab === 'viewer' && ( {activeTab === 'viewer' && (
<div> <div>
@@ -975,6 +1181,144 @@ export const SettingsView: React.FC<SettingsViewProps> = ({
</div> </div>
</div> </div>
)} )}
{/* Create Automation Rule Modal */}
{showRuleModal && (
<div className="modal-overlay" onClick={() => setShowRuleModal(false)}>
<div className="modal-card" style={{ maxWidth: 520 }} onClick={(e) => e.stopPropagation()}>
<div className="modal-header">
<div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
<Sparkles size={17} style={{ color: 'var(--accent)' }} />
<h3>Create Automation / Hook Rule</h3>
</div>
<button className="btn-icon" onClick={() => setShowRuleModal(false)}>
<X size={16} />
</button>
</div>
<form onSubmit={handleCreateRule}>
<div className="modal-body">
<div className="form-group">
<label className="form-label">Rule Name</label>
<input
type="text"
className="form-input"
placeholder="e.g. Auto Scrape TechCrunch"
value={ruleName}
onChange={(e) => setRuleName(e.target.value)}
required
/>
</div>
<div style={{ display: 'grid', gridTemplateColumns: '1fr 1fr', gap: '0.75rem' }}>
<div className="form-group">
<label className="form-label">Trigger Event</label>
<select
className="form-select"
value={ruleTrigger}
onChange={(e) => setRuleTrigger(e.target.value)}
>
<option value="on_post_created">On New Post Ingested</option>
<option value="on_variant_updated">On Variant Updated</option>
<option value="manual">Manual Execution</option>
</select>
</div>
<div className="form-group">
<label className="form-label">Condition Matching</label>
<select
className="form-select"
value={ruleMatchMode}
onChange={(e) => setRuleMatchMode(e.target.value)}
>
<option value="all">All Conditions Match (AND)</option>
<option value="any">Any Condition Matches (OR)</option>
</select>
</div>
</div>
<div style={{ background: 'var(--bg-tertiary)', padding: '0.85rem', borderRadius: 8, margin: '0.5rem 0 1rem 0' }}>
<label className="form-label" style={{ marginBottom: '0.5rem' }}>Condition Predicate</label>
<div style={{ display: 'grid', gridTemplateColumns: '1.2fr 1fr', gap: '0.5rem', marginBottom: '0.5rem' }}>
<select
className="form-select"
value={ruleField}
onChange={(e) => setRuleField(e.target.value)}
>
<option value="feed_id">Feed ID</option>
<option value="feed_title">Feed Title</option>
<option value="title">Article Title</option>
<option value="author">Author</option>
<option value="url">URL</option>
<option value="variants.keywords">Keywords (Variant)</option>
<option value="variants.tags">Tags (Variant)</option>
</select>
<select
className="form-select"
value={ruleOp}
onChange={(e) => setRuleOp(e.target.value)}
>
<option value="eq">Equals (==)</option>
<option value="neq">Not Equals (!=)</option>
<option value="contains">Contains substring</option>
<option value="not_contains">Does not contain</option>
<option value="matches">Regex Match</option>
</select>
</div>
{ruleField === 'feed_id' ? (
<select
className="form-select"
value={ruleVal}
onChange={(e) => setRuleVal(e.target.value)}
required
>
<option value="">Select Subscribed Feed...</option>
{feeds.map((f) => (
<option key={f.id} value={f.id}>
{f.title}
</option>
))}
</select>
) : (
<input
type="text"
className="form-input"
placeholder="Expected value (e.g. AI, OpenAI)"
value={ruleVal}
onChange={(e) => setRuleVal(e.target.value)}
required
/>
)}
</div>
<div className="form-group">
<label className="form-label">Action To Trigger</label>
<select
className="form-select"
value={ruleActionType}
onChange={(e) => setRuleActionType(e.target.value)}
>
<option value="readability_scrape">⚡ Scrape Full-Text (Readability)</option>
<option value="mark_starred">⭐ Mark as Starred</option>
<option value="mark_read">✓ Mark as Read</option>
</select>
</div>
</div>
<div style={{ padding: '0.85rem 1.25rem', borderTop: '1px solid var(--border)', display: 'flex', justifyContent: 'flex-end', gap: '0.5rem' }}>
<button type="button" className="btn-secondary" onClick={() => setShowRuleModal(false)}>
Cancel
</button>
<button type="submit" className="btn-primary">
Create Rule
</button>
</div>
</form>
</div>
</div>
)}
</div> </div>
); );
}; };
+153 -2
View File
@@ -1,7 +1,8 @@
import React, { useEffect, useRef } from 'react'; import React, { useEffect, useRef, useState } from 'react';
import { Category, Feed, FeedEntry, FilterState, Stats } from '../types'; import { Category, Feed, FeedEntry, FilterState, Stats } from '../types';
import { Sidebar } from './Sidebar'; import { Sidebar } from './Sidebar';
import { formatRelativeDate, stripHtml, estimateReadingTime } from '../utils'; import { formatRelativeDate, stripHtml, estimateReadingTime } from '../utils';
import { api } from '../services/api';
import { import {
Star, Star,
CheckCircle2, CheckCircle2,
@@ -10,6 +11,11 @@ import {
Clock, Clock,
User as UserIcon, User as UserIcon,
CheckCheck, CheckCheck,
FileText,
Zap,
Sparkles,
Volume2,
Loader2,
} from 'lucide-react'; } from 'lucide-react';
interface ThreePaneViewProps { interface ThreePaneViewProps {
@@ -25,6 +31,7 @@ interface ThreePaneViewProps {
onToggleRead: (entry: FeedEntry, e?: React.MouseEvent) => void; onToggleRead: (entry: FeedEntry, e?: React.MouseEvent) => void;
onToggleStar: (entry: FeedEntry, e?: React.MouseEvent) => void; onToggleStar: (entry: FeedEntry, e?: React.MouseEvent) => void;
onMarkAllRead: () => void; onMarkAllRead: () => void;
onUpdateEntry?: (entry: FeedEntry) => void;
sidebarCollapsed: boolean; sidebarCollapsed: boolean;
} }
@@ -41,10 +48,13 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
onToggleRead, onToggleRead,
onToggleStar, onToggleStar,
onMarkAllRead, onMarkAllRead,
onUpdateEntry,
sidebarCollapsed, sidebarCollapsed,
}) => { }) => {
const cardRefs = useRef<(HTMLDivElement | null)[]>([]); const cardRefs = useRef<(HTMLDivElement | null)[]>([]);
const readerPaneRef = useRef<HTMLDivElement>(null); const readerPaneRef = useRef<HTMLDivElement>(null);
const [selectedVariantKey, setSelectedVariantKey] = useState<string>('original');
const [isScraping, setIsScraping] = useState<boolean>(false);
const selectedEntry = activeArticle || entries[selectedIndex] || entries[0]; const selectedEntry = activeArticle || entries[selectedIndex] || entries[0];
@@ -58,6 +68,13 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
} }
}, [selectedIndex, entries.length]); }, [selectedIndex, entries.length]);
// Reset variant selection when article changes
useEffect(() => {
if (selectedEntry?.variants?.['readability_full_text']) {
// If full text is available, you can choose to stay on current or default to original
}
}, [selectedEntry?.id]);
// When switching selected article, reset reader scroll position to top and handle 2s timer for short content // When switching selected article, reset reader scroll position to top and handle 2s timer for short content
useEffect(() => { useEffect(() => {
if (readerPaneRef.current) { if (readerPaneRef.current) {
@@ -108,6 +125,22 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
} }
}; };
const handleScrapeFullText = async () => {
if (!selectedEntry || isScraping) return;
setIsScraping(true);
try {
const updated = await api.scrapeEntry(selectedEntry.id);
if (onUpdateEntry) {
onUpdateEntry(updated);
}
setSelectedVariantKey('readability_full_text');
} catch (err: any) {
alert(`Scraping failed: ${err.message || err}`);
} finally {
setIsScraping(false);
}
};
const getListTitle = () => { const getListTitle = () => {
if (filter.search) return `Search: "${filter.search}"`; if (filter.search) return `Search: "${filter.search}"`;
if (filter.feedId) { if (filter.feedId) {
@@ -123,6 +156,21 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
return 'Inbox'; return 'Inbox';
}; };
const hasReadability = Boolean(selectedEntry?.variants?.['readability_full_text']);
const hasAISummary = Boolean(selectedEntry?.variants?.['ai_summary']);
const hasTTS = Boolean(selectedEntry?.variants?.['tts'] || selectedEntry?.variants?.['podcast_tts']);
const getRenderedContent = () => {
if (!selectedEntry) return '';
if (selectedVariantKey === 'readability_full_text' && selectedEntry.variants?.['readability_full_text']) {
return selectedEntry.variants['readability_full_text'].text_value || '';
}
if (selectedVariantKey === 'ai_summary' && selectedEntry.variants?.['ai_summary']) {
return `<div style="padding: 1rem; border-left: 3px solid var(--accent); background: rgba(56, 189, 248, 0.05); margin-bottom: 1.5rem; border-radius: 4px;"><strong>AI Summary:</strong><p>${selectedEntry.variants['ai_summary'].text_value}</p></div>` + (selectedEntry.content || selectedEntry.raw_content || '');
}
return selectedEntry.content || selectedEntry.raw_content || '';
};
return ( return (
<div className="three-pane-container"> <div className="three-pane-container">
{/* Pane 1: Sidebar (Collapsible & Independently Scrollable) */} {/* Pane 1: Sidebar (Collapsible & Independently Scrollable) */}
@@ -235,6 +283,96 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
> >
<CheckCircle2 size={16} /> <CheckCircle2 size={16} />
</button> </button>
{/* Sliding Ball Capsule Toggle Button */}
<button
type="button"
className={`capsule-switch-btn ${
selectedVariantKey === 'readability_full_text' ? 'active' : ''
} ${isScraping ? 'scraping' : ''}`}
onClick={() => {
if (isScraping) return;
if (selectedVariantKey === 'readability_full_text') {
setSelectedVariantKey('original');
} else {
if (hasReadability) {
setSelectedVariantKey('readability_full_text');
} else {
handleScrapeFullText();
}
}
}}
title={
selectedVariantKey === 'readability_full_text'
? 'Switch to Original RSS content'
: hasReadability
? 'Switch to Clean Full Text'
: 'Scrape and view Clean Full Text'
}
>
<div className="capsule-switch-track">
<span className="capsule-switch-label">
{isScraping
? 'Scraping...'
: selectedVariantKey === 'readability_full_text'
? 'Full Text'
: 'Original'}
</span>
<div className="capsule-switch-thumb">
{isScraping ? (
<Loader2 size={12} className="spin" />
) : selectedVariantKey === 'readability_full_text' ? (
<Zap size={12} />
) : (
<FileText size={12} />
)}
</div>
</div>
</button>
{hasAISummary && (
<button
type="button"
className={`btn-secondary ${selectedVariantKey === 'ai_summary' ? 'active' : ''}`}
style={{
border: 'none',
borderRadius: '16px',
padding: '3px 9px',
fontSize: '0.72rem',
background: selectedVariantKey === 'ai_summary' ? 'var(--card-bg)' : 'transparent',
color: selectedVariantKey === 'ai_summary' ? '#a855f7' : 'var(--text-muted)',
boxShadow: selectedVariantKey === 'ai_summary' ? '0 1px 3px rgba(0,0,0,0.1)' : 'none',
marginLeft: '0.4rem',
}}
onClick={() => setSelectedVariantKey('ai_summary')}
title="AI Summary"
>
<Sparkles size={12} style={{ marginRight: 4 }} />
Summary
</button>
)}
{hasTTS && (
<button
type="button"
className={`btn-secondary ${selectedVariantKey === 'tts' ? 'active' : ''}`}
style={{
border: 'none',
borderRadius: '16px',
padding: '3px 9px',
fontSize: '0.72rem',
background: selectedVariantKey === 'tts' ? 'var(--card-bg)' : 'transparent',
color: selectedVariantKey === 'tts' ? '#10b981' : 'var(--text-muted)',
boxShadow: selectedVariantKey === 'tts' ? '0 1px 3px rgba(0,0,0,0.1)' : 'none',
marginLeft: '0.4rem',
}}
onClick={() => setSelectedVariantKey('tts')}
title="Audio Stream"
>
<Volume2 size={12} style={{ marginRight: 4 }} />
Audio
</button>
)}
</div> </div>
<div style={{ display: 'flex', alignItems: 'center', gap: '0.5rem' }}> <div style={{ display: 'flex', alignItems: 'center', gap: '0.5rem' }}>
@@ -295,10 +433,22 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
</a> </a>
</h1> </h1>
{/* Audio player if TTS variant is selected */}
{selectedVariantKey === 'tts' && (
<div style={{ margin: '1rem 0', padding: '1rem', background: 'var(--bg-tertiary)', borderRadius: '8px' }}>
<p style={{ margin: '0 0 0.5rem 0', fontSize: '0.85rem', fontWeight: 600 }}>🎧 Audio Stream:</p>
<audio
controls
style={{ width: '100%' }}
src={`/api/entries/${selectedEntry.id}/variants/tts/stream`}
/>
</div>
)}
<div <div
className="reader-body" className="reader-body"
dangerouslySetInnerHTML={{ dangerouslySetInnerHTML={{
__html: selectedEntry.content || selectedEntry.raw_content || '', __html: getRenderedContent(),
}} }}
/> />
</div> </div>
@@ -314,3 +464,4 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
</div> </div>
); );
}; };
+150 -3
View File
@@ -1,4 +1,5 @@
import React, { useState, useRef } from 'react'; import React, { useState, useEffect, useRef } from 'react';
import { api } from '../services/api';
import { Category, FeedEntry, FilterState } from '../types'; import { Category, FeedEntry, FilterState } from '../types';
import { extractFirstImage, formatRelativeDate, stripHtml, estimateReadingTime } from '../utils'; import { extractFirstImage, formatRelativeDate, stripHtml, estimateReadingTime } from '../utils';
import { import {
@@ -13,6 +14,11 @@ import {
CheckCircle2, CheckCircle2,
Clock, Clock,
User as UserIcon, User as UserIcon,
FileText,
Zap,
Loader2,
Sparkles,
Volume2,
} from 'lucide-react'; } from 'lucide-react';
interface WaterfallViewProps { interface WaterfallViewProps {
@@ -22,6 +28,7 @@ interface WaterfallViewProps {
entries: FeedEntry[]; entries: FeedEntry[];
onToggleRead: (entry: FeedEntry, e?: React.MouseEvent) => void; onToggleRead: (entry: FeedEntry, e?: React.MouseEvent) => void;
onToggleStar: (entry: FeedEntry, e?: React.MouseEvent) => void; onToggleStar: (entry: FeedEntry, e?: React.MouseEvent) => void;
onUpdateEntry?: (entry: FeedEntry) => void;
} }
export const WaterfallView: React.FC<WaterfallViewProps> = ({ export const WaterfallView: React.FC<WaterfallViewProps> = ({
@@ -31,10 +38,51 @@ export const WaterfallView: React.FC<WaterfallViewProps> = ({
entries, entries,
onToggleRead, onToggleRead,
onToggleStar, onToggleStar,
onUpdateEntry,
}) => { }) => {
const [selectedEntry, setSelectedEntry] = useState<FeedEntry | null>(null); const [selectedEntry, setSelectedEntry] = useState<FeedEntry | null>(null);
const [selectedVariantKey, setSelectedVariantKey] = useState<string>('original');
const [isScraping, setIsScraping] = useState(false);
const readerScrollRef = useRef<HTMLDivElement>(null); const readerScrollRef = useRef<HTMLDivElement>(null);
// Reset variant selection when selected article changes
useEffect(() => {
setSelectedVariantKey('original');
}, [selectedEntry?.id]);
const handleScrapeFullText = async () => {
if (!selectedEntry || isScraping) return;
setIsScraping(true);
try {
const updated = await api.scrapeEntry(selectedEntry.id);
setSelectedEntry(updated);
onUpdateEntry?.(updated);
setSelectedVariantKey('readability_full_text');
} catch (err: any) {
alert(`Scraping failed: ${err.message || err}`);
} finally {
setIsScraping(false);
}
};
const hasReadability = Boolean(selectedEntry?.variants?.['readability_full_text']);
const hasAISummary = Boolean(selectedEntry?.variants?.['ai_summary']);
const hasTTS = Boolean(selectedEntry?.variants?.['tts'] || selectedEntry?.variants?.['podcast_tts']);
const getRenderedContent = () => {
if (!selectedEntry) return '';
if (selectedVariantKey === 'readability_full_text' && selectedEntry.variants?.['readability_full_text']) {
return selectedEntry.variants['readability_full_text'].text_value || '';
}
if (selectedVariantKey === 'ai_summary' && selectedEntry.variants?.['ai_summary']) {
return (
`<div style="padding: 1rem; border-left: 3px solid var(--accent); background: rgba(56, 189, 248, 0.05); margin-bottom: 1.5rem; border-radius: 4px;"><strong>AI Summary:</strong><p>${selectedEntry.variants['ai_summary'].text_value}</p></div>` +
(selectedEntry.content || selectedEntry.raw_content || '')
);
}
return selectedEntry.content || selectedEntry.raw_content || '';
};
// When card is clicked, open in full-screen reader view and immediately mark as read // When card is clicked, open in full-screen reader view and immediately mark as read
const handleCardClick = (entry: FeedEntry) => { const handleCardClick = (entry: FeedEntry) => {
setSelectedEntry(entry); setSelectedEntry(entry);
@@ -232,7 +280,95 @@ export const WaterfallView: React.FC<WaterfallViewProps> = ({
{selectedEntry.feed_title} {selectedEntry.feed_title}
</span> </span>
<div style={{ display: 'flex', alignItems: 'center', gap: 4 }}> <div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
{/* Sliding Ball Capsule Toggle Button */}
<button
type="button"
className={`capsule-switch-btn ${
selectedVariantKey === 'readability_full_text' ? 'active' : ''
} ${isScraping ? 'scraping' : ''}`}
onClick={() => {
if (isScraping) return;
if (selectedVariantKey === 'readability_full_text') {
setSelectedVariantKey('original');
} else {
if (hasReadability) {
setSelectedVariantKey('readability_full_text');
} else {
handleScrapeFullText();
}
}
}}
title={
selectedVariantKey === 'readability_full_text'
? 'Switch to Original RSS content'
: hasReadability
? 'Switch to Clean Full Text'
: 'Scrape and view Clean Full Text'
}
>
<div className="capsule-switch-track">
<span className="capsule-switch-label">
{isScraping
? 'Scraping...'
: selectedVariantKey === 'readability_full_text'
? 'Full Text'
: 'Original'}
</span>
<div className="capsule-switch-thumb">
{isScraping ? (
<Loader2 size={12} className="spin" />
) : selectedVariantKey === 'readability_full_text' ? (
<Zap size={12} />
) : (
<FileText size={12} />
)}
</div>
</div>
</button>
{hasAISummary && (
<button
type="button"
className={`btn-secondary ${selectedVariantKey === 'ai_summary' ? 'active' : ''}`}
style={{
border: 'none',
borderRadius: '16px',
padding: '3px 8px',
fontSize: '0.7rem',
background: selectedVariantKey === 'ai_summary' ? 'var(--card-bg)' : 'transparent',
color: selectedVariantKey === 'ai_summary' ? '#a855f7' : 'var(--text-muted)',
boxShadow: selectedVariantKey === 'ai_summary' ? '0 1px 3px rgba(0,0,0,0.1)' : 'none',
}}
onClick={() => setSelectedVariantKey('ai_summary')}
title="AI Summary"
>
<Sparkles size={12} style={{ marginRight: 3 }} />
Summary
</button>
)}
{hasTTS && (
<button
type="button"
className={`btn-secondary ${selectedVariantKey === 'tts' ? 'active' : ''}`}
style={{
border: 'none',
borderRadius: '16px',
padding: '3px 8px',
fontSize: '0.7rem',
background: selectedVariantKey === 'tts' ? 'var(--card-bg)' : 'transparent',
color: selectedVariantKey === 'tts' ? '#10b981' : 'var(--text-muted)',
boxShadow: selectedVariantKey === 'tts' ? '0 1px 3px rgba(0,0,0,0.1)' : 'none',
}}
onClick={() => setSelectedVariantKey('tts')}
title="Audio Stream"
>
<Volume2 size={12} style={{ marginRight: 3 }} />
Audio
</button>
)}
<button <button
className="btn-icon" className="btn-icon"
style={{ style={{
@@ -312,12 +448,23 @@ export const WaterfallView: React.FC<WaterfallViewProps> = ({
</a> </a>
</h1> </h1>
{/* Dynamic Content Variant Rendering */}
{selectedVariantKey === 'tts' && hasTTS ? (
<div className="audio-player-container" style={{ margin: '1.5rem 0', padding: '1rem', background: 'var(--bg-card-hover)', borderRadius: '12px' }}>
<audio
controls
src={`/api/entries/${selectedEntry.id}/variants/tts/stream`}
style={{ width: '100%' }}
/>
</div>
) : (
<div <div
className="reader-body" className="reader-body"
dangerouslySetInnerHTML={{ dangerouslySetInnerHTML={{
__html: selectedEntry.content || selectedEntry.raw_content || '', __html: getRenderedContent(),
}} }}
/> />
)}
</div> </div>
</div> </div>
</div> </div>
+103 -2
View File
@@ -560,8 +560,8 @@ html, body, #root {
.reader-toolbar { .reader-toolbar {
position: sticky; position: sticky;
top: 0; top: 0;
height: 44px; height: 46px;
min-height: 44px; min-height: 46px;
background: rgba(255, 255, 255, 0.95); background: rgba(255, 255, 255, 0.95);
backdrop-filter: blur(8px); backdrop-filter: blur(8px);
border-bottom: 1px solid var(--border); border-bottom: 1px solid var(--border);
@@ -572,6 +572,107 @@ html, body, #root {
z-index: 10; z-index: 10;
} }
/* Sliding Ball Capsule Toggle Switch */
.capsule-switch-btn {
display: inline-flex;
align-items: center;
border: none;
background: transparent;
padding: 0;
cursor: pointer;
outline: none;
user-select: none;
margin-left: 0.5rem;
}
.capsule-switch-track {
position: relative;
display: flex;
align-items: center;
width: 86px;
height: 26px;
border-radius: 9999px;
background-color: var(--bg-card-hover);
border: 1px solid var(--border-dark);
padding: 2px;
transition: background-color 0.25s cubic-bezier(0.4, 0, 0.2, 1),
border-color 0.25s cubic-bezier(0.4, 0, 0.2, 1),
box-shadow 0.25s ease;
box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.05);
}
.capsule-switch-btn:hover .capsule-switch-track {
border-color: var(--border-focus);
}
.capsule-switch-btn.active .capsule-switch-track {
background-color: #0284c7;
border-color: #0284c7;
}
.capsule-switch-btn.scraping .capsule-switch-track {
background-color: #0369a1;
border-color: #0369a1;
cursor: wait;
}
.capsule-switch-thumb {
position: absolute;
top: 2px;
left: 2px;
width: 20px;
height: 20px;
border-radius: 50%;
background-color: #ffffff;
color: #64748b;
display: flex;
align-items: center;
justify-content: center;
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.1);
transition: transform 0.25s cubic-bezier(0.4, 0, 0.2, 1),
color 0.25s ease;
z-index: 2;
}
.capsule-switch-btn.active .capsule-switch-thumb {
transform: translateX(60px);
color: #0284c7;
}
.capsule-switch-label {
width: 100%;
font-size: 0.69rem;
font-weight: 600;
text-align: center;
transition: color 0.25s ease, padding 0.25s ease;
z-index: 1;
white-space: nowrap;
letter-spacing: -0.01em;
}
/* When thumb is on the left, label is padded to the right */
.capsule-switch-btn:not(.active) .capsule-switch-label {
padding-left: 17px;
padding-right: 2px;
color: var(--text-main);
}
/* When thumb is on the right, label is padded to the left */
.capsule-switch-btn.active .capsule-switch-label {
padding-right: 17px;
padding-left: 2px;
color: #ffffff;
}
@keyframes spin {
from { transform: rotate(0deg); }
to { transform: rotate(360deg); }
}
.spin {
animation: spin 1s linear infinite;
}
.reader-content-canvas { .reader-content-canvas {
max-width: 680px; max-width: 680px;
width: 100%; width: 100%;
+40
View File
@@ -172,6 +172,45 @@ export const api = {
}); });
}, },
async getEntry(id: string): Promise<FeedEntry> {
return request<FeedEntry>(`/entries/${id}`);
},
async getVariants(id: string): Promise<Record<string, any>> {
return request<Record<string, any>>(`/entries/${id}/variants`);
},
async scrapeEntry(id: string): Promise<FeedEntry> {
return request<FeedEntry>(`/entries/${id}/scrape`, {
method: 'POST',
});
},
// Rules
async listRules(): Promise<any[]> {
return request<any[]>('/rules');
},
async createRule(rule: any): Promise<any> {
return request<any>('/rules', {
method: 'POST',
body: JSON.stringify(rule),
});
},
async updateRule(id: string, rule: any): Promise<any> {
return request<any>(`/rules/${id}`, {
method: 'PUT',
body: JSON.stringify(rule),
});
},
async deleteRule(id: string): Promise<void> {
return request<void>(`/rules/${id}`, {
method: 'DELETE',
});
},
// OPML // OPML
async importOPML(file: File): Promise<{ imported_count: number; total_parsed: number }> { async importOPML(file: File): Promise<{ imported_count: number; total_parsed: number }> {
const token = localStorage.getItem('rsspipe_jwt'); const token = localStorage.getItem('rsspipe_jwt');
@@ -200,3 +239,4 @@ export const api = {
return `${API_BASE}/opml/export?token=${token || ''}`; return `${API_BASE}/opml/export?token=${token || ''}`;
} }
}; };
+40 -2
View File
@@ -32,6 +32,18 @@ export interface Feed {
created_at: string; created_at: string;
} }
export interface EntryVariant {
entry_id: string;
variant_key: string;
content_type: string; // 'text', 'html', 'json', 'yaml', 'audio/mp3', 'binary'
text_value?: string;
blob_path?: string;
stream_url?: string;
metadata?: Record<string, any>;
created_at: string;
updated_at: string;
}
export interface FeedEntry { export interface FeedEntry {
id: string; id: string;
feed_id: string; feed_id: string;
@@ -41,21 +53,46 @@ export interface FeedEntry {
url: string; url: string;
title: string; title: string;
author?: string; author?: string;
content: string; content: string; // Immutable original RSS content
raw_content?: string; raw_content?: string;
published_at: string; published_at: string;
is_read: boolean; is_read: boolean;
is_starred: boolean; is_starred: boolean;
share_uuid?: string; share_uuid?: string;
metadata: Record<string, any>; variants?: Record<string, EntryVariant>;
metadata?: Record<string, any>;
created_at: string; created_at: string;
} }
export interface RuleCondition {
field: string;
operator: string;
value: any;
}
export interface RuleAction {
type: string;
params?: Record<string, any>;
}
export interface Rule {
id: string;
name: string;
trigger: string;
conditions: RuleCondition[];
match_mode: string;
actions: RuleAction[];
is_active: boolean;
created_at: string;
updated_at: string;
}
export interface Stats { export interface Stats {
total_feeds: number; total_feeds: number;
total_unread: number; total_unread: number;
total_starred: number; total_starred: number;
total_categories: number; total_categories: number;
total_rules?: number;
} }
export type ViewType = 'inbox' | 'starred' | 'all'; export type ViewType = 'inbox' | 'starred' | 'all';
@@ -67,3 +104,4 @@ export interface FilterState {
feedId?: string; feedId?: string;
search: string; search: string;
} }