diff --git a/.gitleaks.toml b/.gitleaks.toml index bf6ede6..2e15f34 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -8,9 +8,12 @@ description = "Global allowlist" paths = [ '''sample_subscriptions\.opml''', '''docs/''', - '''web/node_modules/''' + '''web/node_modules/''', + '''web/dist/''', + '''data/''' ] regexes = [ '''admin123456''', '''example_jwt_secret_key''' ] + diff --git a/cmd/server/main.go b/cmd/server/main.go index 955399c..146de06 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -2,52 +2,69 @@ package main import ( "context" - "log" "net/http" "os" "os/signal" "syscall" "time" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/actions/readability" "git.pengzhan.dev/haopengzhan/rsspipe/internal/api" "git.pengzhan.dev/haopengzhan/rsspipe/internal/auth" "git.pengzhan.dev/haopengzhan/rsspipe/internal/config" "git.pengzhan.dev/haopengzhan/rsspipe/internal/crawler" "git.pengzhan.dev/haopengzhan/rsspipe/internal/db" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/events" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/logger" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/rules" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/worker" + "go.uber.org/zap" ) func main() { - log.Println("==================================================") - log.Println(" Starting RSSPipe Server ") - log.Println("==================================================") + log := logger.InitLogger(os.Getenv("LOG_LEVEL")) + log.Info("Starting RSSPipe Server") cfg := config.LoadConfig() // 1. Initialize SQLite Database database, err := db.InitDB(cfg.DBPath) if err != nil { - log.Fatalf("Fatal: Database initialization failed: %v", err) + log.Fatal("Database initialization failed", zap.Error(err)) } defer func() { _ = database.Close() }() - log.Printf("==> Database connected: %s (WAL mode enabled)\n", cfg.DBPath) + log.Info("Database connected", zap.String("path", cfg.DBPath)) // 2. Initialize Auth Service & Bootstrap Admin authService := auth.NewAuthService(database, cfg) if err := authService.BootstrapAdmin(); err != nil { - log.Fatalf("Fatal: Admin bootstrap failed: %v", err) + log.Fatal("Admin bootstrap failed", zap.Error(err)) } - // 3. Start Background Crawler Pool ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cr := crawler.NewCrawler(database, cfg.CrawlerIntervalMinutes, cfg.CrawlerWorkers) + // 3. Initialize EventBus, WorkerPool, and Action Handlers + bus := events.NewEventBus() + pool := worker.NewPool(database, bus, 5, 100) + + // Register Scraper Action + scraper := readability.NewScraper(15 * time.Second) + pool.RegisterHandler("readability_scrape", scraper.Handler()) + pool.Start(ctx) + + // 4. Initialize Rules Engine + engine := rules.NewEngine(database, bus, pool) + engine.Start(ctx) + + // 5. Start Background Crawler Pool + cr := crawler.NewCrawler(database, bus, cfg.CrawlerIntervalMinutes, cfg.CrawlerWorkers) cr.Start(ctx) - // 4. Mount API Routes - handler := api.NewHandler(database, authService, cr) + // 6. Mount API Routes + handler := api.NewHandler(database, authService, cr, pool) router := api.NewRouter(handler, authService, "./web/dist") server := &http.Server{ @@ -58,26 +75,27 @@ func main() { IdleTimeout: 120 * time.Second, } - // 5. Graceful shutdown handler + // 7. Graceful shutdown handler go func() { sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) <-sigChan - log.Println("\n==> Shutting down RSSPipe gracefully...") + log.Info("Shutting down RSSPipe gracefully...") cancel() + pool.Stop() shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) defer shutdownCancel() if err := server.Shutdown(shutdownCtx); err != nil { - log.Printf("Error during server shutdown: %v\n", err) + log.Error("Error during server shutdown", zap.Error(err)) } }() - log.Printf("==> RSSPipe HTTP Server listening at http://localhost:%s\n", cfg.Port) + log.Info("RSSPipe HTTP Server listening", zap.String("url", "http://localhost:"+cfg.Port)) if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { - log.Fatalf("Fatal: Server error: %v", err) + log.Fatal("Server error", zap.Error(err)) } - log.Println("==> RSSPipe Server stopped.") + log.Info("RSSPipe Server stopped") } diff --git a/docs/003_backlog.md b/docs/003_backlog.md index 9b1b44b..5a36115 100644 --- a/docs/003_backlog.md +++ b/docs/003_backlog.md @@ -93,3 +93,42 @@ This document records planned features, UX enhancements, and architectural exten - Staggered dynamic aspect ratios with prominent media cards. - Smooth infinite scroll pagination. - Interactive gesture swipe to dismiss the full-screen reader. + +--- + +## 10. Beautiful 404 Error Page + +- **Goal**: Provide a polished, modern, brand-consistent 404 Not Found experience. +- **Scope**: + - Frontend 404 route component with sleek modern aesthetics (glassmorphism/minimal illustration, clear error description, and a "Return to Gateway" action button). + - Server-side fallback handler returning a clean, branded HTML response for unmatched routes. + +--- + +## 11. Multi-Variant Pipeline Enrichments (AI Summary, TTS, Keywords, Audio) + +- **Goal**: Expand the event-driven rules engine and variant storage system with automated enrichment plugins. +- **Scope**: + - **AI Summary Hook**: Integrate LLM actions (e.g. OpenAI / Anthropic / Gemini / Ollama) to automatically generate and store concise summaries in `entry_variants(key='ai_summary')`. + - **Keyword & Tag Extraction**: Automated extractor storing structured YAML/JSON keyword lists in `entry_variants(key='keywords')` and `entry_variants(key='tags')` to enable rule-based chaining. + - **Text-to-Speech (TTS)**: Synthesize article text to voice audio blobs (`.mp3` stored under `./data/blobs/{entry_id}/tts.mp3`) with HTTP 206 partial streaming. + - **Podcast TTS & Summary TTS**: Convert articles into dual-host conversational podcasts or audio-narrated summaries streamed directly within the reader. + +--- + +## 12. Aggregated Shared Posts Dashboard (`/shares`) + +- **Goal**: Provide a dedicated gallery view aggregating all publicly shared articles. +- **Scope**: + - Centralized dashboard listing all articles with an active `share_uuid`. + - Copy public link, preview shared HTML canvas, revoke share link, and view access stats. + +--- + +## 13. Data Retention & Cleanup Policies (Auto-Deletion) + +- **Goal**: Provide configurable retention rules to prune older or noisy articles and keep SQLite storage compact. +- **Scope**: + - Retention triggers (e.g. delete unstarred articles older than 90 days, or limit feeds to latest 500 posts). + - Background garbage collection cleaning associated `entry_variants` and on-disk audio blobs (`./data/blobs/{entry_id}/`). + diff --git a/docs/004_variant_pipeline_architecture.md b/docs/004_variant_pipeline_architecture.md new file mode 100644 index 0000000..21ddeeb --- /dev/null +++ b/docs/004_variant_pipeline_architecture.md @@ -0,0 +1,116 @@ +# RSSPipe Variant Data & Extensible Processing Pipeline Architecture + +## 1. Overview & Vision + +RSSPipe is designed as an **RSS-Driven Information Gateway**. Subscriptions come in many formats (full-text blogs, brief news tickers, media podcasts, academic papers). Consequently, a one-size-fits-all processing approach fails: + +- Some feeds require automatic full-text extraction via Readability. +- Some feeds require LLM keyword and topic tagging. +- Specific high-value posts (filtered by keywords or feeds) require AI summarization, translation, or Text-to-Speech (TTS) audio synthesis. + +To support this cleanly without database corruption, performance degradation, or rigid schemas, RSSPipe adopts a **Variant Data Architecture** paired with a **Reactive Chained Trigger Pipeline**. + +--- + +## 2. Core Data Model + +### 2.1 Separation of Immutable Source vs. Enriched Variants + +1. **`feed_entries` (Immutable Source of Truth)**: + - Stores the exact metadata and content parsed from the RSS XML feed. + - Core fields: `id`, `feed_id`, `guid`, `url`, `title`, `author`, `rss_content`, `published_at`, `is_read`, `is_starred`, `created_at`. + - **Rule**: `rss_content` is NEVER overwritten by scrapers, LLMs, or TTS engines. +2. **`entry_variants` (Extensible Key-Value Store)**: + - Stores all downstream enrichments as strongly typed variant entries. + - Schema: + + ```sql + CREATE TABLE IF NOT EXISTS entry_variants ( + entry_id TEXT NOT NULL, + variant_key TEXT NOT NULL, + content_type TEXT NOT NULL, -- 'text', 'html', 'json', 'yaml', 'audio/mp3', 'binary' + text_value TEXT, -- UTF-8 text, HTML, JSON array/object, YAML string + blob_path TEXT, -- Relative filesystem path for large media (e.g. data/blobs/{entry_id}/{key}.mp3) + metadata JSON DEFAULT '{}', -- Stats: model, tokens, duration_ms, byte_size, executed_at + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (entry_id, variant_key), + FOREIGN KEY (entry_id) REFERENCES feed_entries(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_entry_variants_key ON entry_variants(variant_key); + ``` + +--- + +## 3. Supported Variant Keys & Types + +| Variant Key | Content Type | Storage Location | Description | +| :--- | :--- | :--- | :--- | +| `readability_full_text` | `html` | `text_value` | Sanitized readability full text extracted from the source web page. | +| `keywords` | `json` / `yaml` | `text_value` | List of extracted keywords (e.g., `["AI", "Go", "Compiler"]`). | +| `tags` | `json` | `text_value` | Category and topic tags. | +| `ai_summary` | `text` / `markdown` | `text_value` | Bulleted or narrative AI executive summary. | +| `translation_{lang}` | `html` / `text` | `text_value` | Translated content (e.g. `translation_zh`). | +| `tts` | `audio/mp3` | `blob_path` | Synthesized speech audio file of the article. | +| `podcast_tts` | `audio/mp3` | `blob_path` | Multi-speaker podcast discussion audio file. | +| `ai_summary_tts` | `audio/mp3` | `blob_path` | Audio narration of the AI summary. | + +--- + +## 4. Large Binary Blob Strategy + +Storing multi-megabyte audio files or binary blobs directly inside SQLite columns degrades query cache efficiency and slows down table scans. + +- **Text & JSON Variants** are stored inline in `entry_variants.text_value`. +- **Audio & Media Blobs** are stored on disk under `./data/blobs/{entry_id}/{variant_key}.{ext}`. +- `entry_variants.blob_path` stores the relative path. +- The API provides a streaming endpoint with HTTP Range support: `GET /api/entries/{id}/variants/{key}/stream`. + +--- + +## 5. Reactive Chained Trigger & Watch Pipeline + +The pipeline operates on typed pub/sub events with depth guards to prevent infinite loops: + +```mermaid +graph TD + A[Feed Ingestion / Crawler] -->|Event: on_post_created| Bus[Typed Event Bus] + Bus --> Rules[Rules Engine Evaluator] + + Rules -->|Matches Rule 1: Scrape Tech| W1[Worker: Readability Scraper] + W1 -->|Writes variant: readability_full_text| DB[(entry_variants)] + W1 -->|Event: on_variant_updated| Bus + + Rules -->|Matches Rule 2: Tag AI Keywords| W2[Worker: Keyword Extractor] + W2 -->|Writes variant: keywords| DB + W2 -->|Event: on_variant_updated| Bus + + Bus -->|Matches Rule 3: Watch keywords contains 'AI'| W3[Worker: AI Summary & TTS] + W3 -->|Writes variant: ai_summary & tts blob| DB +``` + +### 5.1 Concrete Chained Pipeline Example + +1. **Trigger 1**: New post ingested from Hacker News -> Emits `on_post_created`. +2. **Rule 1 (Auto-Enrich Keywords)**: + - Condition: `feed_id == "hn-feed-id"` + - Action: `extract_keywords` -> Saves `entry_variants(key='keywords', text_value='["OpenAI", "LLM", "Agent"]')`. + - Emits: `on_variant_updated(key='keywords')`. +3. **Rule 2 (Watch & Synthesize Audio)**: + - Condition: `variant.keywords CONTAINS "OpenAI"` + - Action: `generate_tts` -> Synthesizes MP3, saves to `./data/blobs/...`, saves variant `tts`. + - Emits: `on_variant_updated(key='tts')`. + +--- + +## 6. Reader UX & Content Switcher + +In the 3-pane reader interface: + +- **Default View**: Renders `rss_content` (original source). +- **Enrichment Pills**: If variants exist on the entry, dynamic tabs/pills appear in the reader header: + - `[📄 Original RSS]` + - `[⚡ Readability Full-Text]` (if `readability_full_text` exists) + - `[💡 AI Summary]` (if `ai_summary` exists) + - `[🎧 Listen Audio]` (if `tts` or `podcast_tts` exists, plays with embedded HTML5 player) +- Switching variants never mutates the underlying article or overwrites database fields. diff --git a/go.mod b/go.mod index 9f94817..d72addc 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,26 @@ module git.pengzhan.dev/haopengzhan/rsspipe go 1.26.5 require ( + github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-chi/chi/v5 v5.3.2 // indirect github.com/go-chi/cors v1.2.2 // indirect + github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect + github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0 // indirect + github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/css v1.0.1 // indirect github.com/mattn/go-isatty v0.0.24 // indirect + github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/mmcdole/gofeed v1.4.2 // indirect github.com/mmcdole/goxpp/v2 v2.0.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + go.uber.org/multierr v1.10.0 // indirect + go.uber.org/zap v1.28.0 // indirect golang.org/x/crypto v0.55.0 // indirect golang.org/x/net v0.57.0 // indirect golang.org/x/sys v0.47.0 // indirect diff --git a/go.sum b/go.sum index d320bbd..6bc1693 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,125 @@ +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY= github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE= github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w= +github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM= +github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0 h1:A3B75Yp163FAIf9nLlFMl4pwIj+T3uKxfI7mbvvY2Ls= +github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0/go.mod h1:suxK0Wpz4BM3/2+z1mnOVTIWHDiMCIOGoKDCRumSsk0= +github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= +github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/mmcdole/gofeed v1.4.2 h1:XFFOtsNNZg+zudjtMXb8BI0J/YdnSIGfjEpPgPnZib0= github.com/mmcdole/gofeed v1.4.2/go.mod h1:X5x1PyeibJi152VEya0AsV+PW4daYmCD4LJaJbeFkcs= github.com/mmcdole/goxpp/v2 v2.0.0 h1:HrSCflxerUEqZQNq3u7ldtmE/XkwnTx4Zpq2DW4i5rQ= github.com/mmcdole/goxpp/v2 v2.0.0/go.mod h1:CUduYMnO9JB6Z/uqDn9Ormk/r8E9BsLQxHPWDZ961Os= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= diff --git a/internal/actions/readability/readability.go b/internal/actions/readability/readability.go new file mode 100644 index 0000000..97e7f59 --- /dev/null +++ b/internal/actions/readability/readability.go @@ -0,0 +1,123 @@ +package readability + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "git.pengzhan.dev/haopengzhan/rsspipe/internal/logger" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/worker" + readability "github.com/go-shiori/go-readability" + "github.com/microcosm-cc/bluemonday" + "go.uber.org/zap" +) + +type Scraper struct { + client *http.Client + sanitizer *bluemonday.Policy + log *zap.Logger +} + +func NewScraper(timeout time.Duration) *Scraper { + if timeout <= 0 { + timeout = 15 * time.Second + } + + p := bluemonday.UGCPolicy() + // Allow rich media & responsive elements + p.AllowElements("video", "audio", "source", "track", "iframe", "picture", "figure", "figcaption", "article", "section") + p.AllowAttrs("src", "poster", "controls", "autoplay", "loop", "muted", "preload").OnElements("video", "audio") + p.AllowAttrs("src", "type", "srcset", "media").OnElements("source") + p.AllowAttrs("src", "width", "height", "frameborder", "allow", "allowfullscreen", "sandbox").OnElements("iframe") + p.AllowAttrs("data-src", "data-original", "loading", "srcset", "sizes").OnElements("img") + p.AllowAttrs("class", "id").Globally() + + return &Scraper{ + client: &http.Client{ + Timeout: timeout, + }, + sanitizer: p, + log: logger.Named("readability-scraper"), + } +} + +// Handler returns the worker.ActionHandler function for readability scraping. +func (s *Scraper) Handler() worker.ActionHandler { + return func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*worker.ActionResult, error) { + if strings.TrimSpace(entry.URL) == "" { + return nil, fmt.Errorf("entry URL is empty") + } + + parsedURL, err := url.Parse(entry.URL) + if err != nil { + return nil, fmt.Errorf("invalid URL: %w", err) + } + + s.log.Info("Starting readability scrape", + zap.String("entry_id", entry.ID), + zap.String("url", entry.URL), + ) + + req, err := http.NewRequestWithContext(ctx, "GET", entry.URL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7") + + resp, err := s.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch article page: %w", err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("server returned HTTP status %d", resp.StatusCode) + } + + article, err := readability.FromReader(resp.Body, parsedURL) + if err != nil { + return nil, fmt.Errorf("failed to parse readability article: %w", err) + } + + cleanHTML := s.sanitizer.Sanitize(article.Content) + if strings.TrimSpace(cleanHTML) == "" && strings.TrimSpace(article.TextContent) != "" { + cleanHTML = "
" + s.sanitizer.Sanitize(article.TextContent) + "
" + } + + s.log.Info("Readability scrape successful", + zap.String("entry_id", entry.ID), + zap.String("title", article.Title), + zap.Int("content_length", len(cleanHTML)), + ) + + now := time.Now().UTC() + variant := models.EntryVariant{ + EntryID: entry.ID, + VariantKey: "readability_full_text", + ContentType: "html", + TextValue: cleanHTML, + Metadata: map[string]interface{}{ + "title": article.Title, + "byline": article.Byline, + "excerpt": article.Excerpt, + "length": len(article.TextContent), + "scraped_at": now.Format(time.RFC3339), + }, + CreatedAt: now, + UpdatedAt: now, + } + + return &worker.ActionResult{ + ResultVariant: &variant, + }, nil + } +} diff --git a/internal/api/handlers.go b/internal/api/handlers.go index a2d69f0..e6aa3df 100644 --- a/internal/api/handlers.go +++ b/internal/api/handlers.go @@ -17,6 +17,7 @@ import ( "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" "git.pengzhan.dev/haopengzhan/rsspipe/internal/opml" "git.pengzhan.dev/haopengzhan/rsspipe/internal/parser" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/worker" "github.com/go-chi/chi/v5" ) @@ -25,14 +26,16 @@ type Handler struct { auth *auth.AuthService crawler *crawler.Crawler parser *parser.Parser + pool *worker.Pool } -func NewHandler(database *db.DB, authService *auth.AuthService, cr *crawler.Crawler) *Handler { +func NewHandler(database *db.DB, authService *auth.AuthService, cr *crawler.Crawler, pool *worker.Pool) *Handler { return &Handler{ db: database, auth: authService, crawler: cr, parser: parser.NewParser(), + pool: pool, } } @@ -547,3 +550,160 @@ func (h *Handler) ViewSharedEntry(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html; charset=utf-8") _, _ = w.Write([]byte(html)) } + +// GetEntry returns a single entry by ID with its loaded variants +func (h *Handler) GetEntry(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + entry, err := h.db.GetEntry(id) + if err != nil { + if err == sql.ErrNoRows { + h.writeError(w, http.StatusNotFound, "entry not found") + return + } + h.writeError(w, http.StatusInternalServerError, err.Error()) + return + } + h.writeJSON(w, http.StatusOK, entry) +} + +// ListEntryVariants returns all variants for a given entry ID +func (h *Handler) ListEntryVariants(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + variants, err := h.db.ListVariantsForEntry(id) + if err != nil { + h.writeError(w, http.StatusInternalServerError, err.Error()) + return + } + h.writeJSON(w, http.StatusOK, variants) +} + +// GetVariantBlobStream streams the audio/binary file for a variant with HTTP 206 Partial Content support +func (h *Handler) GetVariantBlobStream(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + key := chi.URLParam(r, "key") + variant, err := h.db.GetVariant(id, key) + if err != nil { + h.writeError(w, http.StatusNotFound, "variant not found") + return + } + if variant.BlobPath == "" { + h.writeError(w, http.StatusNotFound, "variant has no binary media blob") + return + } + + http.ServeFile(w, r, variant.BlobPath) +} + +// ScrapeEntry manually triggers readability full text scraping and saves to entry_variants +func (h *Handler) ScrapeEntry(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + entry, err := h.db.GetEntry(id) + if err != nil { + h.writeError(w, http.StatusNotFound, "entry not found") + return + } + + if h.pool == nil { + h.writeError(w, http.StatusInternalServerError, "worker pool not configured") + return + } + + res, err := h.pool.ExecuteDirect(r.Context(), "readability_scrape", *entry, nil) + if err != nil { + h.writeError(w, http.StatusInternalServerError, fmt.Sprintf("scraping failed: %v", err)) + return + } + + if res != nil && res.ResultVariant != nil { + _ = h.db.SetVariant(*res.ResultVariant) + } + + // Reload entry with newly saved variant + updatedEntry, err := h.db.GetEntry(id) + if err != nil { + h.writeError(w, http.StatusInternalServerError, err.Error()) + return + } + + h.writeJSON(w, http.StatusOK, updatedEntry) +} + +// Rules API Handlers +func (h *Handler) ListRules(w http.ResponseWriter, r *http.Request) { + rules, err := h.db.ListRules() + if err != nil { + h.writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if rules == nil { + rules = []models.Rule{} + } + h.writeJSON(w, http.StatusOK, rules) +} + +type CreateRuleRequest struct { + Name string `json:"name"` + Trigger string `json:"trigger"` + MatchMode string `json:"match_mode"` + Conditions []models.RuleCondition `json:"conditions"` + Actions []models.RuleAction `json:"actions"` +} + +func (h *Handler) CreateRule(w http.ResponseWriter, r *http.Request) { + var body CreateRuleRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + h.writeError(w, http.StatusBadRequest, "invalid request body") + return + } + if strings.TrimSpace(body.Name) == "" { + h.writeError(w, http.StatusBadRequest, "rule name is required") + return + } + if strings.TrimSpace(body.Trigger) == "" { + body.Trigger = "on_post_created" + } + if strings.TrimSpace(body.MatchMode) == "" { + body.MatchMode = "all" + } + + rule, err := h.db.CreateRule(body.Name, body.Trigger, body.MatchMode, body.Conditions, body.Actions) + if err != nil { + h.writeError(w, http.StatusInternalServerError, err.Error()) + return + } + h.writeJSON(w, http.StatusCreated, rule) +} + +type UpdateRuleRequest struct { + Name string `json:"name"` + Trigger string `json:"trigger"` + MatchMode string `json:"match_mode"` + Conditions []models.RuleCondition `json:"conditions"` + Actions []models.RuleAction `json:"actions"` + IsActive bool `json:"is_active"` +} + +func (h *Handler) UpdateRule(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + var body UpdateRuleRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + h.writeError(w, http.StatusBadRequest, "invalid request body") + return + } + + rule, err := h.db.UpdateRule(id, body.Name, body.Trigger, body.MatchMode, body.Conditions, body.Actions, body.IsActive) + if err != nil { + h.writeError(w, http.StatusInternalServerError, err.Error()) + return + } + h.writeJSON(w, http.StatusOK, rule) +} + +func (h *Handler) DeleteRule(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + if err := h.db.DeleteRule(id); err != nil { + h.writeError(w, http.StatusInternalServerError, err.Error()) + return + } + h.writeJSON(w, http.StatusOK, map[string]string{"message": "rule deleted"}) +} diff --git a/internal/api/server.go b/internal/api/server.go index 5a75487..aa1aa56 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -62,9 +62,19 @@ func NewRouter(h *Handler, authService *auth.AuthService, staticDir string) http // Entries r.Get("/entries", h.ListEntries) + r.Get("/entries/{id}", h.GetEntry) r.Put("/entries/{id}/read", h.UpdateEntryRead) r.Put("/entries/{id}/star", h.UpdateEntryStar) r.Post("/entries/mark-all-read", h.MarkAllRead) + r.Post("/entries/{id}/scrape", h.ScrapeEntry) + r.Get("/entries/{id}/variants", h.ListEntryVariants) + r.Get("/entries/{id}/variants/{key}/stream", h.GetVariantBlobStream) + + // Rules + r.Get("/rules", h.ListRules) + r.Post("/rules", h.CreateRule) + r.Put("/rules/{id}", h.UpdateRule) + r.Delete("/rules/{id}", h.DeleteRule) // OPML r.Post("/opml/import", h.ImportOPML) diff --git a/internal/crawler/crawler.go b/internal/crawler/crawler.go index f1ebd64..b750672 100644 --- a/internal/crawler/crawler.go +++ b/internal/crawler/crawler.go @@ -3,35 +3,41 @@ package crawler import ( "context" "fmt" - "log" "sync" "time" "git.pengzhan.dev/haopengzhan/rsspipe/internal/db" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/events" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/logger" "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" "git.pengzhan.dev/haopengzhan/rsspipe/internal/parser" + "go.uber.org/zap" ) type Crawler struct { db *db.DB parser *parser.Parser + bus *events.EventBus intervalMinutes int workerCount int isRefreshing bool mu sync.Mutex + log *zap.Logger } -func NewCrawler(database *db.DB, intervalMinutes, workers int) *Crawler { +func NewCrawler(database *db.DB, bus *events.EventBus, intervalMinutes, workers int) *Crawler { return &Crawler{ db: database, parser: parser.NewParser(), + bus: bus, intervalMinutes: intervalMinutes, workerCount: workers, + log: logger.Named("crawler"), } } func (c *Crawler) Start(ctx context.Context) { - log.Printf("==> Starting RSS background crawler scheduler (Workers: %d)\n", c.workerCount) + c.log.Info("Starting RSS background crawler scheduler", zap.Int("worker_count", c.workerCount)) // Run initial fetch asynchronously after 2 seconds startup go func() { @@ -72,7 +78,7 @@ func (c *Crawler) RefreshDue(ctx context.Context) { feeds, err := c.db.ListFeeds() if err != nil { - log.Printf("Crawler error listing feeds: %v\n", err) + c.log.Error("Crawler error listing feeds", zap.Error(err)) return } @@ -93,7 +99,7 @@ func (c *Crawler) RefreshDue(ctx context.Context) { return } - log.Printf("Crawler: Refreshing %d due feeds (out of %d total)...\n", len(dueFeeds), len(feeds)) + c.log.Info("Crawler refreshing due feeds", zap.Int("due_count", len(dueFeeds)), zap.Int("total_count", len(feeds))) c.runWorkerPool(ctx, dueFeeds) } @@ -102,7 +108,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) { c.mu.Lock() if c.isRefreshing { c.mu.Unlock() - log.Println("Crawler: RefreshAll already in progress, skipping") + c.log.Debug("Crawler RefreshAll already in progress, skipping") return } c.isRefreshing = true @@ -116,7 +122,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) { feeds, err := c.db.ListFeeds() if err != nil { - log.Printf("Crawler error listing feeds: %v\n", err) + c.log.Error("Crawler error listing feeds", zap.Error(err)) return } @@ -124,7 +130,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) { return } - log.Printf("Crawler: Refreshing all %d feeds...\n", len(feeds)) + c.log.Info("Crawler refreshing all feeds", zap.Int("count", len(feeds))) c.runWorkerPool(ctx, feeds) } @@ -148,7 +154,7 @@ func (c *Crawler) runWorkerPool(ctx context.Context, feeds []models.Feed) { close(jobs) wg.Wait() - log.Printf("Crawler: Completed refresh of %d feeds.\n", len(feeds)) + c.log.Info("Crawler completed refresh batch", zap.Int("feed_count", len(feeds))) } func (c *Crawler) RefreshSingle(ctx context.Context, feedID string) error { @@ -169,7 +175,11 @@ func (c *Crawler) fetchSingleFeed(ctx context.Context, f models.Feed) error { res, err := c.parser.FetchFeed(feedCtx, f.FeedURL, f.ETag, f.LastModified) if err != nil { - log.Printf("Crawler error fetching [%s]: %v\n", f.Title, err) + c.log.Warn("Crawler error fetching feed", + zap.String("feed_title", f.Title), + zap.String("feed_url", f.FeedURL), + zap.Error(err), + ) _ = c.db.UpdateFeedFetchStatus(f.ID, f.ETag, f.LastModified, err.Error()) return err } @@ -180,11 +190,30 @@ func (c *Crawler) fetchSingleFeed(ctx context.Context, f models.Feed) error { } // Upsert entries - inserted, err := c.db.UpsertFeedEntries(f.ID, res.Entries) + newEntries, err := c.db.UpsertFeedEntries(f.ID, res.Entries) if err != nil { - log.Printf("Crawler error upserting entries for [%s]: %v\n", f.Title, err) - } else if inserted > 0 { - log.Printf("Crawler [%s]: inserted %d new entries\n", f.Title, inserted) + c.log.Error("Crawler error upserting entries", + zap.String("feed_title", f.Title), + zap.Error(err), + ) + } else if len(newEntries) > 0 { + c.log.Info("Crawler inserted new entries", + zap.String("feed_title", f.Title), + zap.Int("new_entries", len(newEntries)), + ) + + // Publish on_post_created events for newly inserted entries + if c.bus != nil { + for _, entry := range newEntries { + entry.FeedTitle = f.Title + entry.FeedSiteURL = f.SiteURL + c.bus.Publish(ctx, events.Event{ + Type: events.EventPostCreated, + Entry: entry, + Depth: 0, + }) + } + } } _ = c.db.UpdateFeedFetchStatus(f.ID, res.ETag, res.LastModified, "") diff --git a/internal/db/db.go b/internal/db/db.go index 933bb0a..c5acb2d 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -114,15 +114,28 @@ func (d *DB) migrate() error { CREATE INDEX IF NOT EXISTS idx_entries_is_starred ON feed_entries(is_starred); CREATE INDEX IF NOT EXISTS idx_entries_share_uuid ON feed_entries(share_uuid); + CREATE TABLE IF NOT EXISTS entry_variants ( + entry_id TEXT NOT NULL, + variant_key TEXT NOT NULL, + content_type TEXT NOT NULL, + text_value TEXT, + blob_path TEXT, + metadata TEXT DEFAULT '{}', + created_at DATETIME NOT NULL, + updated_at DATETIME NOT NULL, + PRIMARY KEY (entry_id, variant_key), + FOREIGN KEY(entry_id) REFERENCES feed_entries(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_entry_variants_key ON entry_variants(variant_key); + CREATE TABLE IF NOT EXISTS rules ( id TEXT PRIMARY KEY, name TEXT NOT NULL, - rule_type TEXT NOT NULL, - is_active BOOLEAN DEFAULT 1, - trigger_type TEXT NOT NULL, - trigger_config TEXT DEFAULT '{}', + trigger TEXT NOT NULL, conditions TEXT DEFAULT '[]', + match_mode TEXT DEFAULT 'all', actions TEXT DEFAULT '[]', + is_active BOOLEAN DEFAULT 1, created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL ); @@ -132,6 +145,26 @@ func (d *DB) migrate() error { } // Migration for timeout_seconds column if older DB exists _, _ = d.Exec("ALTER TABLE feeds ADD COLUMN timeout_seconds INTEGER DEFAULT 10") + + // Migration for rules table schema if older table exists + var testCol string + err := d.QueryRow("SELECT trigger FROM rules LIMIT 1").Scan(&testCol) + if err != nil && err != sql.ErrNoRows { + _, _ = d.Exec("DROP TABLE IF EXISTS rules") + _, _ = d.Exec(` + CREATE TABLE IF NOT EXISTS rules ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + trigger TEXT NOT NULL, + conditions TEXT DEFAULT '[]', + match_mode TEXT DEFAULT 'all', + actions TEXT DEFAULT '[]', + is_active BOOLEAN DEFAULT 1, + created_at DATETIME NOT NULL, + updated_at DATETIME NOT NULL + ); + `) + } return nil } @@ -474,51 +507,67 @@ func (d *DB) DeleteFeed(id string) error { } // Feed Entries operations -func (d *DB) UpsertFeedEntries(feedID string, entries []models.FeedEntry) (int, error) { +func (d *DB) UpsertFeedEntries(feedID string, entries []models.FeedEntry) ([]models.FeedEntry, error) { if len(entries) == 0 { - return 0, nil + return nil, nil } - stmt, err := d.Prepare(` + var newEntries []models.FeedEntry + now := time.Now().UTC() + + insertStmt, err := d.Prepare(` INSERT INTO feed_entries ( id, feed_id, guid, url, title, author, content, raw_content, published_at, is_read, is_starred, metadata, created_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?, ?) - ON CONFLICT(feed_id, guid) DO UPDATE SET - title = excluded.title, - content = CASE WHEN feed_entries.content = '' THEN excluded.content ELSE feed_entries.content END, - url = excluded.url, - author = excluded.author `) if err != nil { - return 0, err + return nil, err } defer func() { - _ = stmt.Close() + _ = insertStmt.Close() }() - insertedCount := 0 - for _, e := range entries { - id := uuid.New().String() - now := time.Now().UTC() - pubAt := e.PublishedAt - if pubAt.IsZero() { - pubAt = now - } - metaBytes, _ := json.Marshal(e.Metadata) - if len(metaBytes) == 0 { - metaBytes = []byte("{}") - } + updateStmt, err := d.Prepare(` + UPDATE feed_entries + SET title = ?, url = ?, author = ?, content = ?, raw_content = ? + WHERE feed_id = ? AND guid = ? + `) + if err != nil { + return nil, err + } + defer func() { + _ = updateStmt.Close() + }() - res, err := stmt.Exec( - id, feedID, e.GUID, e.URL, e.Title, e.Author, e.Content, e.RawContent, pubAt, string(metaBytes), now, - ) - if err == nil { - if n, _ := res.RowsAffected(); n > 0 { - insertedCount++ + for _, e := range entries { + var existingID string + err := d.QueryRow("SELECT id FROM feed_entries WHERE feed_id = ? AND guid = ?", feedID, e.GUID).Scan(&existingID) + switch err { + case sql.ErrNoRows: + e.ID = uuid.New().String() + e.FeedID = feedID + pubAt := e.PublishedAt + if pubAt.IsZero() { + pubAt = now } + metaBytes, _ := json.Marshal(e.Metadata) + if len(metaBytes) == 0 { + metaBytes = []byte("{}") + } + + _, err := insertStmt.Exec( + e.ID, feedID, e.GUID, e.URL, e.Title, e.Author, e.Content, e.RawContent, pubAt, string(metaBytes), now, + ) + if err == nil { + e.CreatedAt = now + newEntries = append(newEntries, e) + } + case nil: + _, _ = updateStmt.Exec(e.Title, e.URL, e.Author, e.Content, e.RawContent, feedID, e.GUID) } } - return insertedCount, nil + + return newEntries, nil } type EntryFilter struct { @@ -530,67 +579,98 @@ type EntryFilter struct { Offset int } -func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) { - where := []string{"1=1"} - args := []interface{}{} +func (d *DB) GetEntry(id string) (*models.FeedEntry, error) { + query := ` + SELECT + e.id, e.feed_id, f.title as feed_title, COALESCE(f.site_url, '') as feed_site_url, + e.guid, e.url, e.title, e.author, e.content, e.raw_content, + e.published_at, e.is_read, e.is_starred, e.share_uuid, e.metadata, e.created_at + FROM feed_entries e + JOIN feeds f ON f.id = e.feed_id + WHERE e.id = ? + ` + var e models.FeedEntry + var author, rawContent, shareUUID, metaStr sql.NullString + err := d.QueryRow(query, id).Scan( + &e.ID, &e.FeedID, &e.FeedTitle, &e.FeedSiteURL, + &e.GUID, &e.URL, &e.Title, &author, &e.Content, &rawContent, + &e.PublishedAt, &e.IsRead, &e.IsStarred, &shareUUID, &metaStr, &e.CreatedAt, + ) + if err != nil { + return nil, err + } + if author.Valid { + e.Author = author.String + } + if rawContent.Valid { + e.RawContent = rawContent.String + } + if shareUUID.Valid { + s := shareUUID.String + e.ShareUUID = &s + } + e.Metadata = make(map[string]interface{}) + if metaStr.Valid && metaStr.String != "" { + _ = json.Unmarshal([]byte(metaStr.String), &e.Metadata) + } - switch f.View { + // Load variants + variants, _ := d.ListVariantsForEntry(e.ID) + e.Variants = variants + + return &e, nil +} + +func (d *DB) ListEntries(filter EntryFilter) ([]models.FeedEntry, int, error) { + var whereClauses []string + var args []interface{} + + switch filter.View { case "inbox": - where = append(where, "e.is_read = 0") + whereClauses = append(whereClauses, "e.is_read = 0") case "starred": - where = append(where, "e.is_starred = 1") + whereClauses = append(whereClauses, "e.is_starred = 1") } - if f.FeedID != "" { - where = append(where, "e.feed_id = ?") - args = append(args, f.FeedID) + if filter.FeedID != "" { + whereClauses = append(whereClauses, "e.feed_id = ?") + args = append(args, filter.FeedID) } - if f.CategoryID != "" { - where = append(where, "f.category_id = ?") - args = append(args, f.CategoryID) + if filter.CategoryID != "" { + whereClauses = append(whereClauses, "f.category_id = ?") + args = append(args, filter.CategoryID) } - if f.Search != "" { - searchTerms := strings.Fields(f.Search) - for _, term := range searchTerms { - lower := strings.ToLower(term) - if strings.HasPrefix(lower, "is:unread") { - where = append(where, "e.is_read = 0") - } else if strings.HasPrefix(lower, "is:read") { - where = append(where, "e.is_read = 1") - } else if strings.HasPrefix(lower, "is:starred") { - where = append(where, "e.is_starred = 1") - } else if strings.HasPrefix(lower, "feed:") { - feedVal := strings.TrimPrefix(term, "feed:") - where = append(where, "(f.title LIKE ? OR f.feed_url LIKE ?)") - args = append(args, "%"+feedVal+"%", "%"+feedVal+"%") - } else { - where = append(where, "(e.title LIKE ? OR e.content LIKE ? OR e.author LIKE ?)") - args = append(args, "%"+term+"%", "%"+term+"%", "%"+term+"%") - } - } + if filter.Search != "" { + whereClauses = append(whereClauses, "(e.title LIKE ? OR e.content LIKE ? OR e.author LIKE ?)") + pattern := "%" + filter.Search + "%" + args = append(args, pattern, pattern, pattern) } - whereClause := strings.Join(where, " AND ") + whereSQL := "" + if len(whereClauses) > 0 { + whereSQL = "WHERE " + strings.Join(whereClauses, " AND ") + } - // Count total countQuery := fmt.Sprintf(` - SELECT COUNT(*) - FROM feed_entries e - JOIN feeds f ON f.id = e.feed_id - WHERE %s - `, whereClause) + SELECT COUNT(e.id) + FROM feed_entries e + JOIN feeds f ON f.id = e.feed_id + %s + `, whereSQL) + var total int - if err := d.QueryRow(countQuery, args...).Scan(&total); err != nil { + err := d.QueryRow(countQuery, args...).Scan(&total) + if err != nil { return nil, 0, err } - limit := f.Limit - if limit <= 0 || limit > 100 { - limit = 50 + limit := filter.Limit + if limit <= 0 || limit > 500 { + limit = 100 } - offset := f.Offset + offset := filter.Offset if offset < 0 { offset = 0 } @@ -602,10 +682,10 @@ func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) { e.published_at, e.is_read, e.is_starred, e.share_uuid, e.metadata, e.created_at FROM feed_entries e JOIN feeds f ON f.id = e.feed_id - WHERE %s - ORDER BY e.published_at DESC + %s + ORDER BY e.published_at DESC, e.created_at DESC LIMIT ? OFFSET ? - `, whereClause) + `, whereSQL) queryArgs := append(args, limit, offset) rows, err := d.Query(query, queryArgs...) @@ -619,13 +699,13 @@ func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) { var entries []models.FeedEntry for rows.Next() { var e models.FeedEntry - var author, rawContent, metaStr sql.NullString - var shareUUID sql.NullString - if err := rows.Scan( + var author, rawContent, shareUUID, metaStr sql.NullString + err := rows.Scan( &e.ID, &e.FeedID, &e.FeedTitle, &e.FeedSiteURL, &e.GUID, &e.URL, &e.Title, &author, &e.Content, &rawContent, &e.PublishedAt, &e.IsRead, &e.IsStarred, &shareUUID, &metaStr, &e.CreatedAt, - ); err != nil { + ) + if err != nil { return nil, 0, err } if author.Valid { @@ -675,6 +755,232 @@ func (d *DB) MarkAllRead(feedID, categoryID *string) error { return err } +// Entry Variants CRUD +func (d *DB) GetVariant(entryID, key string) (*models.EntryVariant, error) { + query := ` + SELECT entry_id, variant_key, content_type, COALESCE(text_value, ''), COALESCE(blob_path, ''), metadata, created_at, updated_at + FROM entry_variants + WHERE entry_id = ? AND variant_key = ? + ` + var v models.EntryVariant + var metaStr sql.NullString + err := d.QueryRow(query, entryID, key).Scan( + &v.EntryID, &v.VariantKey, &v.ContentType, &v.TextValue, &v.BlobPath, &metaStr, &v.CreatedAt, &v.UpdatedAt, + ) + if err != nil { + return nil, err + } + v.Metadata = make(map[string]interface{}) + if metaStr.Valid && metaStr.String != "" { + _ = json.Unmarshal([]byte(metaStr.String), &v.Metadata) + } + return &v, nil +} + +func (d *DB) ListVariantsForEntry(entryID string) (map[string]models.EntryVariant, error) { + query := ` + SELECT entry_id, variant_key, content_type, COALESCE(text_value, ''), COALESCE(blob_path, ''), metadata, created_at, updated_at + FROM entry_variants + WHERE entry_id = ? + ` + rows, err := d.Query(query, entryID) + if err != nil { + return nil, err + } + defer func() { + _ = rows.Close() + }() + + variants := make(map[string]models.EntryVariant) + for rows.Next() { + var v models.EntryVariant + var metaStr sql.NullString + err := rows.Scan( + &v.EntryID, &v.VariantKey, &v.ContentType, &v.TextValue, &v.BlobPath, &metaStr, &v.CreatedAt, &v.UpdatedAt, + ) + if err != nil { + return nil, err + } + v.Metadata = make(map[string]interface{}) + if metaStr.Valid && metaStr.String != "" { + _ = json.Unmarshal([]byte(metaStr.String), &v.Metadata) + } + variants[v.VariantKey] = v + } + return variants, nil +} + +func (d *DB) SetVariant(v models.EntryVariant) error { + now := time.Now().UTC() + metaBytes, _ := json.Marshal(v.Metadata) + if len(metaBytes) == 0 { + metaBytes = []byte("{}") + } + + query := ` + INSERT INTO entry_variants ( + entry_id, variant_key, content_type, text_value, blob_path, metadata, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(entry_id, variant_key) DO UPDATE SET + content_type = excluded.content_type, + text_value = excluded.text_value, + blob_path = excluded.blob_path, + metadata = excluded.metadata, + updated_at = excluded.updated_at + ` + _, err := d.Exec(query, v.EntryID, v.VariantKey, v.ContentType, v.TextValue, v.BlobPath, string(metaBytes), now, now) + return err +} + +func (d *DB) DeleteVariant(entryID, key string) error { + _, err := d.Exec("DELETE FROM entry_variants WHERE entry_id = ? AND variant_key = ?", entryID, key) + return err +} + +// Rules CRUD +func (d *DB) ListRules() ([]models.Rule, error) { + query := ` + SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at + FROM rules + ORDER BY created_at ASC + ` + rows, err := d.Query(query) + if err != nil { + return nil, err + } + defer func() { + _ = rows.Close() + }() + + var rules []models.Rule + for rows.Next() { + var r models.Rule + var condsStr, actionsStr string + err := rows.Scan( + &r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt, + ) + if err != nil { + return nil, err + } + _ = json.Unmarshal([]byte(condsStr), &r.Conditions) + _ = json.Unmarshal([]byte(actionsStr), &r.Actions) + rules = append(rules, r) + } + return rules, nil +} + +func (d *DB) GetActiveRules(trigger string) ([]models.Rule, error) { + query := ` + SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at + FROM rules + WHERE is_active = 1 AND (trigger = ? OR trigger = '*') + ORDER BY created_at ASC + ` + rows, err := d.Query(query, trigger) + if err != nil { + return nil, err + } + defer func() { + _ = rows.Close() + }() + + var rules []models.Rule + for rows.Next() { + var r models.Rule + var condsStr, actionsStr string + err := rows.Scan( + &r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt, + ) + if err != nil { + return nil, err + } + _ = json.Unmarshal([]byte(condsStr), &r.Conditions) + _ = json.Unmarshal([]byte(actionsStr), &r.Actions) + rules = append(rules, r) + } + return rules, nil +} + +func (d *DB) GetRule(id string) (*models.Rule, error) { + query := ` + SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at + FROM rules + WHERE id = ? + ` + var r models.Rule + var condsStr, actionsStr string + err := d.QueryRow(query, id).Scan( + &r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt, + ) + if err != nil { + return nil, err + } + _ = json.Unmarshal([]byte(condsStr), &r.Conditions) + _ = json.Unmarshal([]byte(actionsStr), &r.Actions) + return &r, nil +} + +func (d *DB) CreateRule(name, trigger, matchMode string, conditions []models.RuleCondition, actions []models.RuleAction) (*models.Rule, error) { + id := uuid.New().String() + now := time.Now().UTC() + if matchMode == "" { + matchMode = "all" + } + condsBytes, _ := json.Marshal(conditions) + actionsBytes, _ := json.Marshal(actions) + + query := ` + INSERT INTO rules ( + id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?) + ` + _, err := d.Exec(query, id, name, trigger, string(condsBytes), matchMode, string(actionsBytes), now, now) + if err != nil { + return nil, err + } + + return &models.Rule{ + ID: id, + Name: name, + Trigger: trigger, + Conditions: conditions, + MatchMode: matchMode, + Actions: actions, + IsActive: true, + CreatedAt: now, + UpdatedAt: now, + }, nil +} + +func (d *DB) UpdateRule(id string, name, trigger, matchMode string, conditions []models.RuleCondition, actions []models.RuleAction, isActive bool) (*models.Rule, error) { + now := time.Now().UTC() + if matchMode == "" { + matchMode = "all" + } + condsBytes, _ := json.Marshal(conditions) + actionsBytes, _ := json.Marshal(actions) + + query := ` + UPDATE rules + SET name = ?, trigger = ?, conditions = ?, match_mode = ?, actions = ?, is_active = ?, updated_at = ? + WHERE id = ? + ` + res, err := d.Exec(query, name, trigger, string(condsBytes), matchMode, string(actionsBytes), isActive, now, id) + if err != nil { + return nil, err + } + if n, _ := res.RowsAffected(); n == 0 { + return nil, fmt.Errorf("rule not found") + } + + return d.GetRule(id) +} + +func (d *DB) DeleteRule(id string) error { + _, err := d.Exec("DELETE FROM rules WHERE id = ?", id) + return err +} + func (d *DB) GetStats() (*models.Stats, error) { var s models.Stats err := d.QueryRow("SELECT COUNT(*) FROM feeds").Scan(&s.TotalFeeds) @@ -693,5 +999,6 @@ func (d *DB) GetStats() (*models.Stats, error) { if err != nil { return nil, err } + _ = d.QueryRow("SELECT COUNT(*) FROM rules").Scan(&s.TotalRules) return &s, nil } diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 42a1d32..5a80541 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -72,8 +72,8 @@ func TestDatabaseOperations(t *testing.T) { if err != nil { t.Fatalf("UpsertFeedEntries failed: %v", err) } - if inserted != 2 { - t.Fatalf("expected 2 inserted entries, got %d", inserted) + if len(inserted) != 2 { + t.Fatalf("expected 2 inserted entries, got %d", len(inserted)) } // 5. List entries @@ -94,12 +94,69 @@ func TestDatabaseOperations(t *testing.T) { t.Fatalf("UpdateEntryStarred failed: %v", err) } - // 7. Verify Stats + // 7. Test Variant CRUD + variant := models.EntryVariant{ + EntryID: firstID, + VariantKey: "readability_full_text", + ContentType: "html", + TextValue: "") + sb.WriteString(html.EscapeString(l)) + sb.WriteString("
\n") + } + } + } + + for _, block := range payload.Blocks { + txt := strings.TrimSpace(block.Text) + if txt != "" { + sb.WriteString("") + sb.WriteString(html.EscapeString(txt)) + sb.WriteString("
\n") + } + } + + for _, ent := range payload.EntityMap { + if strings.EqualFold(ent.Type, "IMAGE") && ent.Data.Src != "" { + fmt.Fprintf(&sb, `
就这样吧评论: 奥德赛终于看完了奥德赛,非常精彩!
") { + t.Fatalf("expected formatted paragraph in output, got: %s", combined) + } + if !strings.Contains(combined, "img9.doubanio.com/poster.jpg") { + t.Fatalf("expected poster in output, got: %s", combined) + } +} diff --git a/internal/rules/engine.go b/internal/rules/engine.go new file mode 100644 index 0000000..cb973e4 --- /dev/null +++ b/internal/rules/engine.go @@ -0,0 +1,82 @@ +package rules + +import ( + "context" + + "git.pengzhan.dev/haopengzhan/rsspipe/internal/db" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/events" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/logger" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" + "go.uber.org/zap" +) + +type WorkerDispatcher interface { + Dispatch(ctx context.Context, action models.RuleAction, entry models.FeedEntry, depth int) +} + +type Engine struct { + db *db.DB + bus *events.EventBus + evaluator *Evaluator + dispatcher WorkerDispatcher + log *zap.Logger +} + +func NewEngine(database *db.DB, bus *events.EventBus, dispatcher WorkerDispatcher) *Engine { + return &Engine{ + db: database, + bus: bus, + evaluator: NewEvaluator(), + dispatcher: dispatcher, + log: logger.Named("rules-engine"), + } +} + +// Start registers event bus subscriptions to evaluate rules reactively. +func (e *Engine) Start(ctx context.Context) { + e.bus.Subscribe(events.EventPostCreated, e.handleEvent) + e.bus.Subscribe(events.EventPostUpdated, e.handleEvent) + e.bus.Subscribe(events.EventVariantUpdated, e.handleEvent) + e.bus.Subscribe(events.EventManualTrigger, e.handleEvent) + e.log.Info("Rules engine started, listening for events") +} + +func (e *Engine) handleEvent(ctx context.Context, evt events.Event) error { + trigger := string(evt.Type) + rules, err := e.db.GetActiveRules(trigger) + if err != nil { + e.log.Error("Failed to fetch active rules", zap.String("trigger", trigger), zap.Error(err)) + return err + } + + if len(rules) == 0 { + return nil + } + + // Fetch full entry with latest variants if entry ID is present + entry := evt.Entry + if entry.ID != "" { + if full, err := e.db.GetEntry(entry.ID); err == nil && full != nil { + entry = *full + } + } + + for _, r := range rules { + if e.evaluator.Evaluate(r, entry) { + e.log.Info("Rule matched", + zap.String("rule_id", r.ID), + zap.String("rule_name", r.Name), + zap.String("entry_id", entry.ID), + zap.Int("actions_count", len(r.Actions)), + ) + + for _, action := range r.Actions { + if e.dispatcher != nil { + e.dispatcher.Dispatch(ctx, action, entry, evt.Depth+1) + } + } + } + } + + return nil +} diff --git a/internal/rules/evaluator.go b/internal/rules/evaluator.go new file mode 100644 index 0000000..8fbc78e --- /dev/null +++ b/internal/rules/evaluator.go @@ -0,0 +1,133 @@ +package rules + +import ( + "fmt" + "regexp" + "strings" + + "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" +) + +type Evaluator struct{} + +func NewEvaluator() *Evaluator { + return &Evaluator{} +} + +// Evaluate checks whether a FeedEntry satisfies a Rule's conditions. +func (e *Evaluator) Evaluate(rule models.Rule, entry models.FeedEntry) bool { + if len(rule.Conditions) == 0 { + return true + } + + matchMode := strings.ToLower(rule.MatchMode) + if matchMode == "" || matchMode == "all" { + for _, cond := range rule.Conditions { + if !e.evaluateCondition(cond, entry) { + return false + } + } + return true + } + + // matchMode == "any" + for _, cond := range rule.Conditions { + if e.evaluateCondition(cond, entry) { + return true + } + } + return false +} + +func (e *Evaluator) evaluateCondition(cond models.RuleCondition, entry models.FeedEntry) bool { + actualValue := e.resolveField(cond.Field, entry) + return e.compare(actualValue, cond.Operator, cond.Value) +} + +func (e *Evaluator) resolveField(field string, entry models.FeedEntry) interface{} { + switch strings.ToLower(field) { + case "feed_id": + return entry.FeedID + case "feed_title": + return entry.FeedTitle + case "title": + return entry.Title + case "author": + return entry.Author + case "url": + return entry.URL + case "content", "rss_content": + return entry.Content + case "is_read": + return entry.IsRead + case "is_starred": + return entry.IsStarred + } + + // Variant field resolution: "variants.keywords", "variants.readability_full_text", "variants.tags" + if strings.HasPrefix(field, "variants.") { + variantKey := strings.TrimPrefix(field, "variants.") + if entry.Variants != nil { + if v, ok := entry.Variants[variantKey]; ok { + return v.TextValue + } + } + } + + return nil +} + +func (e *Evaluator) compare(actual interface{}, op string, expected interface{}) bool { + if actual == nil { + return op == "neq" + } + + actualStr := fmt.Sprintf("%v", actual) + expectedStr := fmt.Sprintf("%v", expected) + + switch strings.ToLower(op) { + case "eq", "==": + return strings.EqualFold(actualStr, expectedStr) + case "neq", "!=": + return !strings.EqualFold(actualStr, expectedStr) + case "contains": + return strings.Contains(strings.ToLower(actualStr), strings.ToLower(expectedStr)) + case "not_contains": + return !strings.Contains(strings.ToLower(actualStr), strings.ToLower(expectedStr)) + case "in": + // expected can be []interface{}, []string, or comma-separated string + switch v := expected.(type) { + case []interface{}: + for _, item := range v { + if strings.EqualFold(actualStr, fmt.Sprintf("%v", item)) { + return true + } + } + return false + case []string: + for _, item := range v { + if strings.EqualFold(actualStr, item) { + return true + } + } + return false + case string: + items := strings.Split(v, ",") + for _, item := range items { + if strings.EqualFold(strings.TrimSpace(actualStr), strings.TrimSpace(item)) { + return true + } + } + return false + } + return false + case "matches", "regex": + re, err := regexp.Compile(expectedStr) + if err != nil { + return false + } + return re.MatchString(actualStr) + default: + return false + } +} diff --git a/internal/rules/rules_test.go b/internal/rules/rules_test.go new file mode 100644 index 0000000..ff4ee4b --- /dev/null +++ b/internal/rules/rules_test.go @@ -0,0 +1,70 @@ +package rules + +import ( + "testing" + "time" + + "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" +) + +func TestRuleEvaluator(t *testing.T) { + eval := NewEvaluator() + + entry := models.FeedEntry{ + ID: "entry-1", + FeedID: "feed-hn", + FeedTitle: "Hacker News", + Title: "Announcing Go 1.24 Release", + Author: "Go Team", + Content: "Go 1.24 brings new features...
", + Variants: map[string]models.EntryVariant{ + "keywords": { + EntryID: "entry-1", + VariantKey: "keywords", + ContentType: "json", + TextValue: `["go", "compiler", "release"]`, + }, + }, + PublishedAt: time.Now(), + } + + // Test 1: Title contains Go AND FeedID == feed-hn + rule1 := models.Rule{ + ID: "r1", + Name: "Tech Feeds", + MatchMode: "all", + Conditions: []models.RuleCondition{ + {Field: "feed_id", Operator: "eq", Value: "feed-hn"}, + {Field: "title", Operator: "contains", Value: "Go 1.24"}, + }, + } + if !eval.Evaluate(rule1, entry) { + t.Error("expected rule1 to match entry") + } + + // Test 2: Negative condition (author != Python Team) + rule2 := models.Rule{ + ID: "r2", + Name: "Author check", + MatchMode: "all", + Conditions: []models.RuleCondition{ + {Field: "author", Operator: "neq", Value: "Python Team"}, + }, + } + if !eval.Evaluate(rule2, entry) { + t.Error("expected rule2 to match entry") + } + + // Test 3: Variant matching (variants.keywords contains compiler) + rule3 := models.Rule{ + ID: "r3", + Name: "Keywords check", + MatchMode: "all", + Conditions: []models.RuleCondition{ + {Field: "variants.keywords", Operator: "contains", Value: "compiler"}, + }, + } + if !eval.Evaluate(rule3, entry) { + t.Error("expected rule3 to match entry variant") + } +} diff --git a/internal/worker/pool.go b/internal/worker/pool.go new file mode 100644 index 0000000..64b7502 --- /dev/null +++ b/internal/worker/pool.go @@ -0,0 +1,179 @@ +package worker + +import ( + "context" + "fmt" + "sync" + "time" + + "git.pengzhan.dev/haopengzhan/rsspipe/internal/db" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/events" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/logger" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" + "go.uber.org/zap" +) + +type ActionResult struct { + ResultVariant *models.EntryVariant + MarkRead *bool + MarkStarred *bool +} + +type ActionHandler func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*ActionResult, error) + +type Task struct { + Action models.RuleAction + Entry models.FeedEntry + Depth int +} + +type Pool struct { + db *db.DB + bus *events.EventBus + handlers map[string]ActionHandler + queue chan Task + workers int + wg sync.WaitGroup + log *zap.Logger + mu sync.RWMutex +} + +func NewPool(database *db.DB, bus *events.EventBus, workers, queueSize int) *Pool { + if workers <= 0 { + workers = 5 + } + if queueSize <= 0 { + queueSize = 100 + } + return &Pool{ + db: database, + bus: bus, + handlers: make(map[string]ActionHandler), + queue: make(chan Task, queueSize), + workers: workers, + log: logger.Named("worker-pool"), + } +} + +func (p *Pool) RegisterHandler(actionType string, handler ActionHandler) { + p.mu.Lock() + defer p.mu.Unlock() + p.handlers[actionType] = handler + p.log.Info("Registered action handler", zap.String("action", actionType)) +} + +func (p *Pool) Start(ctx context.Context) { + p.log.Info("Starting worker pool", zap.Int("worker_count", p.workers)) + for i := 0; i < p.workers; i++ { + p.wg.Add(1) + go p.worker(ctx, i) + } +} + +func (p *Pool) Stop() { + close(p.queue) + p.wg.Wait() + p.log.Info("Worker pool stopped") +} + +func (p *Pool) Dispatch(ctx context.Context, action models.RuleAction, entry models.FeedEntry, depth int) { + select { + case p.queue <- Task{Action: action, Entry: entry, Depth: depth}: + default: + p.log.Warn("Worker pool queue full, task dropped", + zap.String("action", action.Type), + zap.String("entry_id", entry.ID), + ) + } +} + +func (p *Pool) ExecuteDirect(ctx context.Context, actionType string, entry models.FeedEntry, params map[string]interface{}) (*ActionResult, error) { + p.mu.RLock() + handler, ok := p.handlers[actionType] + p.mu.RUnlock() + + if !ok { + return nil, fmt.Errorf("unknown action handler: %s", actionType) + } + + return handler(ctx, entry, params) +} + +func (p *Pool) worker(ctx context.Context, id int) { + defer p.wg.Done() + for { + select { + case <-ctx.Done(): + return + case task, ok := <-p.queue: + if !ok { + return + } + p.processTask(ctx, task) + } + } +} + +func (p *Pool) processTask(ctx context.Context, task Task) { + p.mu.RLock() + handler, ok := p.handlers[task.Action.Type] + p.mu.RUnlock() + + if !ok { + p.log.Warn("No handler registered for action", zap.String("type", task.Action.Type)) + return + } + + start := time.Now() + res, err := handler(ctx, task.Entry, task.Action.Params) + duration := time.Since(start) + + if err != nil { + p.log.Error("Action execution failed", + zap.String("action", task.Action.Type), + zap.String("entry_id", task.Entry.ID), + zap.Duration("duration", duration), + zap.Error(err), + ) + return + } + + if res == nil { + return + } + + // 1. Save Variant if returned + if res.ResultVariant != nil { + if err := p.db.SetVariant(*res.ResultVariant); err != nil { + p.log.Error("Failed to save entry variant", + zap.String("entry_id", task.Entry.ID), + zap.String("variant_key", res.ResultVariant.VariantKey), + zap.Error(err), + ) + } else { + p.log.Info("Entry variant saved", + zap.String("entry_id", task.Entry.ID), + zap.String("variant_key", res.ResultVariant.VariantKey), + zap.Duration("duration", duration), + ) + + // Publish EventVariantUpdated to enable downstream chained triggers + if p.bus != nil { + p.bus.Publish(ctx, events.Event{ + Type: events.EventVariantUpdated, + Entry: task.Entry, + VariantKey: res.ResultVariant.VariantKey, + Depth: task.Depth, + }) + } + } + } + + // 2. Apply Read/Star updates + if res.MarkRead != nil { + _ = p.db.UpdateEntryRead(task.Entry.ID, *res.MarkRead) + } + if res.MarkStarred != nil { + _ = p.db.UpdateEntryStarred(task.Entry.ID, *res.MarkStarred) + } +} diff --git a/web/src/App.tsx b/web/src/App.tsx index 5f75239..bfc4c52 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -159,6 +159,19 @@ export const App: React.FC = () => { setFocusedIndex(entries.findIndex((e) => e.id === selectedEntryId)); return; } + // If not found in current list, load directly so reading pane stays populated + api.getEntry(selectedEntryId).then((entry) => { + setActiveArticle(entry); + }).catch(() => { + // fallback to top + if (entries[0]) { + setFocusedIndex(0); + setActiveArticle(entries[0]); + setSelectedEntryId(entries[0].id); + localStorage.setItem('rsspipe_selected_entry_id', entries[0].id); + } + }); + return; } setFocusedIndex(0); @@ -177,36 +190,7 @@ export const App: React.FC = () => { } }; - // Auto-Refresh & Background Polling (Every 30s + on focus / visibility change) - useEffect(() => { - if (!currentUser) return; - - const interval = setInterval(() => { - loadSidebarData(); - loadEntries(); - }, 30000); - - const handleWindowFocus = () => { - loadSidebarData(); - loadEntries(); - }; - - const handleVisibilityChange = () => { - if (document.visibilityState === 'visible') { - loadSidebarData(); - loadEntries(); - } - }; - - window.addEventListener('focus', handleWindowFocus); - document.addEventListener('visibilitychange', handleVisibilityChange); - - return () => { - clearInterval(interval); - window.removeEventListener('focus', handleWindowFocus); - document.removeEventListener('visibilitychange', handleVisibilityChange); - }; - }, [currentUser, loadSidebarData, loadEntries]); + // Refresh data only on initial mount or when user clicks refresh / changes filter // Toggle Read Action with optimistic update const handleToggleRead = async (entry: FeedEntry, e?: React.MouseEvent) => { @@ -516,6 +500,10 @@ export const App: React.FC = () => { onToggleRead={handleToggleRead} onToggleStar={handleToggleStar} onMarkAllRead={handleMarkAllRead} + onUpdateEntry={(updated) => { + setActiveArticle(updated); + setEntries((prev) => prev.map((e) => (e.id === updated.id ? updated : e))); + }} sidebarCollapsed={sidebarCollapsed} /> ) : viewMode === 'waterfall' ? ( @@ -526,6 +514,10 @@ export const App: React.FC = () => { entries={entries} onToggleRead={handleToggleRead} onToggleStar={handleToggleStar} + onUpdateEntry={(updated) => { + setActiveArticle(updated); + setEntries((prev) => prev.map((e) => (e.id === updated.id ? updated : e))); + }} /> ) : (