diff --git a/.gitleaks.toml b/.gitleaks.toml index bf6ede6..2e15f34 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -8,9 +8,12 @@ description = "Global allowlist" paths = [ '''sample_subscriptions\.opml''', '''docs/''', - '''web/node_modules/''' + '''web/node_modules/''', + '''web/dist/''', + '''data/''' ] regexes = [ '''admin123456''', '''example_jwt_secret_key''' ] + diff --git a/cmd/server/main.go b/cmd/server/main.go index 955399c..146de06 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -2,52 +2,69 @@ package main import ( "context" - "log" "net/http" "os" "os/signal" "syscall" "time" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/actions/readability" "git.pengzhan.dev/haopengzhan/rsspipe/internal/api" "git.pengzhan.dev/haopengzhan/rsspipe/internal/auth" "git.pengzhan.dev/haopengzhan/rsspipe/internal/config" "git.pengzhan.dev/haopengzhan/rsspipe/internal/crawler" "git.pengzhan.dev/haopengzhan/rsspipe/internal/db" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/events" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/logger" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/rules" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/worker" + "go.uber.org/zap" ) func main() { - log.Println("==================================================") - log.Println(" Starting RSSPipe Server ") - log.Println("==================================================") + log := logger.InitLogger(os.Getenv("LOG_LEVEL")) + log.Info("Starting RSSPipe Server") cfg := config.LoadConfig() // 1. Initialize SQLite Database database, err := db.InitDB(cfg.DBPath) if err != nil { - log.Fatalf("Fatal: Database initialization failed: %v", err) + log.Fatal("Database initialization failed", zap.Error(err)) } defer func() { _ = database.Close() }() - log.Printf("==> Database connected: %s (WAL mode enabled)\n", cfg.DBPath) + log.Info("Database connected", zap.String("path", cfg.DBPath)) // 2. Initialize Auth Service & Bootstrap Admin authService := auth.NewAuthService(database, cfg) if err := authService.BootstrapAdmin(); err != nil { - log.Fatalf("Fatal: Admin bootstrap failed: %v", err) + log.Fatal("Admin bootstrap failed", zap.Error(err)) } - // 3. Start Background Crawler Pool ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cr := crawler.NewCrawler(database, cfg.CrawlerIntervalMinutes, cfg.CrawlerWorkers) + // 3. Initialize EventBus, WorkerPool, and Action Handlers + bus := events.NewEventBus() + pool := worker.NewPool(database, bus, 5, 100) + + // Register Scraper Action + scraper := readability.NewScraper(15 * time.Second) + pool.RegisterHandler("readability_scrape", scraper.Handler()) + pool.Start(ctx) + + // 4. Initialize Rules Engine + engine := rules.NewEngine(database, bus, pool) + engine.Start(ctx) + + // 5. Start Background Crawler Pool + cr := crawler.NewCrawler(database, bus, cfg.CrawlerIntervalMinutes, cfg.CrawlerWorkers) cr.Start(ctx) - // 4. Mount API Routes - handler := api.NewHandler(database, authService, cr) + // 6. Mount API Routes + handler := api.NewHandler(database, authService, cr, pool) router := api.NewRouter(handler, authService, "./web/dist") server := &http.Server{ @@ -58,26 +75,27 @@ func main() { IdleTimeout: 120 * time.Second, } - // 5. Graceful shutdown handler + // 7. Graceful shutdown handler go func() { sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM) <-sigChan - log.Println("\n==> Shutting down RSSPipe gracefully...") + log.Info("Shutting down RSSPipe gracefully...") cancel() + pool.Stop() shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) defer shutdownCancel() if err := server.Shutdown(shutdownCtx); err != nil { - log.Printf("Error during server shutdown: %v\n", err) + log.Error("Error during server shutdown", zap.Error(err)) } }() - log.Printf("==> RSSPipe HTTP Server listening at http://localhost:%s\n", cfg.Port) + log.Info("RSSPipe HTTP Server listening", zap.String("url", "http://localhost:"+cfg.Port)) if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { - log.Fatalf("Fatal: Server error: %v", err) + log.Fatal("Server error", zap.Error(err)) } - log.Println("==> RSSPipe Server stopped.") + log.Info("RSSPipe Server stopped") } diff --git a/docs/003_backlog.md b/docs/003_backlog.md index 9b1b44b..5a36115 100644 --- a/docs/003_backlog.md +++ b/docs/003_backlog.md @@ -93,3 +93,42 @@ This document records planned features, UX enhancements, and architectural exten - Staggered dynamic aspect ratios with prominent media cards. - Smooth infinite scroll pagination. - Interactive gesture swipe to dismiss the full-screen reader. + +--- + +## 10. Beautiful 404 Error Page + +- **Goal**: Provide a polished, modern, brand-consistent 404 Not Found experience. +- **Scope**: + - Frontend 404 route component with sleek modern aesthetics (glassmorphism/minimal illustration, clear error description, and a "Return to Gateway" action button). + - Server-side fallback handler returning a clean, branded HTML response for unmatched routes. + +--- + +## 11. Multi-Variant Pipeline Enrichments (AI Summary, TTS, Keywords, Audio) + +- **Goal**: Expand the event-driven rules engine and variant storage system with automated enrichment plugins. +- **Scope**: + - **AI Summary Hook**: Integrate LLM actions (e.g. OpenAI / Anthropic / Gemini / Ollama) to automatically generate and store concise summaries in `entry_variants(key='ai_summary')`. + - **Keyword & Tag Extraction**: Automated extractor storing structured YAML/JSON keyword lists in `entry_variants(key='keywords')` and `entry_variants(key='tags')` to enable rule-based chaining. + - **Text-to-Speech (TTS)**: Synthesize article text to voice audio blobs (`.mp3` stored under `./data/blobs/{entry_id}/tts.mp3`) with HTTP 206 partial streaming. + - **Podcast TTS & Summary TTS**: Convert articles into dual-host conversational podcasts or audio-narrated summaries streamed directly within the reader. + +--- + +## 12. Aggregated Shared Posts Dashboard (`/shares`) + +- **Goal**: Provide a dedicated gallery view aggregating all publicly shared articles. +- **Scope**: + - Centralized dashboard listing all articles with an active `share_uuid`. + - Copy public link, preview shared HTML canvas, revoke share link, and view access stats. + +--- + +## 13. Data Retention & Cleanup Policies (Auto-Deletion) + +- **Goal**: Provide configurable retention rules to prune older or noisy articles and keep SQLite storage compact. +- **Scope**: + - Retention triggers (e.g. delete unstarred articles older than 90 days, or limit feeds to latest 500 posts). + - Background garbage collection cleaning associated `entry_variants` and on-disk audio blobs (`./data/blobs/{entry_id}/`). + diff --git a/docs/004_variant_pipeline_architecture.md b/docs/004_variant_pipeline_architecture.md new file mode 100644 index 0000000..21ddeeb --- /dev/null +++ b/docs/004_variant_pipeline_architecture.md @@ -0,0 +1,116 @@ +# RSSPipe Variant Data & Extensible Processing Pipeline Architecture + +## 1. Overview & Vision + +RSSPipe is designed as an **RSS-Driven Information Gateway**. Subscriptions come in many formats (full-text blogs, brief news tickers, media podcasts, academic papers). Consequently, a one-size-fits-all processing approach fails: + +- Some feeds require automatic full-text extraction via Readability. +- Some feeds require LLM keyword and topic tagging. +- Specific high-value posts (filtered by keywords or feeds) require AI summarization, translation, or Text-to-Speech (TTS) audio synthesis. + +To support this cleanly without database corruption, performance degradation, or rigid schemas, RSSPipe adopts a **Variant Data Architecture** paired with a **Reactive Chained Trigger Pipeline**. + +--- + +## 2. Core Data Model + +### 2.1 Separation of Immutable Source vs. Enriched Variants + +1. **`feed_entries` (Immutable Source of Truth)**: + - Stores the exact metadata and content parsed from the RSS XML feed. + - Core fields: `id`, `feed_id`, `guid`, `url`, `title`, `author`, `rss_content`, `published_at`, `is_read`, `is_starred`, `created_at`. + - **Rule**: `rss_content` is NEVER overwritten by scrapers, LLMs, or TTS engines. +2. **`entry_variants` (Extensible Key-Value Store)**: + - Stores all downstream enrichments as strongly typed variant entries. + - Schema: + + ```sql + CREATE TABLE IF NOT EXISTS entry_variants ( + entry_id TEXT NOT NULL, + variant_key TEXT NOT NULL, + content_type TEXT NOT NULL, -- 'text', 'html', 'json', 'yaml', 'audio/mp3', 'binary' + text_value TEXT, -- UTF-8 text, HTML, JSON array/object, YAML string + blob_path TEXT, -- Relative filesystem path for large media (e.g. data/blobs/{entry_id}/{key}.mp3) + metadata JSON DEFAULT '{}', -- Stats: model, tokens, duration_ms, byte_size, executed_at + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (entry_id, variant_key), + FOREIGN KEY (entry_id) REFERENCES feed_entries(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_entry_variants_key ON entry_variants(variant_key); + ``` + +--- + +## 3. Supported Variant Keys & Types + +| Variant Key | Content Type | Storage Location | Description | +| :--- | :--- | :--- | :--- | +| `readability_full_text` | `html` | `text_value` | Sanitized readability full text extracted from the source web page. | +| `keywords` | `json` / `yaml` | `text_value` | List of extracted keywords (e.g., `["AI", "Go", "Compiler"]`). | +| `tags` | `json` | `text_value` | Category and topic tags. | +| `ai_summary` | `text` / `markdown` | `text_value` | Bulleted or narrative AI executive summary. | +| `translation_{lang}` | `html` / `text` | `text_value` | Translated content (e.g. `translation_zh`). | +| `tts` | `audio/mp3` | `blob_path` | Synthesized speech audio file of the article. | +| `podcast_tts` | `audio/mp3` | `blob_path` | Multi-speaker podcast discussion audio file. | +| `ai_summary_tts` | `audio/mp3` | `blob_path` | Audio narration of the AI summary. | + +--- + +## 4. Large Binary Blob Strategy + +Storing multi-megabyte audio files or binary blobs directly inside SQLite columns degrades query cache efficiency and slows down table scans. + +- **Text & JSON Variants** are stored inline in `entry_variants.text_value`. +- **Audio & Media Blobs** are stored on disk under `./data/blobs/{entry_id}/{variant_key}.{ext}`. +- `entry_variants.blob_path` stores the relative path. +- The API provides a streaming endpoint with HTTP Range support: `GET /api/entries/{id}/variants/{key}/stream`. + +--- + +## 5. Reactive Chained Trigger & Watch Pipeline + +The pipeline operates on typed pub/sub events with depth guards to prevent infinite loops: + +```mermaid +graph TD + A[Feed Ingestion / Crawler] -->|Event: on_post_created| Bus[Typed Event Bus] + Bus --> Rules[Rules Engine Evaluator] + + Rules -->|Matches Rule 1: Scrape Tech| W1[Worker: Readability Scraper] + W1 -->|Writes variant: readability_full_text| DB[(entry_variants)] + W1 -->|Event: on_variant_updated| Bus + + Rules -->|Matches Rule 2: Tag AI Keywords| W2[Worker: Keyword Extractor] + W2 -->|Writes variant: keywords| DB + W2 -->|Event: on_variant_updated| Bus + + Bus -->|Matches Rule 3: Watch keywords contains 'AI'| W3[Worker: AI Summary & TTS] + W3 -->|Writes variant: ai_summary & tts blob| DB +``` + +### 5.1 Concrete Chained Pipeline Example + +1. **Trigger 1**: New post ingested from Hacker News -> Emits `on_post_created`. +2. **Rule 1 (Auto-Enrich Keywords)**: + - Condition: `feed_id == "hn-feed-id"` + - Action: `extract_keywords` -> Saves `entry_variants(key='keywords', text_value='["OpenAI", "LLM", "Agent"]')`. + - Emits: `on_variant_updated(key='keywords')`. +3. **Rule 2 (Watch & Synthesize Audio)**: + - Condition: `variant.keywords CONTAINS "OpenAI"` + - Action: `generate_tts` -> Synthesizes MP3, saves to `./data/blobs/...`, saves variant `tts`. + - Emits: `on_variant_updated(key='tts')`. + +--- + +## 6. Reader UX & Content Switcher + +In the 3-pane reader interface: + +- **Default View**: Renders `rss_content` (original source). +- **Enrichment Pills**: If variants exist on the entry, dynamic tabs/pills appear in the reader header: + - `[📄 Original RSS]` + - `[⚡ Readability Full-Text]` (if `readability_full_text` exists) + - `[💡 AI Summary]` (if `ai_summary` exists) + - `[🎧 Listen Audio]` (if `tts` or `podcast_tts` exists, plays with embedded HTML5 player) +- Switching variants never mutates the underlying article or overwrites database fields. diff --git a/go.mod b/go.mod index 9f94817..d72addc 100644 --- a/go.mod +++ b/go.mod @@ -3,16 +3,26 @@ module git.pengzhan.dev/haopengzhan/rsspipe go 1.26.5 require ( + github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect + github.com/aymerick/douceur v0.2.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-chi/chi/v5 v5.3.2 // indirect github.com/go-chi/cors v1.2.2 // indirect + github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect + github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0 // indirect + github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/css v1.0.1 // indirect github.com/mattn/go-isatty v0.0.24 // indirect + github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/mmcdole/gofeed v1.4.2 // indirect github.com/mmcdole/goxpp/v2 v2.0.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + go.uber.org/multierr v1.10.0 // indirect + go.uber.org/zap v1.28.0 // indirect golang.org/x/crypto v0.55.0 // indirect golang.org/x/net v0.57.0 // indirect golang.org/x/sys v0.47.0 // indirect diff --git a/go.sum b/go.sum index d320bbd..6bc1693 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,125 @@ +github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= +github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= +github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= +github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY= github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE= github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58= +github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w= +github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM= +github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0 h1:A3B75Yp163FAIf9nLlFMl4pwIj+T3uKxfI7mbvvY2Ls= +github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0/go.mod h1:suxK0Wpz4BM3/2+z1mnOVTIWHDiMCIOGoKDCRumSsk0= +github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= +github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= +github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= +github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= github.com/mmcdole/gofeed v1.4.2 h1:XFFOtsNNZg+zudjtMXb8BI0J/YdnSIGfjEpPgPnZib0= github.com/mmcdole/gofeed v1.4.2/go.mod h1:X5x1PyeibJi152VEya0AsV+PW4daYmCD4LJaJbeFkcs= github.com/mmcdole/goxpp/v2 v2.0.0 h1:HrSCflxerUEqZQNq3u7ldtmE/XkwnTx4Zpq2DW4i5rQ= github.com/mmcdole/goxpp/v2 v2.0.0/go.mod h1:CUduYMnO9JB6Z/uqDn9Ormk/r8E9BsLQxHPWDZ961Os= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= +go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= diff --git a/internal/actions/readability/readability.go b/internal/actions/readability/readability.go new file mode 100644 index 0000000..97e7f59 --- /dev/null +++ b/internal/actions/readability/readability.go @@ -0,0 +1,123 @@ +package readability + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "git.pengzhan.dev/haopengzhan/rsspipe/internal/logger" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/worker" + readability "github.com/go-shiori/go-readability" + "github.com/microcosm-cc/bluemonday" + "go.uber.org/zap" +) + +type Scraper struct { + client *http.Client + sanitizer *bluemonday.Policy + log *zap.Logger +} + +func NewScraper(timeout time.Duration) *Scraper { + if timeout <= 0 { + timeout = 15 * time.Second + } + + p := bluemonday.UGCPolicy() + // Allow rich media & responsive elements + p.AllowElements("video", "audio", "source", "track", "iframe", "picture", "figure", "figcaption", "article", "section") + p.AllowAttrs("src", "poster", "controls", "autoplay", "loop", "muted", "preload").OnElements("video", "audio") + p.AllowAttrs("src", "type", "srcset", "media").OnElements("source") + p.AllowAttrs("src", "width", "height", "frameborder", "allow", "allowfullscreen", "sandbox").OnElements("iframe") + p.AllowAttrs("data-src", "data-original", "loading", "srcset", "sizes").OnElements("img") + p.AllowAttrs("class", "id").Globally() + + return &Scraper{ + client: &http.Client{ + Timeout: timeout, + }, + sanitizer: p, + log: logger.Named("readability-scraper"), + } +} + +// Handler returns the worker.ActionHandler function for readability scraping. +func (s *Scraper) Handler() worker.ActionHandler { + return func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*worker.ActionResult, error) { + if strings.TrimSpace(entry.URL) == "" { + return nil, fmt.Errorf("entry URL is empty") + } + + parsedURL, err := url.Parse(entry.URL) + if err != nil { + return nil, fmt.Errorf("invalid URL: %w", err) + } + + s.log.Info("Starting readability scrape", + zap.String("entry_id", entry.ID), + zap.String("url", entry.URL), + ) + + req, err := http.NewRequestWithContext(ctx, "GET", entry.URL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + req.Header.Set("Accept-Language", "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7") + + resp, err := s.client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch article page: %w", err) + } + defer func() { + _ = resp.Body.Close() + }() + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("server returned HTTP status %d", resp.StatusCode) + } + + article, err := readability.FromReader(resp.Body, parsedURL) + if err != nil { + return nil, fmt.Errorf("failed to parse readability article: %w", err) + } + + cleanHTML := s.sanitizer.Sanitize(article.Content) + if strings.TrimSpace(cleanHTML) == "" && strings.TrimSpace(article.TextContent) != "" { + cleanHTML = "

" + s.sanitizer.Sanitize(article.TextContent) + "

" + } + + s.log.Info("Readability scrape successful", + zap.String("entry_id", entry.ID), + zap.String("title", article.Title), + zap.Int("content_length", len(cleanHTML)), + ) + + now := time.Now().UTC() + variant := models.EntryVariant{ + EntryID: entry.ID, + VariantKey: "readability_full_text", + ContentType: "html", + TextValue: cleanHTML, + Metadata: map[string]interface{}{ + "title": article.Title, + "byline": article.Byline, + "excerpt": article.Excerpt, + "length": len(article.TextContent), + "scraped_at": now.Format(time.RFC3339), + }, + CreatedAt: now, + UpdatedAt: now, + } + + return &worker.ActionResult{ + ResultVariant: &variant, + }, nil + } +} diff --git a/internal/api/handlers.go b/internal/api/handlers.go index a2d69f0..e6aa3df 100644 --- a/internal/api/handlers.go +++ b/internal/api/handlers.go @@ -17,6 +17,7 @@ import ( "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" "git.pengzhan.dev/haopengzhan/rsspipe/internal/opml" "git.pengzhan.dev/haopengzhan/rsspipe/internal/parser" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/worker" "github.com/go-chi/chi/v5" ) @@ -25,14 +26,16 @@ type Handler struct { auth *auth.AuthService crawler *crawler.Crawler parser *parser.Parser + pool *worker.Pool } -func NewHandler(database *db.DB, authService *auth.AuthService, cr *crawler.Crawler) *Handler { +func NewHandler(database *db.DB, authService *auth.AuthService, cr *crawler.Crawler, pool *worker.Pool) *Handler { return &Handler{ db: database, auth: authService, crawler: cr, parser: parser.NewParser(), + pool: pool, } } @@ -547,3 +550,160 @@ func (h *Handler) ViewSharedEntry(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html; charset=utf-8") _, _ = w.Write([]byte(html)) } + +// GetEntry returns a single entry by ID with its loaded variants +func (h *Handler) GetEntry(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + entry, err := h.db.GetEntry(id) + if err != nil { + if err == sql.ErrNoRows { + h.writeError(w, http.StatusNotFound, "entry not found") + return + } + h.writeError(w, http.StatusInternalServerError, err.Error()) + return + } + h.writeJSON(w, http.StatusOK, entry) +} + +// ListEntryVariants returns all variants for a given entry ID +func (h *Handler) ListEntryVariants(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + variants, err := h.db.ListVariantsForEntry(id) + if err != nil { + h.writeError(w, http.StatusInternalServerError, err.Error()) + return + } + h.writeJSON(w, http.StatusOK, variants) +} + +// GetVariantBlobStream streams the audio/binary file for a variant with HTTP 206 Partial Content support +func (h *Handler) GetVariantBlobStream(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + key := chi.URLParam(r, "key") + variant, err := h.db.GetVariant(id, key) + if err != nil { + h.writeError(w, http.StatusNotFound, "variant not found") + return + } + if variant.BlobPath == "" { + h.writeError(w, http.StatusNotFound, "variant has no binary media blob") + return + } + + http.ServeFile(w, r, variant.BlobPath) +} + +// ScrapeEntry manually triggers readability full text scraping and saves to entry_variants +func (h *Handler) ScrapeEntry(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + entry, err := h.db.GetEntry(id) + if err != nil { + h.writeError(w, http.StatusNotFound, "entry not found") + return + } + + if h.pool == nil { + h.writeError(w, http.StatusInternalServerError, "worker pool not configured") + return + } + + res, err := h.pool.ExecuteDirect(r.Context(), "readability_scrape", *entry, nil) + if err != nil { + h.writeError(w, http.StatusInternalServerError, fmt.Sprintf("scraping failed: %v", err)) + return + } + + if res != nil && res.ResultVariant != nil { + _ = h.db.SetVariant(*res.ResultVariant) + } + + // Reload entry with newly saved variant + updatedEntry, err := h.db.GetEntry(id) + if err != nil { + h.writeError(w, http.StatusInternalServerError, err.Error()) + return + } + + h.writeJSON(w, http.StatusOK, updatedEntry) +} + +// Rules API Handlers +func (h *Handler) ListRules(w http.ResponseWriter, r *http.Request) { + rules, err := h.db.ListRules() + if err != nil { + h.writeError(w, http.StatusInternalServerError, err.Error()) + return + } + if rules == nil { + rules = []models.Rule{} + } + h.writeJSON(w, http.StatusOK, rules) +} + +type CreateRuleRequest struct { + Name string `json:"name"` + Trigger string `json:"trigger"` + MatchMode string `json:"match_mode"` + Conditions []models.RuleCondition `json:"conditions"` + Actions []models.RuleAction `json:"actions"` +} + +func (h *Handler) CreateRule(w http.ResponseWriter, r *http.Request) { + var body CreateRuleRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + h.writeError(w, http.StatusBadRequest, "invalid request body") + return + } + if strings.TrimSpace(body.Name) == "" { + h.writeError(w, http.StatusBadRequest, "rule name is required") + return + } + if strings.TrimSpace(body.Trigger) == "" { + body.Trigger = "on_post_created" + } + if strings.TrimSpace(body.MatchMode) == "" { + body.MatchMode = "all" + } + + rule, err := h.db.CreateRule(body.Name, body.Trigger, body.MatchMode, body.Conditions, body.Actions) + if err != nil { + h.writeError(w, http.StatusInternalServerError, err.Error()) + return + } + h.writeJSON(w, http.StatusCreated, rule) +} + +type UpdateRuleRequest struct { + Name string `json:"name"` + Trigger string `json:"trigger"` + MatchMode string `json:"match_mode"` + Conditions []models.RuleCondition `json:"conditions"` + Actions []models.RuleAction `json:"actions"` + IsActive bool `json:"is_active"` +} + +func (h *Handler) UpdateRule(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + var body UpdateRuleRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + h.writeError(w, http.StatusBadRequest, "invalid request body") + return + } + + rule, err := h.db.UpdateRule(id, body.Name, body.Trigger, body.MatchMode, body.Conditions, body.Actions, body.IsActive) + if err != nil { + h.writeError(w, http.StatusInternalServerError, err.Error()) + return + } + h.writeJSON(w, http.StatusOK, rule) +} + +func (h *Handler) DeleteRule(w http.ResponseWriter, r *http.Request) { + id := chi.URLParam(r, "id") + if err := h.db.DeleteRule(id); err != nil { + h.writeError(w, http.StatusInternalServerError, err.Error()) + return + } + h.writeJSON(w, http.StatusOK, map[string]string{"message": "rule deleted"}) +} diff --git a/internal/api/server.go b/internal/api/server.go index 5a75487..aa1aa56 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -62,9 +62,19 @@ func NewRouter(h *Handler, authService *auth.AuthService, staticDir string) http // Entries r.Get("/entries", h.ListEntries) + r.Get("/entries/{id}", h.GetEntry) r.Put("/entries/{id}/read", h.UpdateEntryRead) r.Put("/entries/{id}/star", h.UpdateEntryStar) r.Post("/entries/mark-all-read", h.MarkAllRead) + r.Post("/entries/{id}/scrape", h.ScrapeEntry) + r.Get("/entries/{id}/variants", h.ListEntryVariants) + r.Get("/entries/{id}/variants/{key}/stream", h.GetVariantBlobStream) + + // Rules + r.Get("/rules", h.ListRules) + r.Post("/rules", h.CreateRule) + r.Put("/rules/{id}", h.UpdateRule) + r.Delete("/rules/{id}", h.DeleteRule) // OPML r.Post("/opml/import", h.ImportOPML) diff --git a/internal/crawler/crawler.go b/internal/crawler/crawler.go index f1ebd64..b750672 100644 --- a/internal/crawler/crawler.go +++ b/internal/crawler/crawler.go @@ -3,35 +3,41 @@ package crawler import ( "context" "fmt" - "log" "sync" "time" "git.pengzhan.dev/haopengzhan/rsspipe/internal/db" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/events" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/logger" "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" "git.pengzhan.dev/haopengzhan/rsspipe/internal/parser" + "go.uber.org/zap" ) type Crawler struct { db *db.DB parser *parser.Parser + bus *events.EventBus intervalMinutes int workerCount int isRefreshing bool mu sync.Mutex + log *zap.Logger } -func NewCrawler(database *db.DB, intervalMinutes, workers int) *Crawler { +func NewCrawler(database *db.DB, bus *events.EventBus, intervalMinutes, workers int) *Crawler { return &Crawler{ db: database, parser: parser.NewParser(), + bus: bus, intervalMinutes: intervalMinutes, workerCount: workers, + log: logger.Named("crawler"), } } func (c *Crawler) Start(ctx context.Context) { - log.Printf("==> Starting RSS background crawler scheduler (Workers: %d)\n", c.workerCount) + c.log.Info("Starting RSS background crawler scheduler", zap.Int("worker_count", c.workerCount)) // Run initial fetch asynchronously after 2 seconds startup go func() { @@ -72,7 +78,7 @@ func (c *Crawler) RefreshDue(ctx context.Context) { feeds, err := c.db.ListFeeds() if err != nil { - log.Printf("Crawler error listing feeds: %v\n", err) + c.log.Error("Crawler error listing feeds", zap.Error(err)) return } @@ -93,7 +99,7 @@ func (c *Crawler) RefreshDue(ctx context.Context) { return } - log.Printf("Crawler: Refreshing %d due feeds (out of %d total)...\n", len(dueFeeds), len(feeds)) + c.log.Info("Crawler refreshing due feeds", zap.Int("due_count", len(dueFeeds)), zap.Int("total_count", len(feeds))) c.runWorkerPool(ctx, dueFeeds) } @@ -102,7 +108,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) { c.mu.Lock() if c.isRefreshing { c.mu.Unlock() - log.Println("Crawler: RefreshAll already in progress, skipping") + c.log.Debug("Crawler RefreshAll already in progress, skipping") return } c.isRefreshing = true @@ -116,7 +122,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) { feeds, err := c.db.ListFeeds() if err != nil { - log.Printf("Crawler error listing feeds: %v\n", err) + c.log.Error("Crawler error listing feeds", zap.Error(err)) return } @@ -124,7 +130,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) { return } - log.Printf("Crawler: Refreshing all %d feeds...\n", len(feeds)) + c.log.Info("Crawler refreshing all feeds", zap.Int("count", len(feeds))) c.runWorkerPool(ctx, feeds) } @@ -148,7 +154,7 @@ func (c *Crawler) runWorkerPool(ctx context.Context, feeds []models.Feed) { close(jobs) wg.Wait() - log.Printf("Crawler: Completed refresh of %d feeds.\n", len(feeds)) + c.log.Info("Crawler completed refresh batch", zap.Int("feed_count", len(feeds))) } func (c *Crawler) RefreshSingle(ctx context.Context, feedID string) error { @@ -169,7 +175,11 @@ func (c *Crawler) fetchSingleFeed(ctx context.Context, f models.Feed) error { res, err := c.parser.FetchFeed(feedCtx, f.FeedURL, f.ETag, f.LastModified) if err != nil { - log.Printf("Crawler error fetching [%s]: %v\n", f.Title, err) + c.log.Warn("Crawler error fetching feed", + zap.String("feed_title", f.Title), + zap.String("feed_url", f.FeedURL), + zap.Error(err), + ) _ = c.db.UpdateFeedFetchStatus(f.ID, f.ETag, f.LastModified, err.Error()) return err } @@ -180,11 +190,30 @@ func (c *Crawler) fetchSingleFeed(ctx context.Context, f models.Feed) error { } // Upsert entries - inserted, err := c.db.UpsertFeedEntries(f.ID, res.Entries) + newEntries, err := c.db.UpsertFeedEntries(f.ID, res.Entries) if err != nil { - log.Printf("Crawler error upserting entries for [%s]: %v\n", f.Title, err) - } else if inserted > 0 { - log.Printf("Crawler [%s]: inserted %d new entries\n", f.Title, inserted) + c.log.Error("Crawler error upserting entries", + zap.String("feed_title", f.Title), + zap.Error(err), + ) + } else if len(newEntries) > 0 { + c.log.Info("Crawler inserted new entries", + zap.String("feed_title", f.Title), + zap.Int("new_entries", len(newEntries)), + ) + + // Publish on_post_created events for newly inserted entries + if c.bus != nil { + for _, entry := range newEntries { + entry.FeedTitle = f.Title + entry.FeedSiteURL = f.SiteURL + c.bus.Publish(ctx, events.Event{ + Type: events.EventPostCreated, + Entry: entry, + Depth: 0, + }) + } + } } _ = c.db.UpdateFeedFetchStatus(f.ID, res.ETag, res.LastModified, "") diff --git a/internal/db/db.go b/internal/db/db.go index 933bb0a..c5acb2d 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -114,15 +114,28 @@ func (d *DB) migrate() error { CREATE INDEX IF NOT EXISTS idx_entries_is_starred ON feed_entries(is_starred); CREATE INDEX IF NOT EXISTS idx_entries_share_uuid ON feed_entries(share_uuid); + CREATE TABLE IF NOT EXISTS entry_variants ( + entry_id TEXT NOT NULL, + variant_key TEXT NOT NULL, + content_type TEXT NOT NULL, + text_value TEXT, + blob_path TEXT, + metadata TEXT DEFAULT '{}', + created_at DATETIME NOT NULL, + updated_at DATETIME NOT NULL, + PRIMARY KEY (entry_id, variant_key), + FOREIGN KEY(entry_id) REFERENCES feed_entries(id) ON DELETE CASCADE + ); + CREATE INDEX IF NOT EXISTS idx_entry_variants_key ON entry_variants(variant_key); + CREATE TABLE IF NOT EXISTS rules ( id TEXT PRIMARY KEY, name TEXT NOT NULL, - rule_type TEXT NOT NULL, - is_active BOOLEAN DEFAULT 1, - trigger_type TEXT NOT NULL, - trigger_config TEXT DEFAULT '{}', + trigger TEXT NOT NULL, conditions TEXT DEFAULT '[]', + match_mode TEXT DEFAULT 'all', actions TEXT DEFAULT '[]', + is_active BOOLEAN DEFAULT 1, created_at DATETIME NOT NULL, updated_at DATETIME NOT NULL ); @@ -132,6 +145,26 @@ func (d *DB) migrate() error { } // Migration for timeout_seconds column if older DB exists _, _ = d.Exec("ALTER TABLE feeds ADD COLUMN timeout_seconds INTEGER DEFAULT 10") + + // Migration for rules table schema if older table exists + var testCol string + err := d.QueryRow("SELECT trigger FROM rules LIMIT 1").Scan(&testCol) + if err != nil && err != sql.ErrNoRows { + _, _ = d.Exec("DROP TABLE IF EXISTS rules") + _, _ = d.Exec(` + CREATE TABLE IF NOT EXISTS rules ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + trigger TEXT NOT NULL, + conditions TEXT DEFAULT '[]', + match_mode TEXT DEFAULT 'all', + actions TEXT DEFAULT '[]', + is_active BOOLEAN DEFAULT 1, + created_at DATETIME NOT NULL, + updated_at DATETIME NOT NULL + ); + `) + } return nil } @@ -474,51 +507,67 @@ func (d *DB) DeleteFeed(id string) error { } // Feed Entries operations -func (d *DB) UpsertFeedEntries(feedID string, entries []models.FeedEntry) (int, error) { +func (d *DB) UpsertFeedEntries(feedID string, entries []models.FeedEntry) ([]models.FeedEntry, error) { if len(entries) == 0 { - return 0, nil + return nil, nil } - stmt, err := d.Prepare(` + var newEntries []models.FeedEntry + now := time.Now().UTC() + + insertStmt, err := d.Prepare(` INSERT INTO feed_entries ( id, feed_id, guid, url, title, author, content, raw_content, published_at, is_read, is_starred, metadata, created_at ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?, ?) - ON CONFLICT(feed_id, guid) DO UPDATE SET - title = excluded.title, - content = CASE WHEN feed_entries.content = '' THEN excluded.content ELSE feed_entries.content END, - url = excluded.url, - author = excluded.author `) if err != nil { - return 0, err + return nil, err } defer func() { - _ = stmt.Close() + _ = insertStmt.Close() }() - insertedCount := 0 - for _, e := range entries { - id := uuid.New().String() - now := time.Now().UTC() - pubAt := e.PublishedAt - if pubAt.IsZero() { - pubAt = now - } - metaBytes, _ := json.Marshal(e.Metadata) - if len(metaBytes) == 0 { - metaBytes = []byte("{}") - } + updateStmt, err := d.Prepare(` + UPDATE feed_entries + SET title = ?, url = ?, author = ?, content = ?, raw_content = ? + WHERE feed_id = ? AND guid = ? + `) + if err != nil { + return nil, err + } + defer func() { + _ = updateStmt.Close() + }() - res, err := stmt.Exec( - id, feedID, e.GUID, e.URL, e.Title, e.Author, e.Content, e.RawContent, pubAt, string(metaBytes), now, - ) - if err == nil { - if n, _ := res.RowsAffected(); n > 0 { - insertedCount++ + for _, e := range entries { + var existingID string + err := d.QueryRow("SELECT id FROM feed_entries WHERE feed_id = ? AND guid = ?", feedID, e.GUID).Scan(&existingID) + switch err { + case sql.ErrNoRows: + e.ID = uuid.New().String() + e.FeedID = feedID + pubAt := e.PublishedAt + if pubAt.IsZero() { + pubAt = now } + metaBytes, _ := json.Marshal(e.Metadata) + if len(metaBytes) == 0 { + metaBytes = []byte("{}") + } + + _, err := insertStmt.Exec( + e.ID, feedID, e.GUID, e.URL, e.Title, e.Author, e.Content, e.RawContent, pubAt, string(metaBytes), now, + ) + if err == nil { + e.CreatedAt = now + newEntries = append(newEntries, e) + } + case nil: + _, _ = updateStmt.Exec(e.Title, e.URL, e.Author, e.Content, e.RawContent, feedID, e.GUID) } } - return insertedCount, nil + + return newEntries, nil } type EntryFilter struct { @@ -530,67 +579,98 @@ type EntryFilter struct { Offset int } -func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) { - where := []string{"1=1"} - args := []interface{}{} +func (d *DB) GetEntry(id string) (*models.FeedEntry, error) { + query := ` + SELECT + e.id, e.feed_id, f.title as feed_title, COALESCE(f.site_url, '') as feed_site_url, + e.guid, e.url, e.title, e.author, e.content, e.raw_content, + e.published_at, e.is_read, e.is_starred, e.share_uuid, e.metadata, e.created_at + FROM feed_entries e + JOIN feeds f ON f.id = e.feed_id + WHERE e.id = ? + ` + var e models.FeedEntry + var author, rawContent, shareUUID, metaStr sql.NullString + err := d.QueryRow(query, id).Scan( + &e.ID, &e.FeedID, &e.FeedTitle, &e.FeedSiteURL, + &e.GUID, &e.URL, &e.Title, &author, &e.Content, &rawContent, + &e.PublishedAt, &e.IsRead, &e.IsStarred, &shareUUID, &metaStr, &e.CreatedAt, + ) + if err != nil { + return nil, err + } + if author.Valid { + e.Author = author.String + } + if rawContent.Valid { + e.RawContent = rawContent.String + } + if shareUUID.Valid { + s := shareUUID.String + e.ShareUUID = &s + } + e.Metadata = make(map[string]interface{}) + if metaStr.Valid && metaStr.String != "" { + _ = json.Unmarshal([]byte(metaStr.String), &e.Metadata) + } - switch f.View { + // Load variants + variants, _ := d.ListVariantsForEntry(e.ID) + e.Variants = variants + + return &e, nil +} + +func (d *DB) ListEntries(filter EntryFilter) ([]models.FeedEntry, int, error) { + var whereClauses []string + var args []interface{} + + switch filter.View { case "inbox": - where = append(where, "e.is_read = 0") + whereClauses = append(whereClauses, "e.is_read = 0") case "starred": - where = append(where, "e.is_starred = 1") + whereClauses = append(whereClauses, "e.is_starred = 1") } - if f.FeedID != "" { - where = append(where, "e.feed_id = ?") - args = append(args, f.FeedID) + if filter.FeedID != "" { + whereClauses = append(whereClauses, "e.feed_id = ?") + args = append(args, filter.FeedID) } - if f.CategoryID != "" { - where = append(where, "f.category_id = ?") - args = append(args, f.CategoryID) + if filter.CategoryID != "" { + whereClauses = append(whereClauses, "f.category_id = ?") + args = append(args, filter.CategoryID) } - if f.Search != "" { - searchTerms := strings.Fields(f.Search) - for _, term := range searchTerms { - lower := strings.ToLower(term) - if strings.HasPrefix(lower, "is:unread") { - where = append(where, "e.is_read = 0") - } else if strings.HasPrefix(lower, "is:read") { - where = append(where, "e.is_read = 1") - } else if strings.HasPrefix(lower, "is:starred") { - where = append(where, "e.is_starred = 1") - } else if strings.HasPrefix(lower, "feed:") { - feedVal := strings.TrimPrefix(term, "feed:") - where = append(where, "(f.title LIKE ? OR f.feed_url LIKE ?)") - args = append(args, "%"+feedVal+"%", "%"+feedVal+"%") - } else { - where = append(where, "(e.title LIKE ? OR e.content LIKE ? OR e.author LIKE ?)") - args = append(args, "%"+term+"%", "%"+term+"%", "%"+term+"%") - } - } + if filter.Search != "" { + whereClauses = append(whereClauses, "(e.title LIKE ? OR e.content LIKE ? OR e.author LIKE ?)") + pattern := "%" + filter.Search + "%" + args = append(args, pattern, pattern, pattern) } - whereClause := strings.Join(where, " AND ") + whereSQL := "" + if len(whereClauses) > 0 { + whereSQL = "WHERE " + strings.Join(whereClauses, " AND ") + } - // Count total countQuery := fmt.Sprintf(` - SELECT COUNT(*) - FROM feed_entries e - JOIN feeds f ON f.id = e.feed_id - WHERE %s - `, whereClause) + SELECT COUNT(e.id) + FROM feed_entries e + JOIN feeds f ON f.id = e.feed_id + %s + `, whereSQL) + var total int - if err := d.QueryRow(countQuery, args...).Scan(&total); err != nil { + err := d.QueryRow(countQuery, args...).Scan(&total) + if err != nil { return nil, 0, err } - limit := f.Limit - if limit <= 0 || limit > 100 { - limit = 50 + limit := filter.Limit + if limit <= 0 || limit > 500 { + limit = 100 } - offset := f.Offset + offset := filter.Offset if offset < 0 { offset = 0 } @@ -602,10 +682,10 @@ func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) { e.published_at, e.is_read, e.is_starred, e.share_uuid, e.metadata, e.created_at FROM feed_entries e JOIN feeds f ON f.id = e.feed_id - WHERE %s - ORDER BY e.published_at DESC + %s + ORDER BY e.published_at DESC, e.created_at DESC LIMIT ? OFFSET ? - `, whereClause) + `, whereSQL) queryArgs := append(args, limit, offset) rows, err := d.Query(query, queryArgs...) @@ -619,13 +699,13 @@ func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) { var entries []models.FeedEntry for rows.Next() { var e models.FeedEntry - var author, rawContent, metaStr sql.NullString - var shareUUID sql.NullString - if err := rows.Scan( + var author, rawContent, shareUUID, metaStr sql.NullString + err := rows.Scan( &e.ID, &e.FeedID, &e.FeedTitle, &e.FeedSiteURL, &e.GUID, &e.URL, &e.Title, &author, &e.Content, &rawContent, &e.PublishedAt, &e.IsRead, &e.IsStarred, &shareUUID, &metaStr, &e.CreatedAt, - ); err != nil { + ) + if err != nil { return nil, 0, err } if author.Valid { @@ -675,6 +755,232 @@ func (d *DB) MarkAllRead(feedID, categoryID *string) error { return err } +// Entry Variants CRUD +func (d *DB) GetVariant(entryID, key string) (*models.EntryVariant, error) { + query := ` + SELECT entry_id, variant_key, content_type, COALESCE(text_value, ''), COALESCE(blob_path, ''), metadata, created_at, updated_at + FROM entry_variants + WHERE entry_id = ? AND variant_key = ? + ` + var v models.EntryVariant + var metaStr sql.NullString + err := d.QueryRow(query, entryID, key).Scan( + &v.EntryID, &v.VariantKey, &v.ContentType, &v.TextValue, &v.BlobPath, &metaStr, &v.CreatedAt, &v.UpdatedAt, + ) + if err != nil { + return nil, err + } + v.Metadata = make(map[string]interface{}) + if metaStr.Valid && metaStr.String != "" { + _ = json.Unmarshal([]byte(metaStr.String), &v.Metadata) + } + return &v, nil +} + +func (d *DB) ListVariantsForEntry(entryID string) (map[string]models.EntryVariant, error) { + query := ` + SELECT entry_id, variant_key, content_type, COALESCE(text_value, ''), COALESCE(blob_path, ''), metadata, created_at, updated_at + FROM entry_variants + WHERE entry_id = ? + ` + rows, err := d.Query(query, entryID) + if err != nil { + return nil, err + } + defer func() { + _ = rows.Close() + }() + + variants := make(map[string]models.EntryVariant) + for rows.Next() { + var v models.EntryVariant + var metaStr sql.NullString + err := rows.Scan( + &v.EntryID, &v.VariantKey, &v.ContentType, &v.TextValue, &v.BlobPath, &metaStr, &v.CreatedAt, &v.UpdatedAt, + ) + if err != nil { + return nil, err + } + v.Metadata = make(map[string]interface{}) + if metaStr.Valid && metaStr.String != "" { + _ = json.Unmarshal([]byte(metaStr.String), &v.Metadata) + } + variants[v.VariantKey] = v + } + return variants, nil +} + +func (d *DB) SetVariant(v models.EntryVariant) error { + now := time.Now().UTC() + metaBytes, _ := json.Marshal(v.Metadata) + if len(metaBytes) == 0 { + metaBytes = []byte("{}") + } + + query := ` + INSERT INTO entry_variants ( + entry_id, variant_key, content_type, text_value, blob_path, metadata, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(entry_id, variant_key) DO UPDATE SET + content_type = excluded.content_type, + text_value = excluded.text_value, + blob_path = excluded.blob_path, + metadata = excluded.metadata, + updated_at = excluded.updated_at + ` + _, err := d.Exec(query, v.EntryID, v.VariantKey, v.ContentType, v.TextValue, v.BlobPath, string(metaBytes), now, now) + return err +} + +func (d *DB) DeleteVariant(entryID, key string) error { + _, err := d.Exec("DELETE FROM entry_variants WHERE entry_id = ? AND variant_key = ?", entryID, key) + return err +} + +// Rules CRUD +func (d *DB) ListRules() ([]models.Rule, error) { + query := ` + SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at + FROM rules + ORDER BY created_at ASC + ` + rows, err := d.Query(query) + if err != nil { + return nil, err + } + defer func() { + _ = rows.Close() + }() + + var rules []models.Rule + for rows.Next() { + var r models.Rule + var condsStr, actionsStr string + err := rows.Scan( + &r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt, + ) + if err != nil { + return nil, err + } + _ = json.Unmarshal([]byte(condsStr), &r.Conditions) + _ = json.Unmarshal([]byte(actionsStr), &r.Actions) + rules = append(rules, r) + } + return rules, nil +} + +func (d *DB) GetActiveRules(trigger string) ([]models.Rule, error) { + query := ` + SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at + FROM rules + WHERE is_active = 1 AND (trigger = ? OR trigger = '*') + ORDER BY created_at ASC + ` + rows, err := d.Query(query, trigger) + if err != nil { + return nil, err + } + defer func() { + _ = rows.Close() + }() + + var rules []models.Rule + for rows.Next() { + var r models.Rule + var condsStr, actionsStr string + err := rows.Scan( + &r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt, + ) + if err != nil { + return nil, err + } + _ = json.Unmarshal([]byte(condsStr), &r.Conditions) + _ = json.Unmarshal([]byte(actionsStr), &r.Actions) + rules = append(rules, r) + } + return rules, nil +} + +func (d *DB) GetRule(id string) (*models.Rule, error) { + query := ` + SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at + FROM rules + WHERE id = ? + ` + var r models.Rule + var condsStr, actionsStr string + err := d.QueryRow(query, id).Scan( + &r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt, + ) + if err != nil { + return nil, err + } + _ = json.Unmarshal([]byte(condsStr), &r.Conditions) + _ = json.Unmarshal([]byte(actionsStr), &r.Actions) + return &r, nil +} + +func (d *DB) CreateRule(name, trigger, matchMode string, conditions []models.RuleCondition, actions []models.RuleAction) (*models.Rule, error) { + id := uuid.New().String() + now := time.Now().UTC() + if matchMode == "" { + matchMode = "all" + } + condsBytes, _ := json.Marshal(conditions) + actionsBytes, _ := json.Marshal(actions) + + query := ` + INSERT INTO rules ( + id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?) + ` + _, err := d.Exec(query, id, name, trigger, string(condsBytes), matchMode, string(actionsBytes), now, now) + if err != nil { + return nil, err + } + + return &models.Rule{ + ID: id, + Name: name, + Trigger: trigger, + Conditions: conditions, + MatchMode: matchMode, + Actions: actions, + IsActive: true, + CreatedAt: now, + UpdatedAt: now, + }, nil +} + +func (d *DB) UpdateRule(id string, name, trigger, matchMode string, conditions []models.RuleCondition, actions []models.RuleAction, isActive bool) (*models.Rule, error) { + now := time.Now().UTC() + if matchMode == "" { + matchMode = "all" + } + condsBytes, _ := json.Marshal(conditions) + actionsBytes, _ := json.Marshal(actions) + + query := ` + UPDATE rules + SET name = ?, trigger = ?, conditions = ?, match_mode = ?, actions = ?, is_active = ?, updated_at = ? + WHERE id = ? + ` + res, err := d.Exec(query, name, trigger, string(condsBytes), matchMode, string(actionsBytes), isActive, now, id) + if err != nil { + return nil, err + } + if n, _ := res.RowsAffected(); n == 0 { + return nil, fmt.Errorf("rule not found") + } + + return d.GetRule(id) +} + +func (d *DB) DeleteRule(id string) error { + _, err := d.Exec("DELETE FROM rules WHERE id = ?", id) + return err +} + func (d *DB) GetStats() (*models.Stats, error) { var s models.Stats err := d.QueryRow("SELECT COUNT(*) FROM feeds").Scan(&s.TotalFeeds) @@ -693,5 +999,6 @@ func (d *DB) GetStats() (*models.Stats, error) { if err != nil { return nil, err } + _ = d.QueryRow("SELECT COUNT(*) FROM rules").Scan(&s.TotalRules) return &s, nil } diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 42a1d32..5a80541 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -72,8 +72,8 @@ func TestDatabaseOperations(t *testing.T) { if err != nil { t.Fatalf("UpsertFeedEntries failed: %v", err) } - if inserted != 2 { - t.Fatalf("expected 2 inserted entries, got %d", inserted) + if len(inserted) != 2 { + t.Fatalf("expected 2 inserted entries, got %d", len(inserted)) } // 5. List entries @@ -94,12 +94,69 @@ func TestDatabaseOperations(t *testing.T) { t.Fatalf("UpdateEntryStarred failed: %v", err) } - // 7. Verify Stats + // 7. Test Variant CRUD + variant := models.EntryVariant{ + EntryID: firstID, + VariantKey: "readability_full_text", + ContentType: "html", + TextValue: "
Clean full text content
", + Metadata: map[string]interface{}{ + "scraped_at": time.Now().UTC().Format(time.RFC3339), + "length": 100, + }, + } + if err := database.SetVariant(variant); err != nil { + t.Fatalf("SetVariant failed: %v", err) + } + fetchedVariant, err := database.GetVariant(firstID, "readability_full_text") + if err != nil { + t.Fatalf("GetVariant failed: %v", err) + } + if fetchedVariant.TextValue != "
Clean full text content
" { + t.Fatalf("expected variant text, got %s", fetchedVariant.TextValue) + } + + // Test GetEntry with variants + entryWithVariants, err := database.GetEntry(firstID) + if err != nil { + t.Fatalf("GetEntry failed: %v", err) + } + if len(entryWithVariants.Variants) != 1 { + t.Fatalf("expected 1 variant in GetEntry, got %d", len(entryWithVariants.Variants)) + } + + // 8. Test Rules CRUD + rule, err := database.CreateRule( + "Auto Scrape Rule", + "on_post_created", + "all", + []models.RuleCondition{ + {Field: "feed_id", Operator: "eq", Value: feed.ID}, + }, + []models.RuleAction{ + {Type: "readability_scrape"}, + }, + ) + if err != nil { + t.Fatalf("CreateRule failed: %v", err) + } + if rule.Name != "Auto Scrape Rule" { + t.Errorf("expected rule name Auto Scrape Rule, got %s", rule.Name) + } + activeRules, err := database.GetActiveRules("on_post_created") + if err != nil { + t.Fatalf("GetActiveRules failed: %v", err) + } + if len(activeRules) != 1 { + t.Fatalf("expected 1 active rule, got %d", len(activeRules)) + } + + // 9. Verify Stats stats, err := database.GetStats() if err != nil { t.Fatalf("GetStats failed: %v", err) } - if stats.TotalFeeds != 1 || stats.TotalUnread != 1 || stats.TotalStarred != 1 || stats.TotalCategories != 1 { + if stats.TotalFeeds != 1 || stats.TotalUnread != 1 || stats.TotalStarred != 1 || stats.TotalRules != 1 { t.Errorf("unexpected stats: %+v", stats) } diff --git a/internal/events/bus.go b/internal/events/bus.go new file mode 100644 index 0000000..d0cf321 --- /dev/null +++ b/internal/events/bus.go @@ -0,0 +1,92 @@ +package events + +import ( + "context" + "sync" + + "git.pengzhan.dev/haopengzhan/rsspipe/internal/logger" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" + "go.uber.org/zap" +) + +type EventType string + +const ( + EventPostCreated EventType = "on_post_created" + EventPostUpdated EventType = "on_post_updated" + EventVariantUpdated EventType = "on_variant_updated" + EventManualTrigger EventType = "manual" +) + +type Event struct { + Type EventType + Entry models.FeedEntry + VariantKey string // populated when Type == EventVariantUpdated + Depth int + DirectAttrs map[string]interface{} +} + +type EventHandler func(ctx context.Context, evt Event) error + +type EventBus struct { + mu sync.RWMutex + subscribers map[EventType][]EventHandler + log *zap.Logger + maxDepth int +} + +func NewEventBus() *EventBus { + return &EventBus{ + subscribers: make(map[EventType][]EventHandler), + log: logger.Named("eventbus"), + maxDepth: 5, + } +} + +func (b *EventBus) Subscribe(eventType EventType, handler EventHandler) { + b.mu.Lock() + defer b.mu.Unlock() + b.subscribers[eventType] = append(b.subscribers[eventType], handler) +} + +func (b *EventBus) Publish(ctx context.Context, evt Event) { + if evt.Depth > b.maxDepth { + b.log.Warn("Event dropped: recursion depth limit exceeded", + zap.String("type", string(evt.Type)), + zap.String("entry_id", evt.Entry.ID), + zap.Int("depth", evt.Depth), + ) + return + } + + b.mu.RLock() + handlers := append([]EventHandler{}, b.subscribers[evt.Type]...) + wildcardHandlers := append([]EventHandler{}, b.subscribers["*"]...) + b.mu.RUnlock() + + b.log.Debug("Event published", + zap.String("type", string(evt.Type)), + zap.String("entry_id", evt.Entry.ID), + zap.String("variant_key", evt.VariantKey), + zap.Int("depth", evt.Depth), + zap.Int("subscriber_count", len(handlers)+len(wildcardHandlers)), + ) + + allHandlers := append(handlers, wildcardHandlers...) + for _, h := range allHandlers { + go func(fn EventHandler) { + defer func() { + if r := recover(); r != nil { + b.log.Error("Event handler panic recovered", zap.Any("panic", r)) + } + }() + if err := fn(ctx, evt); err != nil { + b.log.Error("Event handler error", + zap.String("type", string(evt.Type)), + zap.String("entry_id", evt.Entry.ID), + zap.Error(err), + ) + } + }(h) + } +} diff --git a/internal/logger/logger.go b/internal/logger/logger.go new file mode 100644 index 0000000..bc9dacc --- /dev/null +++ b/internal/logger/logger.go @@ -0,0 +1,74 @@ +package logger + +import ( + "os" + "strings" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var globalLogger *zap.Logger +var globalSugared *zap.SugaredLogger + +func init() { + InitLogger(os.Getenv("LOG_LEVEL")) +} + +// InitLogger initializes the global Zap logger with proper formatting and log levels. +func InitLogger(level string) *zap.Logger { + zapLevel := zapcore.InfoLevel + switch strings.ToLower(level) { + case "debug": + zapLevel = zapcore.DebugLevel + case "warn", "warning": + zapLevel = zapcore.WarnLevel + case "error": + zapLevel = zapcore.ErrorLevel + } + + encoderConfig := zapcore.EncoderConfig{ + TimeKey: "time", + LevelKey: "level", + NameKey: "logger", + CallerKey: "caller", + MessageKey: "msg", + StacktraceKey: "stacktrace", + LineEnding: zapcore.DefaultLineEnding, + EncodeLevel: zapcore.CapitalColorLevelEncoder, + EncodeTime: zapcore.TimeEncoderOfLayout("15:04:05.000"), + EncodeDuration: zapcore.StringDurationEncoder, + EncodeCaller: zapcore.ShortCallerEncoder, + } + + core := zapcore.NewCore( + zapcore.NewConsoleEncoder(encoderConfig), + zapcore.AddSync(os.Stdout), + zapLevel, + ) + + globalLogger = zap.New(core, zap.AddCaller()) + globalSugared = globalLogger.Sugar() + return globalLogger +} + +// L returns the global zap logger instance. +func L() *zap.Logger { + if globalLogger == nil { + InitLogger("info") + } + return globalLogger +} + +// S returns the global sugared logger instance. +func S() *zap.SugaredLogger { + if globalSugared == nil { + InitLogger("info") + } + return globalSugared +} + +// Named creates a child logger with a component name tag. +func Named(name string) *zap.Logger { + return L().Named(name) +} diff --git a/internal/logger/logger_test.go b/internal/logger/logger_test.go new file mode 100644 index 0000000..08afa12 --- /dev/null +++ b/internal/logger/logger_test.go @@ -0,0 +1,15 @@ +package logger + +import ( + "testing" +) + +func TestLoggerInit(t *testing.T) { + log := InitLogger("debug") + if log == nil { + t.Fatal("expected non-nil logger") + } + L().Info("Test structured logging message") + S().Infof("Test sugared logging message with %s", "parameter") + Named("test-component").Debug("Test component debug log") +} diff --git a/internal/models/models.go b/internal/models/models.go index 6cf9aaf..c30eb2b 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -45,22 +45,62 @@ type Feed struct { // FeedEntry represents an individual article or digest entry type FeedEntry struct { - ID string `json:"id"` - FeedID string `json:"feed_id"` - FeedTitle string `json:"feed_title"` - FeedSiteURL string `json:"feed_site_url,omitempty"` - GUID string `json:"guid"` - URL string `json:"url"` - Title string `json:"title"` - Author string `json:"author,omitempty"` - Content string `json:"content"` - RawContent string `json:"raw_content,omitempty"` - PublishedAt time.Time `json:"published_at"` - IsRead bool `json:"is_read"` - IsStarred bool `json:"is_starred"` - ShareUUID *string `json:"share_uuid,omitempty"` - Metadata map[string]interface{} `json:"metadata"` + ID string `json:"id"` + FeedID string `json:"feed_id"` + FeedTitle string `json:"feed_title"` + FeedSiteURL string `json:"feed_site_url,omitempty"` + GUID string `json:"guid"` + URL string `json:"url"` + Title string `json:"title"` + Author string `json:"author,omitempty"` + Content string `json:"content"` // Immutable original RSS content + RawContent string `json:"raw_content,omitempty"` + PublishedAt time.Time `json:"published_at"` + IsRead bool `json:"is_read"` + IsStarred bool `json:"is_starred"` + ShareUUID *string `json:"share_uuid,omitempty"` + Variants map[string]EntryVariant `json:"variants,omitempty"` + Metadata map[string]interface{} `json:"metadata,omitempty"` + CreatedAt time.Time `json:"created_at"` +} + +// EntryVariant represents an extensible enrichment result (readability, keywords, ai_summary, tts) +type EntryVariant struct { + EntryID string `json:"entry_id"` + VariantKey string `json:"variant_key"` + ContentType string `json:"content_type"` // 'text', 'html', 'json', 'yaml', 'audio/mp3', 'binary' + TextValue string `json:"text_value,omitempty"` + BlobPath string `json:"blob_path,omitempty"` + StreamURL string `json:"stream_url,omitempty"` + Metadata map[string]interface{} `json:"metadata,omitempty"` CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// Rule represents a filter/automation hook rule +type Rule struct { + ID string `json:"id"` + Name string `json:"name"` + Trigger string `json:"trigger"` // 'on_post_created', 'on_post_updated', 'on_variant_updated', 'manual' + Conditions []RuleCondition `json:"conditions"` + MatchMode string `json:"match_mode"` // 'all' (AND) or 'any' (OR) + Actions []RuleAction `json:"actions"` + IsActive bool `json:"is_active"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// RuleCondition represents a single condition predicate +type RuleCondition struct { + Field string `json:"field"` // 'feed_id', 'title', 'author', 'variants.keywords', 'variants.readability_full_text' + Operator string `json:"operator"` // 'eq', 'neq', 'in', 'contains', 'matches' + Value interface{} `json:"value"` +} + +// RuleAction represents an action triggered by a rule +type RuleAction struct { + Type string `json:"type"` // 'readability_scrape', 'mark_read', 'mark_starred', 'extract_keywords', 'ai_summary', 'tts' + Params map[string]interface{} `json:"params,omitempty"` } // Stats represents dashboard aggregate counts @@ -69,6 +109,7 @@ type Stats struct { TotalUnread int `json:"total_unread"` TotalStarred int `json:"total_starred"` TotalCategories int `json:"total_categories"` + TotalRules int `json:"total_rules"` } // LoginRequest is the body for POST /api/auth/login diff --git a/internal/parser/parser.go b/internal/parser/parser.go index ea19aca..328cfb7 100644 --- a/internal/parser/parser.go +++ b/internal/parser/parser.go @@ -2,7 +2,9 @@ package parser import ( "context" + "encoding/json" "fmt" + "html" "io" "net/http" "strings" @@ -104,10 +106,7 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri pubDate = item.UpdatedParsed.UTC() } - content := item.Content - if strings.TrimSpace(content) == "" { - content = item.Description - } + content := sanitizeAndCombineContent(item.Content, item.Description) author := "" if item.Author != nil { @@ -120,7 +119,7 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri Title: item.Title, Author: author, Content: content, - RawContent: item.Description, + RawContent: content, PublishedAt: pubDate, Metadata: make(map[string]interface{}), } @@ -129,3 +128,96 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri return result, nil } + +type draftBlock struct { + Key string `json:"key"` + Text string `json:"text"` + Type string `json:"type"` +} + +type draftEntityData struct { + URL string `json:"url"` + Src string `json:"src"` + Caption string `json:"caption"` +} + +type draftEntity struct { + Type string `json:"type"` + Data draftEntityData `json:"data"` +} + +type draftPayload struct { + Blocks []draftBlock `json:"blocks"` + EntityMap map[string]draftEntity `json:"entityMap"` +} + +func parseDraftJSText(raw string) (string, bool) { + unescaped := html.UnescapeString(raw) + idx := strings.Index(unescaped, `{"entityMap"`) + if idx == -1 { + idx = strings.Index(unescaped, `{"blocks"`) + } + if idx == -1 { + return "", false + } + + prefix := strings.TrimSpace(unescaped[:idx]) + jsonPart := strings.TrimSpace(unescaped[idx:]) + + var payload draftPayload + if err := json.Unmarshal([]byte(jsonPart), &payload); err != nil { + return "", false + } + + if len(payload.Blocks) == 0 && len(payload.EntityMap) == 0 { + return "", false + } + + var sb strings.Builder + if prefix != "" { + lines := strings.Split(prefix, "\n") + for _, line := range lines { + l := strings.TrimSpace(line) + if l != "" { + sb.WriteString("

") + sb.WriteString(html.EscapeString(l)) + sb.WriteString("

\n") + } + } + } + + for _, block := range payload.Blocks { + txt := strings.TrimSpace(block.Text) + if txt != "" { + sb.WriteString("

") + sb.WriteString(html.EscapeString(txt)) + sb.WriteString("

\n") + } + } + + for _, ent := range payload.EntityMap { + if strings.EqualFold(ent.Type, "IMAGE") && ent.Data.Src != "" { + fmt.Fprintf(&sb, `

%s

`+"\n", html.EscapeString(ent.Data.Src), html.EscapeString(ent.Data.Caption)) + } + } + + return sb.String(), true +} + +func sanitizeAndCombineContent(content, description string) string { + if parsedDesc, ok := parseDraftJSText(description); ok { + if strings.TrimSpace(content) != "" { + return content + "
\n" + parsedDesc + } + return parsedDesc + } + + if parsedContent, ok := parseDraftJSText(content); ok { + return parsedContent + } + + if strings.TrimSpace(content) != "" { + return content + } + return description +} diff --git a/internal/parser/parser_test.go b/internal/parser/parser_test.go new file mode 100644 index 0000000..469a2c9 --- /dev/null +++ b/internal/parser/parser_test.go @@ -0,0 +1,27 @@ +package parser + +import ( + "strings" + "testing" +) + +func TestParseDraftJSText(t *testing.T) { + desc := `就这样吧评论: 奥德赛 (https://movie.douban.com/subject/36808876/) +评价: 推荐 + +{"entityMap": {}, "blocks": [{"key": "1", "text": "终于看完了奥德赛,非常精彩!", "type": "unstyled"}]}` + + content := `就这样吧评论: 奥德赛
评价: 推荐` + + combined := sanitizeAndCombineContent(content, desc) + + if !strings.Contains(combined, "终于看完了奥德赛,非常精彩!") { + t.Fatalf("expected parsed text in output, got: %s", combined) + } + if !strings.Contains(combined, "

终于看完了奥德赛,非常精彩!

") { + t.Fatalf("expected formatted paragraph in output, got: %s", combined) + } + if !strings.Contains(combined, "img9.doubanio.com/poster.jpg") { + t.Fatalf("expected poster in output, got: %s", combined) + } +} diff --git a/internal/rules/engine.go b/internal/rules/engine.go new file mode 100644 index 0000000..cb973e4 --- /dev/null +++ b/internal/rules/engine.go @@ -0,0 +1,82 @@ +package rules + +import ( + "context" + + "git.pengzhan.dev/haopengzhan/rsspipe/internal/db" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/events" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/logger" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" + "go.uber.org/zap" +) + +type WorkerDispatcher interface { + Dispatch(ctx context.Context, action models.RuleAction, entry models.FeedEntry, depth int) +} + +type Engine struct { + db *db.DB + bus *events.EventBus + evaluator *Evaluator + dispatcher WorkerDispatcher + log *zap.Logger +} + +func NewEngine(database *db.DB, bus *events.EventBus, dispatcher WorkerDispatcher) *Engine { + return &Engine{ + db: database, + bus: bus, + evaluator: NewEvaluator(), + dispatcher: dispatcher, + log: logger.Named("rules-engine"), + } +} + +// Start registers event bus subscriptions to evaluate rules reactively. +func (e *Engine) Start(ctx context.Context) { + e.bus.Subscribe(events.EventPostCreated, e.handleEvent) + e.bus.Subscribe(events.EventPostUpdated, e.handleEvent) + e.bus.Subscribe(events.EventVariantUpdated, e.handleEvent) + e.bus.Subscribe(events.EventManualTrigger, e.handleEvent) + e.log.Info("Rules engine started, listening for events") +} + +func (e *Engine) handleEvent(ctx context.Context, evt events.Event) error { + trigger := string(evt.Type) + rules, err := e.db.GetActiveRules(trigger) + if err != nil { + e.log.Error("Failed to fetch active rules", zap.String("trigger", trigger), zap.Error(err)) + return err + } + + if len(rules) == 0 { + return nil + } + + // Fetch full entry with latest variants if entry ID is present + entry := evt.Entry + if entry.ID != "" { + if full, err := e.db.GetEntry(entry.ID); err == nil && full != nil { + entry = *full + } + } + + for _, r := range rules { + if e.evaluator.Evaluate(r, entry) { + e.log.Info("Rule matched", + zap.String("rule_id", r.ID), + zap.String("rule_name", r.Name), + zap.String("entry_id", entry.ID), + zap.Int("actions_count", len(r.Actions)), + ) + + for _, action := range r.Actions { + if e.dispatcher != nil { + e.dispatcher.Dispatch(ctx, action, entry, evt.Depth+1) + } + } + } + } + + return nil +} diff --git a/internal/rules/evaluator.go b/internal/rules/evaluator.go new file mode 100644 index 0000000..8fbc78e --- /dev/null +++ b/internal/rules/evaluator.go @@ -0,0 +1,133 @@ +package rules + +import ( + "fmt" + "regexp" + "strings" + + "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" +) + +type Evaluator struct{} + +func NewEvaluator() *Evaluator { + return &Evaluator{} +} + +// Evaluate checks whether a FeedEntry satisfies a Rule's conditions. +func (e *Evaluator) Evaluate(rule models.Rule, entry models.FeedEntry) bool { + if len(rule.Conditions) == 0 { + return true + } + + matchMode := strings.ToLower(rule.MatchMode) + if matchMode == "" || matchMode == "all" { + for _, cond := range rule.Conditions { + if !e.evaluateCondition(cond, entry) { + return false + } + } + return true + } + + // matchMode == "any" + for _, cond := range rule.Conditions { + if e.evaluateCondition(cond, entry) { + return true + } + } + return false +} + +func (e *Evaluator) evaluateCondition(cond models.RuleCondition, entry models.FeedEntry) bool { + actualValue := e.resolveField(cond.Field, entry) + return e.compare(actualValue, cond.Operator, cond.Value) +} + +func (e *Evaluator) resolveField(field string, entry models.FeedEntry) interface{} { + switch strings.ToLower(field) { + case "feed_id": + return entry.FeedID + case "feed_title": + return entry.FeedTitle + case "title": + return entry.Title + case "author": + return entry.Author + case "url": + return entry.URL + case "content", "rss_content": + return entry.Content + case "is_read": + return entry.IsRead + case "is_starred": + return entry.IsStarred + } + + // Variant field resolution: "variants.keywords", "variants.readability_full_text", "variants.tags" + if strings.HasPrefix(field, "variants.") { + variantKey := strings.TrimPrefix(field, "variants.") + if entry.Variants != nil { + if v, ok := entry.Variants[variantKey]; ok { + return v.TextValue + } + } + } + + return nil +} + +func (e *Evaluator) compare(actual interface{}, op string, expected interface{}) bool { + if actual == nil { + return op == "neq" + } + + actualStr := fmt.Sprintf("%v", actual) + expectedStr := fmt.Sprintf("%v", expected) + + switch strings.ToLower(op) { + case "eq", "==": + return strings.EqualFold(actualStr, expectedStr) + case "neq", "!=": + return !strings.EqualFold(actualStr, expectedStr) + case "contains": + return strings.Contains(strings.ToLower(actualStr), strings.ToLower(expectedStr)) + case "not_contains": + return !strings.Contains(strings.ToLower(actualStr), strings.ToLower(expectedStr)) + case "in": + // expected can be []interface{}, []string, or comma-separated string + switch v := expected.(type) { + case []interface{}: + for _, item := range v { + if strings.EqualFold(actualStr, fmt.Sprintf("%v", item)) { + return true + } + } + return false + case []string: + for _, item := range v { + if strings.EqualFold(actualStr, item) { + return true + } + } + return false + case string: + items := strings.Split(v, ",") + for _, item := range items { + if strings.EqualFold(strings.TrimSpace(actualStr), strings.TrimSpace(item)) { + return true + } + } + return false + } + return false + case "matches", "regex": + re, err := regexp.Compile(expectedStr) + if err != nil { + return false + } + return re.MatchString(actualStr) + default: + return false + } +} diff --git a/internal/rules/rules_test.go b/internal/rules/rules_test.go new file mode 100644 index 0000000..ff4ee4b --- /dev/null +++ b/internal/rules/rules_test.go @@ -0,0 +1,70 @@ +package rules + +import ( + "testing" + "time" + + "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" +) + +func TestRuleEvaluator(t *testing.T) { + eval := NewEvaluator() + + entry := models.FeedEntry{ + ID: "entry-1", + FeedID: "feed-hn", + FeedTitle: "Hacker News", + Title: "Announcing Go 1.24 Release", + Author: "Go Team", + Content: "

Go 1.24 brings new features...

", + Variants: map[string]models.EntryVariant{ + "keywords": { + EntryID: "entry-1", + VariantKey: "keywords", + ContentType: "json", + TextValue: `["go", "compiler", "release"]`, + }, + }, + PublishedAt: time.Now(), + } + + // Test 1: Title contains Go AND FeedID == feed-hn + rule1 := models.Rule{ + ID: "r1", + Name: "Tech Feeds", + MatchMode: "all", + Conditions: []models.RuleCondition{ + {Field: "feed_id", Operator: "eq", Value: "feed-hn"}, + {Field: "title", Operator: "contains", Value: "Go 1.24"}, + }, + } + if !eval.Evaluate(rule1, entry) { + t.Error("expected rule1 to match entry") + } + + // Test 2: Negative condition (author != Python Team) + rule2 := models.Rule{ + ID: "r2", + Name: "Author check", + MatchMode: "all", + Conditions: []models.RuleCondition{ + {Field: "author", Operator: "neq", Value: "Python Team"}, + }, + } + if !eval.Evaluate(rule2, entry) { + t.Error("expected rule2 to match entry") + } + + // Test 3: Variant matching (variants.keywords contains compiler) + rule3 := models.Rule{ + ID: "r3", + Name: "Keywords check", + MatchMode: "all", + Conditions: []models.RuleCondition{ + {Field: "variants.keywords", Operator: "contains", Value: "compiler"}, + }, + } + if !eval.Evaluate(rule3, entry) { + t.Error("expected rule3 to match entry variant") + } +} diff --git a/internal/worker/pool.go b/internal/worker/pool.go new file mode 100644 index 0000000..64b7502 --- /dev/null +++ b/internal/worker/pool.go @@ -0,0 +1,179 @@ +package worker + +import ( + "context" + "fmt" + "sync" + "time" + + "git.pengzhan.dev/haopengzhan/rsspipe/internal/db" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/events" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/logger" + "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" + "go.uber.org/zap" +) + +type ActionResult struct { + ResultVariant *models.EntryVariant + MarkRead *bool + MarkStarred *bool +} + +type ActionHandler func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*ActionResult, error) + +type Task struct { + Action models.RuleAction + Entry models.FeedEntry + Depth int +} + +type Pool struct { + db *db.DB + bus *events.EventBus + handlers map[string]ActionHandler + queue chan Task + workers int + wg sync.WaitGroup + log *zap.Logger + mu sync.RWMutex +} + +func NewPool(database *db.DB, bus *events.EventBus, workers, queueSize int) *Pool { + if workers <= 0 { + workers = 5 + } + if queueSize <= 0 { + queueSize = 100 + } + return &Pool{ + db: database, + bus: bus, + handlers: make(map[string]ActionHandler), + queue: make(chan Task, queueSize), + workers: workers, + log: logger.Named("worker-pool"), + } +} + +func (p *Pool) RegisterHandler(actionType string, handler ActionHandler) { + p.mu.Lock() + defer p.mu.Unlock() + p.handlers[actionType] = handler + p.log.Info("Registered action handler", zap.String("action", actionType)) +} + +func (p *Pool) Start(ctx context.Context) { + p.log.Info("Starting worker pool", zap.Int("worker_count", p.workers)) + for i := 0; i < p.workers; i++ { + p.wg.Add(1) + go p.worker(ctx, i) + } +} + +func (p *Pool) Stop() { + close(p.queue) + p.wg.Wait() + p.log.Info("Worker pool stopped") +} + +func (p *Pool) Dispatch(ctx context.Context, action models.RuleAction, entry models.FeedEntry, depth int) { + select { + case p.queue <- Task{Action: action, Entry: entry, Depth: depth}: + default: + p.log.Warn("Worker pool queue full, task dropped", + zap.String("action", action.Type), + zap.String("entry_id", entry.ID), + ) + } +} + +func (p *Pool) ExecuteDirect(ctx context.Context, actionType string, entry models.FeedEntry, params map[string]interface{}) (*ActionResult, error) { + p.mu.RLock() + handler, ok := p.handlers[actionType] + p.mu.RUnlock() + + if !ok { + return nil, fmt.Errorf("unknown action handler: %s", actionType) + } + + return handler(ctx, entry, params) +} + +func (p *Pool) worker(ctx context.Context, id int) { + defer p.wg.Done() + for { + select { + case <-ctx.Done(): + return + case task, ok := <-p.queue: + if !ok { + return + } + p.processTask(ctx, task) + } + } +} + +func (p *Pool) processTask(ctx context.Context, task Task) { + p.mu.RLock() + handler, ok := p.handlers[task.Action.Type] + p.mu.RUnlock() + + if !ok { + p.log.Warn("No handler registered for action", zap.String("type", task.Action.Type)) + return + } + + start := time.Now() + res, err := handler(ctx, task.Entry, task.Action.Params) + duration := time.Since(start) + + if err != nil { + p.log.Error("Action execution failed", + zap.String("action", task.Action.Type), + zap.String("entry_id", task.Entry.ID), + zap.Duration("duration", duration), + zap.Error(err), + ) + return + } + + if res == nil { + return + } + + // 1. Save Variant if returned + if res.ResultVariant != nil { + if err := p.db.SetVariant(*res.ResultVariant); err != nil { + p.log.Error("Failed to save entry variant", + zap.String("entry_id", task.Entry.ID), + zap.String("variant_key", res.ResultVariant.VariantKey), + zap.Error(err), + ) + } else { + p.log.Info("Entry variant saved", + zap.String("entry_id", task.Entry.ID), + zap.String("variant_key", res.ResultVariant.VariantKey), + zap.Duration("duration", duration), + ) + + // Publish EventVariantUpdated to enable downstream chained triggers + if p.bus != nil { + p.bus.Publish(ctx, events.Event{ + Type: events.EventVariantUpdated, + Entry: task.Entry, + VariantKey: res.ResultVariant.VariantKey, + Depth: task.Depth, + }) + } + } + } + + // 2. Apply Read/Star updates + if res.MarkRead != nil { + _ = p.db.UpdateEntryRead(task.Entry.ID, *res.MarkRead) + } + if res.MarkStarred != nil { + _ = p.db.UpdateEntryStarred(task.Entry.ID, *res.MarkStarred) + } +} diff --git a/web/src/App.tsx b/web/src/App.tsx index 5f75239..bfc4c52 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -159,6 +159,19 @@ export const App: React.FC = () => { setFocusedIndex(entries.findIndex((e) => e.id === selectedEntryId)); return; } + // If not found in current list, load directly so reading pane stays populated + api.getEntry(selectedEntryId).then((entry) => { + setActiveArticle(entry); + }).catch(() => { + // fallback to top + if (entries[0]) { + setFocusedIndex(0); + setActiveArticle(entries[0]); + setSelectedEntryId(entries[0].id); + localStorage.setItem('rsspipe_selected_entry_id', entries[0].id); + } + }); + return; } setFocusedIndex(0); @@ -177,36 +190,7 @@ export const App: React.FC = () => { } }; - // Auto-Refresh & Background Polling (Every 30s + on focus / visibility change) - useEffect(() => { - if (!currentUser) return; - - const interval = setInterval(() => { - loadSidebarData(); - loadEntries(); - }, 30000); - - const handleWindowFocus = () => { - loadSidebarData(); - loadEntries(); - }; - - const handleVisibilityChange = () => { - if (document.visibilityState === 'visible') { - loadSidebarData(); - loadEntries(); - } - }; - - window.addEventListener('focus', handleWindowFocus); - document.addEventListener('visibilitychange', handleVisibilityChange); - - return () => { - clearInterval(interval); - window.removeEventListener('focus', handleWindowFocus); - document.removeEventListener('visibilitychange', handleVisibilityChange); - }; - }, [currentUser, loadSidebarData, loadEntries]); + // Refresh data only on initial mount or when user clicks refresh / changes filter // Toggle Read Action with optimistic update const handleToggleRead = async (entry: FeedEntry, e?: React.MouseEvent) => { @@ -516,6 +500,10 @@ export const App: React.FC = () => { onToggleRead={handleToggleRead} onToggleStar={handleToggleStar} onMarkAllRead={handleMarkAllRead} + onUpdateEntry={(updated) => { + setActiveArticle(updated); + setEntries((prev) => prev.map((e) => (e.id === updated.id ? updated : e))); + }} sidebarCollapsed={sidebarCollapsed} /> ) : viewMode === 'waterfall' ? ( @@ -526,6 +514,10 @@ export const App: React.FC = () => { entries={entries} onToggleRead={handleToggleRead} onToggleStar={handleToggleStar} + onUpdateEntry={(updated) => { + setActiveArticle(updated); + setEntries((prev) => prev.map((e) => (e.id === updated.id ? updated : e))); + }} /> ) : ( = ({ onRefreshData, onOpenAddFeed, }) => { - const [activeTab, setActiveTab] = useState<'feeds' | 'categories' | 'viewer' | 'opml' | 'account'>('feeds'); + const [activeTab, setActiveTab] = useState<'feeds' | 'categories' | 'rules' | 'viewer' | 'opml' | 'account'>('feeds'); const [autoReadBottom, setAutoReadBottom] = useState(() => { return localStorage.getItem('rsspipe_auto_read_bottom') !== 'false'; }); @@ -50,6 +52,18 @@ export const SettingsView: React.FC = ({ const [loading, setLoading] = useState(false); const [message, setMessage] = useState(null); + // Rules State + const [rules, setRules] = useState([]); + const [rulesLoading, setRulesLoading] = useState(false); + const [showRuleModal, setShowRuleModal] = useState(false); + const [ruleName, setRuleName] = useState(''); + const [ruleTrigger, setRuleTrigger] = useState('on_post_created'); + const [ruleMatchMode, setRuleMatchMode] = useState('all'); + const [ruleField, setRuleField] = useState('feed_id'); + const [ruleOp, setRuleOp] = useState('eq'); + const [ruleVal, setRuleVal] = useState(''); + const [ruleActionType, setRuleActionType] = useState('readability_scrape'); + // Edit Feed Modal State const [editingFeed, setEditingFeed] = useState(null); const [editTitle, setEditTitle] = useState(''); @@ -110,6 +124,78 @@ export const SettingsView: React.FC = ({ } }; + useEffect(() => { + loadRules(); + }, []); + + const loadRules = async () => { + setRulesLoading(true); + try { + const data = await api.listRules(); + setRules(data || []); + } catch { + // ignore + } finally { + setRulesLoading(false); + } + }; + + const handleCreateRule = async (e: React.FormEvent) => { + e.preventDefault(); + if (!ruleName.trim()) return; + try { + await api.createRule({ + name: ruleName.trim(), + trigger: ruleTrigger, + match_mode: ruleMatchMode, + conditions: [ + { + field: ruleField, + operator: ruleOp, + value: ruleVal.trim(), + }, + ], + actions: [ + { + type: ruleActionType, + }, + ], + }); + setShowRuleModal(false); + setRuleName(''); + setRuleVal(''); + loadRules(); + } catch (err: any) { + alert(err.message || 'Failed to create rule'); + } + }; + + const handleToggleRuleActive = async (rule: Rule) => { + try { + await api.updateRule(rule.id, { + name: rule.name, + trigger: rule.trigger, + match_mode: rule.match_mode, + conditions: rule.conditions, + actions: rule.actions, + is_active: !rule.is_active, + }); + loadRules(); + } catch (err: any) { + alert(err.message || 'Failed to toggle rule'); + } + }; + + const handleDeleteRule = async (id: string) => { + if (!confirm('Are you sure you want to delete this rule?')) return; + try { + await api.deleteRule(id); + loadRules(); + } catch (err: any) { + alert(err.message || 'Failed to delete rule'); + } + }; + // Filter feeds const filteredFeeds = feeds.filter( (f) => @@ -148,14 +234,20 @@ export const SettingsView: React.FC = ({ } }; - const handleRefreshFeed = async (feedId: string) => { + const handleCreateCategory = async (e: React.FormEvent) => { + e.preventDefault(); + if (!newCatTitle.trim()) return; + setLoading(true); try { - await api.refreshFeed(feedId); + await api.createCategory(newCatTitle.trim()); + setNewCatTitle(''); onRefreshData(); - setMessage('Feed refreshed successfully.'); + setMessage(`Category "${newCatTitle}" created.`); setTimeout(() => setMessage(null), 3000); } catch (err: any) { - alert(err.message || 'Failed to refresh feed'); + alert(err.message || 'Failed to create category'); + } finally { + setLoading(false); } }; @@ -170,28 +262,22 @@ export const SettingsView: React.FC = ({ setMessage(`Deleted category "${deleteConfirm.title}".`); } onRefreshData(); - setTimeout(() => setMessage(null), 3500); + setTimeout(() => setMessage(null), 3000); } catch (err: any) { - alert(err.message || 'Delete operation failed'); + alert(err.message || 'Failed to delete'); } finally { setDeleteConfirm(null); } }; - const handleCreateCategory = async (e: React.FormEvent) => { - e.preventDefault(); - if (!newCatTitle.trim()) return; - setLoading(true); + const handleRefreshFeed = async (feedId: string) => { try { - await api.createCategory(newCatTitle.trim()); - setNewCatTitle(''); + await api.refreshFeed(feedId); onRefreshData(); - setMessage('Category created.'); - setTimeout(() => setMessage(null), 3000); + setMessage('Feed refreshed successfully.'); + setTimeout(() => setMessage(null), 2500); } catch (err: any) { - alert(err.message || 'Failed to create category'); - } finally { - setLoading(false); + alert(err.message || 'Failed to refresh feed'); } }; @@ -234,6 +320,14 @@ export const SettingsView: React.FC = ({ Categories ({categories.length}) + + + + + {rulesLoading ? ( +
+ +

Loading rules...

+
+ ) : rules.length === 0 ? ( +
+
⚡
+

No Automation Rules Defined

+

+ Create rules to automatically scrape full-text content or apply tags whenever new articles arrive from specific feeds. +

+ +
+ ) : ( +
+ {rules.map((rule) => ( +
+
+
+
+ +
+
+
+ {rule.name} + + Trigger: {rule.trigger} + + {!rule.is_active && ( + (Inactive) + )} +
+ +
+ {rule.conditions && rule.conditions.length > 0 ? ( + rule.conditions.map((c, i) => ( + + If {c.field} {c.operator} "{String(c.value)}" + + )) + ) : ( + Always matches + )} + ➜ + {rule.actions?.map((a, i) => ( + + Action: {a.type} + + ))} +
+
+
+ +
+ + +
+
+
+ ))} +
+ )} + + )} + {/* 3. Viewer Preferences Tab */} {activeTab === 'viewer' && (
@@ -975,6 +1181,144 @@ export const SettingsView: React.FC = ({
)} + + {/* Create Automation Rule Modal */} + {showRuleModal && ( +
setShowRuleModal(false)}> +
e.stopPropagation()}> +
+
+ +

Create Automation / Hook Rule

+
+ +
+ +
+
+
+ + setRuleName(e.target.value)} + required + /> +
+ +
+
+ + +
+ +
+ + +
+
+ +
+ +
+ + + +
+ + {ruleField === 'feed_id' ? ( + + ) : ( + setRuleVal(e.target.value)} + required + /> + )} +
+ +
+ + +
+
+ +
+ + +
+
+
+
+ )} ); }; diff --git a/web/src/components/ThreePaneView.tsx b/web/src/components/ThreePaneView.tsx index 79cf64e..6e456f2 100644 --- a/web/src/components/ThreePaneView.tsx +++ b/web/src/components/ThreePaneView.tsx @@ -1,7 +1,8 @@ -import React, { useEffect, useRef } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import { Category, Feed, FeedEntry, FilterState, Stats } from '../types'; import { Sidebar } from './Sidebar'; import { formatRelativeDate, stripHtml, estimateReadingTime } from '../utils'; +import { api } from '../services/api'; import { Star, CheckCircle2, @@ -10,6 +11,11 @@ import { Clock, User as UserIcon, CheckCheck, + FileText, + Zap, + Sparkles, + Volume2, + Loader2, } from 'lucide-react'; interface ThreePaneViewProps { @@ -25,6 +31,7 @@ interface ThreePaneViewProps { onToggleRead: (entry: FeedEntry, e?: React.MouseEvent) => void; onToggleStar: (entry: FeedEntry, e?: React.MouseEvent) => void; onMarkAllRead: () => void; + onUpdateEntry?: (entry: FeedEntry) => void; sidebarCollapsed: boolean; } @@ -41,10 +48,13 @@ export const ThreePaneView: React.FC = ({ onToggleRead, onToggleStar, onMarkAllRead, + onUpdateEntry, sidebarCollapsed, }) => { const cardRefs = useRef<(HTMLDivElement | null)[]>([]); const readerPaneRef = useRef(null); + const [selectedVariantKey, setSelectedVariantKey] = useState('original'); + const [isScraping, setIsScraping] = useState(false); const selectedEntry = activeArticle || entries[selectedIndex] || entries[0]; @@ -58,6 +68,13 @@ export const ThreePaneView: React.FC = ({ } }, [selectedIndex, entries.length]); + // Reset variant selection when article changes + useEffect(() => { + if (selectedEntry?.variants?.['readability_full_text']) { + // If full text is available, you can choose to stay on current or default to original + } + }, [selectedEntry?.id]); + // When switching selected article, reset reader scroll position to top and handle 2s timer for short content useEffect(() => { if (readerPaneRef.current) { @@ -108,6 +125,22 @@ export const ThreePaneView: React.FC = ({ } }; + const handleScrapeFullText = async () => { + if (!selectedEntry || isScraping) return; + setIsScraping(true); + try { + const updated = await api.scrapeEntry(selectedEntry.id); + if (onUpdateEntry) { + onUpdateEntry(updated); + } + setSelectedVariantKey('readability_full_text'); + } catch (err: any) { + alert(`Scraping failed: ${err.message || err}`); + } finally { + setIsScraping(false); + } + }; + const getListTitle = () => { if (filter.search) return `Search: "${filter.search}"`; if (filter.feedId) { @@ -123,6 +156,21 @@ export const ThreePaneView: React.FC = ({ return 'Inbox'; }; + const hasReadability = Boolean(selectedEntry?.variants?.['readability_full_text']); + const hasAISummary = Boolean(selectedEntry?.variants?.['ai_summary']); + const hasTTS = Boolean(selectedEntry?.variants?.['tts'] || selectedEntry?.variants?.['podcast_tts']); + + const getRenderedContent = () => { + if (!selectedEntry) return ''; + if (selectedVariantKey === 'readability_full_text' && selectedEntry.variants?.['readability_full_text']) { + return selectedEntry.variants['readability_full_text'].text_value || ''; + } + if (selectedVariantKey === 'ai_summary' && selectedEntry.variants?.['ai_summary']) { + return `
AI Summary:

${selectedEntry.variants['ai_summary'].text_value}

` + (selectedEntry.content || selectedEntry.raw_content || ''); + } + return selectedEntry.content || selectedEntry.raw_content || ''; + }; + return (
{/* Pane 1: Sidebar (Collapsible & Independently Scrollable) */} @@ -235,6 +283,96 @@ export const ThreePaneView: React.FC = ({ > + + {/* Sliding Ball Capsule Toggle Button */} + + + {hasAISummary && ( + + )} + + {hasTTS && ( + + )}
@@ -295,10 +433,22 @@ export const ThreePaneView: React.FC = ({ + {/* Audio player if TTS variant is selected */} + {selectedVariantKey === 'tts' && ( +
+

🎧 Audio Stream:

+
+ )} +
@@ -314,3 +464,4 @@ export const ThreePaneView: React.FC = ({
); }; + diff --git a/web/src/components/WaterfallView.tsx b/web/src/components/WaterfallView.tsx index 93d3de2..f42295d 100644 --- a/web/src/components/WaterfallView.tsx +++ b/web/src/components/WaterfallView.tsx @@ -1,4 +1,5 @@ -import React, { useState, useRef } from 'react'; +import React, { useState, useEffect, useRef } from 'react'; +import { api } from '../services/api'; import { Category, FeedEntry, FilterState } from '../types'; import { extractFirstImage, formatRelativeDate, stripHtml, estimateReadingTime } from '../utils'; import { @@ -13,6 +14,11 @@ import { CheckCircle2, Clock, User as UserIcon, + FileText, + Zap, + Loader2, + Sparkles, + Volume2, } from 'lucide-react'; interface WaterfallViewProps { @@ -22,6 +28,7 @@ interface WaterfallViewProps { entries: FeedEntry[]; onToggleRead: (entry: FeedEntry, e?: React.MouseEvent) => void; onToggleStar: (entry: FeedEntry, e?: React.MouseEvent) => void; + onUpdateEntry?: (entry: FeedEntry) => void; } export const WaterfallView: React.FC = ({ @@ -31,10 +38,51 @@ export const WaterfallView: React.FC = ({ entries, onToggleRead, onToggleStar, + onUpdateEntry, }) => { const [selectedEntry, setSelectedEntry] = useState(null); + const [selectedVariantKey, setSelectedVariantKey] = useState('original'); + const [isScraping, setIsScraping] = useState(false); const readerScrollRef = useRef(null); + // Reset variant selection when selected article changes + useEffect(() => { + setSelectedVariantKey('original'); + }, [selectedEntry?.id]); + + const handleScrapeFullText = async () => { + if (!selectedEntry || isScraping) return; + setIsScraping(true); + try { + const updated = await api.scrapeEntry(selectedEntry.id); + setSelectedEntry(updated); + onUpdateEntry?.(updated); + setSelectedVariantKey('readability_full_text'); + } catch (err: any) { + alert(`Scraping failed: ${err.message || err}`); + } finally { + setIsScraping(false); + } + }; + + const hasReadability = Boolean(selectedEntry?.variants?.['readability_full_text']); + const hasAISummary = Boolean(selectedEntry?.variants?.['ai_summary']); + const hasTTS = Boolean(selectedEntry?.variants?.['tts'] || selectedEntry?.variants?.['podcast_tts']); + + const getRenderedContent = () => { + if (!selectedEntry) return ''; + if (selectedVariantKey === 'readability_full_text' && selectedEntry.variants?.['readability_full_text']) { + return selectedEntry.variants['readability_full_text'].text_value || ''; + } + if (selectedVariantKey === 'ai_summary' && selectedEntry.variants?.['ai_summary']) { + return ( + `
AI Summary:

${selectedEntry.variants['ai_summary'].text_value}

` + + (selectedEntry.content || selectedEntry.raw_content || '') + ); + } + return selectedEntry.content || selectedEntry.raw_content || ''; + }; + // When card is clicked, open in full-screen reader view and immediately mark as read const handleCardClick = (entry: FeedEntry) => { setSelectedEntry(entry); @@ -232,7 +280,95 @@ export const WaterfallView: React.FC = ({ {selectedEntry.feed_title} -
+
+ {/* Sliding Ball Capsule Toggle Button */} + + + {hasAISummary && ( + + )} + + {hasTTS && ( + + )} +
diff --git a/web/src/index.css b/web/src/index.css index da22ad7..c96e3fb 100644 --- a/web/src/index.css +++ b/web/src/index.css @@ -560,8 +560,8 @@ html, body, #root { .reader-toolbar { position: sticky; top: 0; - height: 44px; - min-height: 44px; + height: 46px; + min-height: 46px; background: rgba(255, 255, 255, 0.95); backdrop-filter: blur(8px); border-bottom: 1px solid var(--border); @@ -572,6 +572,107 @@ html, body, #root { z-index: 10; } +/* Sliding Ball Capsule Toggle Switch */ +.capsule-switch-btn { + display: inline-flex; + align-items: center; + border: none; + background: transparent; + padding: 0; + cursor: pointer; + outline: none; + user-select: none; + margin-left: 0.5rem; +} + +.capsule-switch-track { + position: relative; + display: flex; + align-items: center; + width: 86px; + height: 26px; + border-radius: 9999px; + background-color: var(--bg-card-hover); + border: 1px solid var(--border-dark); + padding: 2px; + transition: background-color 0.25s cubic-bezier(0.4, 0, 0.2, 1), + border-color 0.25s cubic-bezier(0.4, 0, 0.2, 1), + box-shadow 0.25s ease; + box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.05); +} + +.capsule-switch-btn:hover .capsule-switch-track { + border-color: var(--border-focus); +} + +.capsule-switch-btn.active .capsule-switch-track { + background-color: #0284c7; + border-color: #0284c7; +} + +.capsule-switch-btn.scraping .capsule-switch-track { + background-color: #0369a1; + border-color: #0369a1; + cursor: wait; +} + +.capsule-switch-thumb { + position: absolute; + top: 2px; + left: 2px; + width: 20px; + height: 20px; + border-radius: 50%; + background-color: #ffffff; + color: #64748b; + display: flex; + align-items: center; + justify-content: center; + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.1); + transition: transform 0.25s cubic-bezier(0.4, 0, 0.2, 1), + color 0.25s ease; + z-index: 2; +} + +.capsule-switch-btn.active .capsule-switch-thumb { + transform: translateX(60px); + color: #0284c7; +} + +.capsule-switch-label { + width: 100%; + font-size: 0.69rem; + font-weight: 600; + text-align: center; + transition: color 0.25s ease, padding 0.25s ease; + z-index: 1; + white-space: nowrap; + letter-spacing: -0.01em; +} + +/* When thumb is on the left, label is padded to the right */ +.capsule-switch-btn:not(.active) .capsule-switch-label { + padding-left: 17px; + padding-right: 2px; + color: var(--text-main); +} + +/* When thumb is on the right, label is padded to the left */ +.capsule-switch-btn.active .capsule-switch-label { + padding-right: 17px; + padding-left: 2px; + color: #ffffff; +} + +@keyframes spin { + from { transform: rotate(0deg); } + to { transform: rotate(360deg); } +} + +.spin { + animation: spin 1s linear infinite; +} + .reader-content-canvas { max-width: 680px; width: 100%; diff --git a/web/src/services/api.ts b/web/src/services/api.ts index 122889f..a85dd65 100644 --- a/web/src/services/api.ts +++ b/web/src/services/api.ts @@ -172,6 +172,45 @@ export const api = { }); }, + async getEntry(id: string): Promise { + return request(`/entries/${id}`); + }, + + async getVariants(id: string): Promise> { + return request>(`/entries/${id}/variants`); + }, + + async scrapeEntry(id: string): Promise { + return request(`/entries/${id}/scrape`, { + method: 'POST', + }); + }, + + // Rules + async listRules(): Promise { + return request('/rules'); + }, + + async createRule(rule: any): Promise { + return request('/rules', { + method: 'POST', + body: JSON.stringify(rule), + }); + }, + + async updateRule(id: string, rule: any): Promise { + return request(`/rules/${id}`, { + method: 'PUT', + body: JSON.stringify(rule), + }); + }, + + async deleteRule(id: string): Promise { + return request(`/rules/${id}`, { + method: 'DELETE', + }); + }, + // OPML async importOPML(file: File): Promise<{ imported_count: number; total_parsed: number }> { const token = localStorage.getItem('rsspipe_jwt'); @@ -200,3 +239,4 @@ export const api = { return `${API_BASE}/opml/export?token=${token || ''}`; } }; + diff --git a/web/src/types.ts b/web/src/types.ts index c3b3336..b31f4a2 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -32,6 +32,18 @@ export interface Feed { created_at: string; } +export interface EntryVariant { + entry_id: string; + variant_key: string; + content_type: string; // 'text', 'html', 'json', 'yaml', 'audio/mp3', 'binary' + text_value?: string; + blob_path?: string; + stream_url?: string; + metadata?: Record; + created_at: string; + updated_at: string; +} + export interface FeedEntry { id: string; feed_id: string; @@ -41,21 +53,46 @@ export interface FeedEntry { url: string; title: string; author?: string; - content: string; + content: string; // Immutable original RSS content raw_content?: string; published_at: string; is_read: boolean; is_starred: boolean; share_uuid?: string; - metadata: Record; + variants?: Record; + metadata?: Record; created_at: string; } +export interface RuleCondition { + field: string; + operator: string; + value: any; +} + +export interface RuleAction { + type: string; + params?: Record; +} + +export interface Rule { + id: string; + name: string; + trigger: string; + conditions: RuleCondition[]; + match_mode: string; + actions: RuleAction[]; + is_active: boolean; + created_at: string; + updated_at: string; +} + export interface Stats { total_feeds: number; total_unread: number; total_starred: number; total_categories: number; + total_rules?: number; } export type ViewType = 'inbox' | 'starred' | 'all'; @@ -67,3 +104,4 @@ export interface FilterState { feedId?: string; search: string; } +