haopengzhan/rsspipe
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: haopengzhan/rsspipe:d7ed8876dcda945047375c09e5160a8ab883c38d
Branches Tags
haopengzhan/rsspipe:main
..
compare: haopengzhan/rsspipe:dfbf646332092dd4360984c4f4400ebcaba61e79
Branches Tags
haopengzhan/rsspipe:main
6 Commits
d7ed8876dc .. dfbf646332
Author SHA1 Message Date
haopengzhan dfbf646332 feat: implement AI integration with backend configuration and rule-based enrichment actions
CI Quality Checks & Tests / Gitleaks Secret Detection (push) Failing after 47s
Details
CI Quality Checks & Tests / Frontend Typecheck & Build (push) Successful in 1m19s
Details
CI Quality Checks & Tests / Go Lint & Unit Tests (push) Failing after 3h11m47s
Details
2026-08-26 22:42:23 -07:00
haopengzhan 9bc3423009 refactor: standardize article header layout in UI and increase worker pool capacity 2026-08-25 23:01:40 -07:00
haopengzhan 83c70cc3b9 feat: implement rule application engine with full-text variant support, batch entry processing, and tag visualization in UI. 2026-08-25 22:57:48 -07:00
haopengzhan 235966e933 feat: implement entry variant tagging, backend action handlers, batch variant loading, and rule application API. 2026-08-25 22:57:42 -07:00
haopengzhan d440728ca5 feat: implement rule engine, worker pool, event bus, and readability scraping with enhanced logging and content sanitization. 2026-08-25 00:31:32 -07:00
haopengzhan 5ea176d5ad feat: add password management, reading persistence on feed switch, and spin animation 2026-08-24 21:53:54 -07:00
41 changed files with 7277 additions and 355 deletions
Expand all files Collapse all files
.gitleaks.toml
+4 -1
View File
@@ -8,9 +8,12 @@ description = "Global allowlist"
paths = [
'''sample_subscriptions\.opml''',
'''docs/''',
'''web/node_modules/'''
'''web/node_modules/''',
'''web/dist/''',
'''data/'''
]
regexes = [
'''admin123456''',
'''example_jwt_secret_key'''
]
cmd/server/main.go
+34 -17
View File
@@ -2,52 +2,68 @@ package main
import (
"context"
"log"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/actions"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/api"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/auth"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/config"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/crawler"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/events"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/rules"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/worker"
"go.uber.org/zap"
)
func main() {
log.Println("==================================================")
log.Println(" Starting RSSPipe Server ")
log.Println("==================================================")
log := logger.InitLogger(os.Getenv("LOG_LEVEL"))
log.Info("Starting RSSPipe Server")
cfg := config.LoadConfig()
// 1. Initialize SQLite Database
database, err := db.InitDB(cfg.DBPath)
if err != nil {
log.Fatalf("Fatal: Database initialization failed: %v", err)
log.Fatal("Database initialization failed", zap.Error(err))
}
defer func() {
_ = database.Close()
}()
log.Printf("==> Database connected: %s (WAL mode enabled)\n", cfg.DBPath)
log.Info("Database connected", zap.String("path", cfg.DBPath))
// 2. Initialize Auth Service & Bootstrap Admin
authService := auth.NewAuthService(database, cfg)
if err := authService.BootstrapAdmin(); err != nil {
log.Fatalf("Fatal: Admin bootstrap failed: %v", err)
log.Fatal("Admin bootstrap failed", zap.Error(err))
}
// 3. Start Background Crawler Pool
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cr := crawler.NewCrawler(database, cfg.CrawlerIntervalMinutes, cfg.CrawlerWorkers)
// 3. Initialize EventBus, WorkerPool, and Action Handlers
bus := events.NewEventBus()
pool := worker.NewPool(database, bus, 5, 1000)
// Register Core Action Handlers (readability, status, variants, AI enrich & triage)
actions.RegisterDefaultHandlers(pool, database, log)
pool.Start(ctx)
// 4. Initialize Rules Engine
engine := rules.NewEngine(database, bus, pool)
engine.Start(ctx)
// 5. Start Background Crawler Pool
cr := crawler.NewCrawler(database, bus, cfg.CrawlerIntervalMinutes, cfg.CrawlerWorkers)
cr.Start(ctx)
// 4. Mount API Routes
handler := api.NewHandler(database, authService, cr)
// 6. Mount API Routes
handler := api.NewHandler(database, authService, cr, pool)
router := api.NewRouter(handler, authService, "./web/dist")
server := &http.Server{
@@ -58,26 +74,27 @@ func main() {
IdleTimeout: 120 * time.Second,
}
// 5. Graceful shutdown handler
// 7. Graceful shutdown handler
go func() {
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
<-sigChan
log.Println("\n==> Shutting down RSSPipe gracefully...")
log.Info("Shutting down RSSPipe gracefully...")
cancel()
pool.Stop()
shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second)
defer shutdownCancel()
if err := server.Shutdown(shutdownCtx); err != nil {
log.Printf("Error during server shutdown: %v\n", err)
log.Error("Error during server shutdown", zap.Error(err))
}
}()
log.Printf("==> RSSPipe HTTP Server listening at http://localhost:%s\n", cfg.Port)
log.Info("RSSPipe HTTP Server listening", zap.String("url", "http://localhost:"+cfg.Port))
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
log.Fatalf("Fatal: Server error: %v", err)
log.Fatal("Server error", zap.Error(err))
}
log.Println("==> RSSPipe Server stopped.")
log.Info("RSSPipe Server stopped")
}
docs/003_backlog.md
+49
View File
@@ -93,3 +93,52 @@ This document records planned features, UX enhancements, and architectural exten
- Staggered dynamic aspect ratios with prominent media cards.
- Smooth infinite scroll pagination.
- Interactive gesture swipe to dismiss the full-screen reader.
---
## 10. Beautiful 404 Error Page
- **Goal**: Provide a polished, modern, brand-consistent 404 Not Found experience.
- **Scope**:
- Frontend 404 route component with sleek modern aesthetics (glassmorphism/minimal illustration, clear error description, and a "Return to Gateway" action button).
- Server-side fallback handler returning a clean, branded HTML response for unmatched routes.
---
## 11. Multi-Variant Pipeline Enrichments (AI Summary, TTS, Keywords, Audio)
- **Goal**: Expand the event-driven rules engine and variant storage system with automated enrichment plugins.
- **Scope**:
- **AI Summary Hook**: Integrate LLM actions (e.g. OpenAI / Anthropic / Gemini / Ollama) to automatically generate and store concise summaries in `entry_variants(key='ai_summary')`.
- **Keyword & Tag Extraction**: Automated extractor storing structured YAML/JSON keyword lists in `entry_variants(key='keywords')` and `entry_variants(key='tags')` to enable rule-based chaining.
- **Text-to-Speech (TTS)**: Synthesize article text to voice audio blobs (`.mp3` stored under `./data/blobs/{entry_id}/tts.mp3`) with HTTP 206 partial streaming.
- **Podcast TTS & Summary TTS**: Convert articles into dual-host conversational podcasts or audio-narrated summaries streamed directly within the reader.
---
## 12. Aggregated Shared Posts Dashboard (`/shares`)
- **Goal**: Provide a dedicated gallery view aggregating all publicly shared articles.
- **Scope**:
- Centralized dashboard listing all articles with an active `share_uuid`.
- Copy public link, preview shared HTML canvas, revoke share link, and view access stats.
---
## 13. Data Retention & Cleanup Policies (Auto-Deletion)
- **Goal**: Provide configurable retention rules to prune older or noisy articles and keep SQLite storage compact.
- **Scope**:
- Retention triggers (e.g. delete unstarred articles older than 90 days, or limit feeds to latest 500 posts).
- Background garbage collection cleaning associated `entry_variants` and on-disk audio blobs (`./data/blobs/{entry_id}/`).
---
## 14. Outbound Webhook Dispatcher (P4)
- **Goal**: Allow rule-triggered notifications and payload delivery to external platforms.
- **Scope**:
- Outbound webhook action in rule engine supporting Telegram, Discord, Slack, and generic JSON HTTP POST.
- Template engine with interpolation of article variables (`title`, `feed_title`, `url`, `published_at`, `ai_summary`, `tags`).
- Retry mechanism with exponential backoff on delivery failure.
docs/004_variant_pipeline_architecture.md
+116
View File
@@ -0,0 +1,116 @@
# RSSPipe Variant Data & Extensible Processing Pipeline Architecture
## 1. Overview & Vision
RSSPipe is designed as an **RSS-Driven Information Gateway**. Subscriptions come in many formats (full-text blogs, brief news tickers, media podcasts, academic papers). Consequently, a one-size-fits-all processing approach fails:
- Some feeds require automatic full-text extraction via Readability.
- Some feeds require LLM keyword and topic tagging.
- Specific high-value posts (filtered by keywords or feeds) require AI summarization, translation, or Text-to-Speech (TTS) audio synthesis.
To support this cleanly without database corruption, performance degradation, or rigid schemas, RSSPipe adopts a **Variant Data Architecture** paired with a **Reactive Chained Trigger Pipeline**.
---
## 2. Core Data Model
### 2.1 Separation of Immutable Source vs. Enriched Variants
1. **`feed_entries` (Immutable Source of Truth)**:
- Stores the exact metadata and content parsed from the RSS XML feed.
- Core fields: `id`, `feed_id`, `guid`, `url`, `title`, `author`, `rss_content`, `published_at`, `is_read`, `is_starred`, `created_at`.
- **Rule**: `rss_content` is NEVER overwritten by scrapers, LLMs, or TTS engines.
2. **`entry_variants` (Extensible Key-Value Store)**:
- Stores all downstream enrichments as strongly typed variant entries.
- Schema:
```sql
CREATE TABLE IF NOT EXISTS entry_variants (
entry_id TEXT NOT NULL,
variant_key TEXT NOT NULL,
content_type TEXT NOT NULL, -- 'text', 'html', 'json', 'yaml', 'audio/mp3', 'binary'
text_value TEXT, -- UTF-8 text, HTML, JSON array/object, YAML string
blob_path TEXT, -- Relative filesystem path for large media (e.g. data/blobs/{entry_id}/{key}.mp3)
metadata JSON DEFAULT '{}', -- Stats: model, tokens, duration_ms, byte_size, executed_at
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (entry_id, variant_key),
FOREIGN KEY (entry_id) REFERENCES feed_entries(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_entry_variants_key ON entry_variants(variant_key);
```
---
## 3. Supported Variant Keys & Types
| Variant Key | Content Type | Storage Location | Description |
| :--- | :--- | :--- | :--- |
| `readability_full_text` | `html` | `text_value` | Sanitized readability full text extracted from the source web page. |
| `keywords` | `json` / `yaml` | `text_value` | List of extracted keywords (e.g., `["AI", "Go", "Compiler"]`). |
| `tags` | `json` | `text_value` | Category and topic tags. |
| `ai_summary` | `text` / `markdown` | `text_value` | Bulleted or narrative AI executive summary. |
| `translation_{lang}` | `html` / `text` | `text_value` | Translated content (e.g. `translation_zh`). |
| `tts` | `audio/mp3` | `blob_path` | Synthesized speech audio file of the article. |
| `podcast_tts` | `audio/mp3` | `blob_path` | Multi-speaker podcast discussion audio file. |
| `ai_summary_tts` | `audio/mp3` | `blob_path` | Audio narration of the AI summary. |
---
## 4. Large Binary Blob Strategy
Storing multi-megabyte audio files or binary blobs directly inside SQLite columns degrades query cache efficiency and slows down table scans.
- **Text & JSON Variants** are stored inline in `entry_variants.text_value`.
- **Audio & Media Blobs** are stored on disk under `./data/blobs/{entry_id}/{variant_key}.{ext}`.
- `entry_variants.blob_path` stores the relative path.
- The API provides a streaming endpoint with HTTP Range support: `GET /api/entries/{id}/variants/{key}/stream`.
---
## 5. Reactive Chained Trigger & Watch Pipeline
The pipeline operates on typed pub/sub events with depth guards to prevent infinite loops:
```mermaid
graph TD
A[Feed Ingestion / Crawler] -->|Event: on_post_created| Bus[Typed Event Bus]
Bus --> Rules[Rules Engine Evaluator]
Rules -->|Matches Rule 1: Scrape Tech| W1[Worker: Readability Scraper]
W1 -->|Writes variant: readability_full_text| DB[(entry_variants)]
W1 -->|Event: on_variant_updated| Bus
Rules -->|Matches Rule 2: Tag AI Keywords| W2[Worker: Keyword Extractor]
W2 -->|Writes variant: keywords| DB
W2 -->|Event: on_variant_updated| Bus
Bus -->|Matches Rule 3: Watch keywords contains 'AI'| W3[Worker: AI Summary & TTS]
W3 -->|Writes variant: ai_summary & tts blob| DB
```
### 5.1 Concrete Chained Pipeline Example
1. **Trigger 1**: New post ingested from Hacker News -> Emits `on_post_created`.
2. **Rule 1 (Auto-Enrich Keywords)**:
- Condition: `feed_id == "hn-feed-id"`
- Action: `extract_keywords` -> Saves `entry_variants(key='keywords', text_value='["OpenAI", "LLM", "Agent"]')`.
- Emits: `on_variant_updated(key='keywords')`.
3. **Rule 2 (Watch & Synthesize Audio)**:
- Condition: `variant.keywords CONTAINS "OpenAI"`
- Action: `generate_tts` -> Synthesizes MP3, saves to `./data/blobs/...`, saves variant `tts`.
- Emits: `on_variant_updated(key='tts')`.
---
## 6. Reader UX & Content Switcher
In the 3-pane reader interface:
- **Default View**: Renders `rss_content` (original source).
- **Enrichment Pills**: If variants exist on the entry, dynamic tabs/pills appear in the reader header:
- `[📄 Original RSS]`
- `[⚡ Readability Full-Text]` (if `readability_full_text` exists)
- `[💡 AI Summary]` (if `ai_summary` exists)
- `[🎧 Listen Audio]` (if `tts` or `podcast_tts` exists, plays with embedded HTML5 player)
- Switching variants never mutates the underlying article or overwrites database fields.
docs/005_rules_and_hooks_specification.md
+150
View File
@@ -0,0 +1,150 @@
# RSSPipe Rules & Automation Hooks Specification (`005_rules_and_hooks_specification.md`)
## 1. Overview & Vision
The **Rules & Hooks Engine** is the reactive event-processing backbone of RSSPipe. It connects the ingestion lifecycle with the **Multi-Variant Data Store** (`entry_variants`), allowing users to automate content enrichment (scraping, tagging, AI, audio), triage (read, starred, archive), and notifications without writing code.
---
## 2. Core Mental Model: 3-Level Execution Flow
Every Rule is evaluated deterministically using a three-level hierarchy:
```mermaid
graph TD
A[Level 1: Event Trigger] --> B{Level 2: Conditions Evaluator}
B -- All / Any Match --> C[Level 3: Action Execution Pipeline]
B -- No Match --> D[No Action]
C --> E[Store Variant / Mutate Status / Dispatch Event]
E -. New Variant Event .-> A
```
---
## 3. Level 1: Event Triggers
The trigger specifies **when** a rule is evaluated.
| Trigger Type | Identifier | Description | Sub-Options / Parameters |
| :--- | :--- | :--- | :--- |
| **New Entry Ingested** | `new_entry` (`on_post_created`) | Fires immediately after an article is discovered and parsed by crawler/OPML. | None |
| **Variant Changed** | `variant_change` (`on_variant_updated`) | Fires when a specific variant key is added, updated, or deleted. | **Sub-Choice (Event Type)**:<br>• `key_added` (Key didn't exist, now created)<br>• `value_changed` (Key existed, value modified)<br>• `key_deleted` (Variant removed)<br>**Target Key Filter**:<br>• Specific `variant_key` (e.g. `keywords`, `readability_full_text`, `ai_summary`) or `*` (any variant). |
| **Cron Schedule (Future)** | `cron` | Recurring background schedule (e.g., daily digests, archival runs). | Cron expression (e.g. `0 8 * * *`) |
| **Manual Execution** | `manual` | Triggered on-demand via UI or API. | None |
---
## 4. Level 2: Conditions & Allowlisted Fields
Rules evaluate conditions against an **allowlist of strongly typed fields**. The UI dynamically presents appropriate operators and input controls based on the field type.
### 4.1 Allowlisted Fields & Operators
| Field Name | Type | Allowed Operators | UI Input Type & Behavior | Example Evaluation |
| :--- | :--- | :--- | :--- | :--- |
| `feed_id` | `string` / `set` | `==`, `!=`, `in`, `not_in` | **Multi-Select Feed Picker** with search, "Select All", and "Invert" buttons. | `feed_id in ["feed-1", "feed-2", ...]` (Applies to 24/30 feeds in 2 clicks) |
| `category_id` | `string` / `set` | `==`, `!=`, `in`, `not_in` | **Category Dropdown / Multi-Select Chips**. | `category_id in ["cat-tech", "cat-ai"]` |
| `title` | `string` | `contains`, `not_contains`, `==`, `!=`, `matches` (regex), `starts_with`, `ends_with` | Text input or comma-separated token badges. | `title contains "DeepSeek"` |
| `raw_content` / `content` | `string` | `contains`, `not_contains`, `matches` (regex) | Text input. | `content contains "breaking news"` |
| `author` | `string` | `contains`, `not_contains`, `==`, `!=` | Text input. | `author == "Paul Graham"` |
| `created_at` / `published_at` | `timestamp` | `>=`, `<=`, `within_last_hours`, `older_than_days` | Number + unit dropdown (e.g. "Within last 24 hours"). | `published_at within_last_hours 48` |
| `is_read` | `boolean` | `==`, `!=` | Toggle (Read / Unread). | `is_read == false` |
| `is_starred` | `boolean` | `==`, `!=` | Toggle (Starred / Not Starred). | `is_starred == true` |
| `variants.<key>` | `variant_value` | `exists`, `not_exists`, `contains`, `not_contains`, `==`, `!=` | Key name + value input. | `variants.keywords contains "LLM"` |
### 4.2 Match Modes
- **`all` (AND)**: Every condition in the rule must evaluate to true.
- **`any` (OR)**: At least one condition must evaluate to true.
---
## 5. Level 3: Action Execution Pipeline
When a rule's conditions are satisfied, one or more actions execute sequentially.
### 5.1 Supported Core Actions
| Action Identifier | Parameters | Description |
| :--- | :--- | :--- |
| **`set_variant`** | `variant_key` (string), `value` (string/json/yaml), `content_type` (string) | Adds or updates a key-value variant (e.g., adding tags, keywords, custom annotations). |
| **`readability_scrape`** | None | Fetches webpage full-text HTML and stores in `entry_variants(key='readability_full_text')`. |
| **`mark_read`** | None | Sets `is_read = true` (auto-triage/archive). |
| **`mark_starred`** | None | Sets `is_starred = true`. |
---
## 6. Safety Architecture: Endless Loop Prevention
Because actions like `set_variant` emit `on_variant_updated` events, rules could theoretically trigger cyclic loops (e.g., Rule A triggers on `variant_change` for `key="tags"` and adds a new tag to `key="tags"`).
### Loop Guard Mechanisms
1. **Static Validation (Rule Creation Time)**:
- If Trigger is `variant_change` on `key="X"` and an Action is `set_variant` on `key="X"`, the UI / API warns and rejects identical recursive rules unless an explicit guard predicate (e.g., `value not contains "Y"`) is specified.
2. **Runtime Execution Context & Depth Guard**:
- Every event emitted by the Event Bus carries a `RecursionDepth: int` (max depth = 5) and `ExecutionPath: []string` tracking rule IDs.
- If an event exceeds depth 5 or encounters the same Rule ID in its call stack, execution is aborted with an error log: `WARN: Cyclic rule execution aborted for rule: {rule_id}`.
---
## 7. UI/UX Design Specification for Settings
### 7.1 Rule List View
- Clean card layout grouped by Trigger type (`New Entry`, `Variant Change`, `Cron`).
- Card shows a natural language sentence:
> *"When **New Post Arrives** in **24 Feeds** and **Title contains 'AI'** $\rightarrow$ **Scrape Full Text**, **Mark Starred**"*
- Toggle switch (Active/Inactive), Edit button, Test button, Delete button.
### 7.2 Interactive Rule Builder Modal Layout
```
┌────────────────────────────────────────────────────────────────────────┐
│ Create Automation Rule [ ✕ ] │
├────────────────────────────────────────────────────────────────────────┤
│ Rule Name: [ Auto-Scrape Tech Feeds ] │
│ │
│ 1. WHEN (Trigger) │
│ (●) New Entry Arrives ( ) Variant Changed ( ) Schedule │
│ │
│ [If Variant Changed Selected] │
│ Event: [ Key Added ▼ ] Target Key: [ keywords ] │
│ │
│ 2. IF (Conditions) Match: [ All Conditions (AND) ]│
│ ┌─────────────────────────────────────────────────────────────────┐ │
│ │ Field: [ Feed ID ▼ ] Op: [ In List (24 Feeds) ▼ ] │ │
│ │ [ Quick Select: All | None | Invert ] │ │
│ │ [✓] Hacker News [✓] 36Kr [✓] FT Chinese [✓] InfoQ [ ] Douban│ │
│ │ [ ✕ Remove]│ │
│ ├─────────────────────────────────────────────────────────────────┤ │
│ │ Field: [ Title ▼ ] Op: [ Contains Any ▼ ] │ │
│ │ Value: [ AI, LLM, Machine Learning ] │ │
│ │ [ ✕ Remove]│ │
│ └─────────────────────────────────────────────────────────────────┘ │
│ [ + Add Condition ] │
│ │
│ 3. THEN DO (Actions) │
│ [✓] ⚡ Scrape Full Text (Readability) │
│ [✓] ⭐ Mark as Starred │
│ [ ] 🏷️ Set Variant / Tag -> Key: [tags ] Val: [ai-digest ] │
│ [ ] ✓ Mark as Read │
│ │
│ [ Cancel ] [ Create Rule ] │
└────────────────────────────────────────────────────────────────────────┘
```
---
## 8. Backlog Roadmap for Rules & Hooks
The following capabilities are reserved for upcoming iterations:
1. **Auto-Delete / Retention Rule Action** (`delete_entry`):
- Scope: Automatically delete matching entries older than $N$ days (requires retention policy discussion).
2. **Aggregated Shares & Auto-Share Action** (`share_entry`):
- Scope: Action to automatically generate public share links for high-priority articles, plus a dedicated `/shares` dashboard aggregating all shared items.
3. **Advanced AI & Audio Pipeline Actions**:
- `generate_ai_summary`: Calls LLM to summarize content $\rightarrow$ stores in `entry_variants.ai_summary`.
- `generate_tts`: Synthesizes article voice audio $\rightarrow$ stores `.mp3` blob in `./data/blobs/{id}/tts.mp3`.
- `generate_podcast`: Generates dual-speaker conversational audio discussions.
docs/006_ai_rules_and_autonomous_triage.md
+140
View File
@@ -0,0 +1,140 @@
# RSSPipe AI Rules & Autonomous Triage Specification (`006_ai_rules_and_autonomous_triage.md`)
This document defines the architecture, data structures, security guardrails, and execution models for integrating AI capabilities into RSSPipe's event-driven rules engine.
---
## 1. Core Principles
1. **AI as Standard Rule Actions**: AI is not a separate silo; it is integrated directly as actions within the rule pipeline (`Trigger ➜ Conditions ➜ Actions`).
2. **Dual AI Modes**:
- **Mode A: Variant Enrichment (`ai_enrich`)**: Extracts structured summaries, key takeaways, sentiment, or translations into `entry_variants` (`variant_key: "ai_summary"`).
- **Mode B: Autonomous Triage Agent (`ai_agent_action`)**: Evaluates content and proposes actions restricted strictly to a user-defined allowed action pool (e.g., auto-archiving, starring, tagging).
3. **Flexible Prompt Hierarchy**:
- **Global Prompt Library**: Saved reusable templates configured in `Settings > AI & Integrations`.
- **Rule-Level Custom Prompts**: Individual rules can override or write feed-specific prompts (e.g. "Summarize for Kubernetes engineers" vs "Summarize for general news").
4. **Security & Cost Guardrails**:
- Authentication-enforced settings and testing endpoints.
- Rate-limited and token-capped test pings (`max_tokens: 15`).
- Strict action validator discarding any hallucinated or unallowed agent actions.
---
## 2. Architecture & Data Flow
```mermaid
flowchart TD
subgraph TriggerLayer ["1. Trigger & Evaluation"]
Event[Event Ingested / Retroactive Apply] --> Evaluator{Conditions Match?}
end
subgraph ActionExecution ["2. Action Routing & AI Dispatch"]
Evaluator -- Yes --> ActionRouter[Action Router]
ActionRouter --> StandardActions[Standard Actions: Star, Read, Scrape, Set Tag]
ActionRouter --> EnrichAction["AI Variant Enrichment (ai_enrich)"]
ActionRouter --> AgentAction["AI Autonomous Triage (ai_agent_action)"]
end
subgraph AIClient ["3. LLM Gateway (internal/ai)"]
Settings[(system_settings Table)] --> Client[OpenAI-Compatible Client]
EnrichAction --> Client
AgentAction --> Client
Client --> RemoteLLM["LLM Provider (OpenAI, Gemini, DeepSeek, Ollama)"]
end
subgraph PersistenceLayer ["4. Persistence & Chaining"]
EnrichAction --> VariantSave["Save to entry_variants (e.g. ai_summary)"]
VariantSave --> ChainedEvent["Publish EventVariantUpdated"]
AgentAction --> ActionValidator["Strict Action Pool Validator"]
ActionValidator -- Allowed Only --> DispatchPool["Worker Pool Dispatch"]
end
```
---
## 3. Configuration & Settings Schema
### 3.1 Database: `system_settings` Table
```sql
CREATE TABLE IF NOT EXISTS system_settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL,
updated_at DATETIME NOT NULL
);
```
### 3.2 Stored Settings Keys
| Key | Type | Description | Example |
|---|---|---|---|
| `ai_enabled` | `bool` | Master toggle for AI processing | `true` |
| `ai_provider` | `string` | Provider preset identifier | `openai`, `deepseek`, `gemini`, `ollama`, `custom` |
| `ai_base_url` | `string` | OpenAI-compatible endpoint URL | `https://api.openai.com/v1`, `http://localhost:11434/v1` |
| `ai_api_key` | `string` | Provider API Key (masked in GET responses) | `sk-...` |
| `ai_model` | `string` | Model identifier | `gpt-4o-mini`, `deepseek-chat`, `gemini-2.0-flash`, `llama3.2` |
| `ai_temperature` | `float` | Sampling temperature | `0.3` |
| `ai_max_tokens` | `int` | Maximum response tokens | `1000` |
| `ai_prompts` | `json` | Array of saved prompt template objects | `[{"id": "tech_brief", "name": "Tech Brief", "prompt": "..."}]` |
---
## 4. AI Action Specifications
### 4.1 Action `ai_enrich` (Variant Enrichment)
* **Goal**: Generate text/markdown and save as an `EntryVariant`.
* **Parameters**:
```json
{
"type": "ai_enrich",
"params": {
"variant_key": "ai_summary",
"content_type": "text",
"prompt": "You are an expert technical editor. Summarize the following article in 3 concise bullet points followed by 1 key takeaway.\n\nTitle: {{title}}\nContent:\n{{content}}"
}
}
```
* **Variables Available for Prompt Interpolation**:
- `{{title}}`: Article headline
- `{{content}}`: Article body (uses full-text / clean RSS content)
- `{{feed_title}}`: Source feed name
- `{{author}}`: Article author
- `{{url}}`: Original article URL
- `{{published_at}}`: Publication date
### 4.2 Action `ai_agent_action` (Autonomous Triage)
* **Goal**: Analyze the article against a policy and execute allowed actions automatically.
* **Parameters**:
```json
{
"type": "ai_agent_action",
"params": {
"prompt": "Evaluate this engineering post. If it contains a critical zero-day or high-impact CVE, tag as 'security-urgent' and star it. If it is promotional marketing fluff, mark as read.",
"allowed_actions": [
"mark_read",
"mark_starred",
"set_variant:tags:security-urgent",
"set_variant:tags:newsletter"
]
}
}
```
* **Enforced Output Schema**:
```json
{
"reasoning": "Post discusses a critical remote execution CVE in Kubernetes ingress controllers.",
"actions": [
{ "type": "mark_starred" },
{ "type": "set_variant", "params": { "variant_key": "tags", "value": "security-urgent" } }
]
}
```
* **Security Validation**: The backend strictly matches proposed actions against `allowed_actions`. Any unallowed action is logged as a rejection and discarded.
---
## 5. Security & DDoS Protection
1. **Authentication Requirement**: All AI configuration and testing endpoints (`/api/settings/ai`, `/api/settings/ai/test`) require valid JWT bearer tokens with administrative privileges.
2. **Test Endpoint Token Cap & Rate Limiting**:
- `POST /api/settings/ai/test` forces `max_tokens: 15` and a lightweight prompt ("Reply with OK in 2 words") to prevent token exhaustion.
- Enforced rate limit: Maximum 5 connection tests per minute per IP/user session.
3. **API Key Masking**: The `GET /api/settings/ai` endpoint never returns full API keys in plaintext (returns `sk-••••••••abcd`).
go.mod
+10
View File
@@ -3,16 +3,26 @@ module git.pengzhan.dev/haopengzhan/rsspipe
go 1.26.5
require (
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-chi/chi/v5 v5.3.2 // indirect
github.com/go-chi/cors v1.2.2 // indirect
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect
github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0 // indirect
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/css v1.0.1 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/microcosm-cc/bluemonday v1.0.27 // indirect
github.com/mmcdole/gofeed v1.4.2 // indirect
github.com/mmcdole/goxpp/v2 v2.0.0 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
go.uber.org/multierr v1.10.0 // indirect
go.uber.org/zap v1.28.0 // indirect
golang.org/x/crypto v0.55.0 // indirect
golang.org/x/net v0.57.0 // indirect
golang.org/x/sys v0.47.0 // indirect
go.sum
+94
View File
@@ -1,31 +1,125 @@
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA=
github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY=
github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0 h1:A3B75Yp163FAIf9nLlFMl4pwIj+T3uKxfI7mbvvY2Ls=
github.com/go-shiori/go-readability v0.0.0-20251205110129-5db1dc9836f0/go.mod h1:suxK0Wpz4BM3/2+z1mnOVTIWHDiMCIOGoKDCRumSsk0=
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY=
github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk=
github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA=
github.com/mmcdole/gofeed v1.4.2 h1:XFFOtsNNZg+zudjtMXb8BI0J/YdnSIGfjEpPgPnZib0=
github.com/mmcdole/gofeed v1.4.2/go.mod h1:X5x1PyeibJi152VEya0AsV+PW4daYmCD4LJaJbeFkcs=
github.com/mmcdole/goxpp/v2 v2.0.0 h1:HrSCflxerUEqZQNq3u7ldtmE/XkwnTx4Zpq2DW4i5rQ=
github.com/mmcdole/goxpp/v2 v2.0.0/go.mod h1:CUduYMnO9JB6Z/uqDn9Ormk/r8E9BsLQxHPWDZ961Os=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo=
go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M=
golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k=
modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
internal/actions/ai/enrich.go
+98
View File
@@ -0,0 +1,98 @@
package ai
import (
"context"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/ai"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/worker"
"go.uber.org/zap"
)
// LoadConfig fetches current AI provider configuration from system settings.
func LoadConfig(database *db.DB) ai.Config {
settings, _ := database.GetSettingsMap()
enabled := settings["ai_enabled"] == "true" || settings["ai_enabled"] == "1"
provider := settings["ai_provider"]
if provider == "" {
provider = "openai"
}
baseURL := settings["ai_base_url"]
if baseURL == "" {
baseURL = "https://api.openai.com/v1"
}
model := settings["ai_model"]
if model == "" {
model = "gpt-4o-mini"
}
return ai.Config{
Enabled: enabled,
Provider: provider,
BaseURL: baseURL,
APIKey: settings["ai_api_key"],
Model: model,
}
}
// NewEnrichmentHandler creates a worker action handler for AI variant enrichment.
func NewEnrichmentHandler(database *db.DB, client *ai.Client, log *zap.Logger) worker.ActionHandler {
return func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*worker.ActionResult, error) {
cfg := LoadConfig(database)
if cfg.APIKey == "" && cfg.Provider != "ollama" {
log.Warn("AI action skipped: API key is not configured in Settings > AI & Integrations",
zap.String("entry_id", entry.ID),
zap.String("provider", cfg.Provider),
)
return nil, nil
}
prompt, _ := params["prompt"].(string)
if prompt == "" {
prompt, _ = params["custom_prompt"].(string)
}
vKey, _ := params["variant_key"].(string)
if vKey == "" {
vKey = "ai_summary"
}
cType, _ := params["content_type"].(string)
if cType == "" {
cType = "text"
}
resultText, err := client.GenerateEnrichment(ctx, cfg, prompt, entry)
if err != nil {
log.Error("AI enrichment generation failed",
zap.String("entry_id", entry.ID),
zap.String("model", cfg.Model),
zap.Error(err),
)
return nil, err
}
variant := models.EntryVariant{
EntryID: entry.ID,
VariantKey: vKey,
ContentType: cType,
TextValue: resultText,
Metadata: map[string]interface{}{
"model": cfg.Model,
"provider": cfg.Provider,
"prompt_used": prompt,
},
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
log.Info("AI enrichment completed",
zap.String("entry_id", entry.ID),
zap.String("variant_key", vKey),
zap.String("model", cfg.Model),
)
return &worker.ActionResult{
ResultVariant: &variant,
}, nil
}
}
internal/actions/ai/triage.go
+88
View File
@@ -0,0 +1,88 @@
package ai
import (
"context"
"fmt"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/ai"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/worker"
"go.uber.org/zap"
)
// NewTriageHandler creates a worker action handler for AI autonomous triage decisions.
func NewTriageHandler(database *db.DB, client *ai.Client, pool *worker.Pool, log *zap.Logger) worker.ActionHandler {
return func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*worker.ActionResult, error) {
cfg := LoadConfig(database)
if cfg.APIKey == "" && cfg.Provider != "ollama" {
log.Warn("AI triage action skipped: API key is not configured in Settings > AI & Integrations",
zap.String("entry_id", entry.ID),
)
return nil, nil
}
policyPrompt, _ := params["prompt"].(string)
var allowedActions []string
if rawAllowed, ok := params["allowed_actions"].([]interface{}); ok {
for _, a := range rawAllowed {
allowedActions = append(allowedActions, fmt.Sprintf("%v", a))
}
} else if strList, ok := params["allowed_actions"].([]string); ok {
allowedActions = strList
}
if len(allowedActions) == 0 {
allowedActions = []string{"mark_read", "mark_starred", "set_variant"}
}
decision, err := client.GenerateAgentDecision(ctx, cfg, policyPrompt, allowedActions, entry)
if err != nil {
log.Error("AI triage decision failed",
zap.String("entry_id", entry.ID),
zap.Error(err),
)
return nil, err
}
log.Info("AI Triage agent executed successfully",
zap.String("entry_id", entry.ID),
zap.String("reasoning", decision.Reasoning),
zap.Int("actions_approved", len(decision.Actions)),
)
res := &worker.ActionResult{}
for _, act := range decision.Actions {
switch act.Type {
case "mark_read":
trueVal := true
res.MarkRead = &trueVal
case "mark_starred":
trueVal := true
res.MarkStarred = &trueVal
case "set_variant":
vKey, _ := act.Params["variant_key"].(string)
if vKey == "" {
vKey, _ = act.Params["key"].(string)
}
vVal, _ := act.Params["value"].(string)
if vKey != "" {
v := models.EntryVariant{
EntryID: entry.ID,
VariantKey: vKey,
ContentType: "text",
TextValue: vVal,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
res.ResultVariant = &v
}
default:
pool.Dispatch(ctx, act, entry, 0)
}
}
return res, nil
}
}
internal/actions/readability/readability.go
+123
View File
@@ -0,0 +1,123 @@
package readability
import (
"context"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/worker"
readability "github.com/go-shiori/go-readability"
"github.com/microcosm-cc/bluemonday"
"go.uber.org/zap"
)
type Scraper struct {
client *http.Client
sanitizer *bluemonday.Policy
log *zap.Logger
}
func NewScraper(timeout time.Duration) *Scraper {
if timeout <= 0 {
timeout = 15 * time.Second
}
p := bluemonday.UGCPolicy()
// Allow rich media & responsive elements
p.AllowElements("video", "audio", "source", "track", "iframe", "picture", "figure", "figcaption", "article", "section")
p.AllowAttrs("src", "poster", "controls", "autoplay", "loop", "muted", "preload").OnElements("video", "audio")
p.AllowAttrs("src", "type", "srcset", "media").OnElements("source")
p.AllowAttrs("src", "width", "height", "frameborder", "allow", "allowfullscreen", "sandbox").OnElements("iframe")
p.AllowAttrs("data-src", "data-original", "loading", "srcset", "sizes").OnElements("img")
p.AllowAttrs("class", "id").Globally()
return &Scraper{
client: &http.Client{
Timeout: timeout,
},
sanitizer: p,
log: logger.Named("readability-scraper"),
}
}
// Handler returns the worker.ActionHandler function for readability scraping.
func (s *Scraper) Handler() worker.ActionHandler {
return func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*worker.ActionResult, error) {
if strings.TrimSpace(entry.URL) == "" {
return nil, fmt.Errorf("entry URL is empty")
}
parsedURL, err := url.Parse(entry.URL)
if err != nil {
return nil, fmt.Errorf("invalid URL: %w", err)
}
s.log.Info("Starting readability scrape",
zap.String("entry_id", entry.ID),
zap.String("url", entry.URL),
)
req, err := http.NewRequestWithContext(ctx, "GET", entry.URL, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
req.Header.Set("Accept-Language", "en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7")
resp, err := s.client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to fetch article page: %w", err)
}
defer func() {
_ = resp.Body.Close()
}()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("server returned HTTP status %d", resp.StatusCode)
}
article, err := readability.FromReader(resp.Body, parsedURL)
if err != nil {
return nil, fmt.Errorf("failed to parse readability article: %w", err)
}
cleanHTML := s.sanitizer.Sanitize(article.Content)
if strings.TrimSpace(cleanHTML) == "" && strings.TrimSpace(article.TextContent) != "" {
cleanHTML = "<p>" + s.sanitizer.Sanitize(article.TextContent) + "</p>"
}
s.log.Info("Readability scrape successful",
zap.String("entry_id", entry.ID),
zap.String("title", article.Title),
zap.Int("content_length", len(cleanHTML)),
)
now := time.Now().UTC()
variant := models.EntryVariant{
EntryID: entry.ID,
VariantKey: "readability_full_text",
ContentType: "html",
TextValue: cleanHTML,
Metadata: map[string]interface{}{
"title": article.Title,
"byline": article.Byline,
"excerpt": article.Excerpt,
"length": len(article.TextContent),
"scraped_at": now.Format(time.RFC3339),
},
CreatedAt: now,
UpdatedAt: now,
}
return &worker.ActionResult{
ResultVariant: &variant,
}, nil
}
}
internal/actions/register.go
+36
View File
@@ -0,0 +1,36 @@
package actions
import (
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/actions/ai"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/actions/readability"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/actions/status"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/actions/variants"
aiClientPkg "git.pengzhan.dev/haopengzhan/rsspipe/internal/ai"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/worker"
"go.uber.org/zap"
)
// RegisterDefaultHandlers registers all core action handlers with the worker pool.
func RegisterDefaultHandlers(pool *worker.Pool, database *db.DB, log *zap.Logger) {
// 1. Readability Full-Text Scraper
scraper := readability.NewScraper(15 * time.Second)
pool.RegisterHandler("readability_scrape", scraper.Handler())
// 2. Status Handlers (mark_read, mark_starred)
pool.RegisterHandler("mark_read", status.NewMarkReadHandler())
pool.RegisterHandler("mark_starred", status.NewMarkStarredHandler())
// 3. Variant Handlers (set_variant with tag merging)
pool.RegisterHandler("set_variant", variants.NewSetVariantHandler(database))
// 4. AI Subsystem Handlers
aiClient := aiClientPkg.NewClient(45 * time.Second)
enrichHandler := ai.NewEnrichmentHandler(database, aiClient, log)
pool.RegisterHandler("ai_enrich", enrichHandler)
pool.RegisterHandler("ai_summary", enrichHandler) // Backward compatibility alias
pool.RegisterHandler("ai_agent_action", ai.NewTriageHandler(database, aiClient, pool, log))
}
internal/actions/status/status.go
+28
View File
@@ -0,0 +1,28 @@
package status
import (
"context"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/worker"
)
// NewMarkReadHandler marks an entry as read.
func NewMarkReadHandler() worker.ActionHandler {
return func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*worker.ActionResult, error) {
trueVal := true
return &worker.ActionResult{
MarkRead: &trueVal,
}, nil
}
}
// NewMarkStarredHandler marks an entry as starred.
func NewMarkStarredHandler() worker.ActionHandler {
return func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*worker.ActionResult, error) {
trueVal := true
return &worker.ActionResult{
MarkStarred: &trueVal,
}, nil
}
}
internal/actions/variants/set_variant.go
+60
View File
@@ -0,0 +1,60 @@
package variants
import (
"context"
"strings"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/worker"
)
// NewSetVariantHandler creates a worker action handler for attaching extensible variant metadata.
func NewSetVariantHandler(database *db.DB) worker.ActionHandler {
return func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*worker.ActionResult, error) {
key, _ := params["variant_key"].(string)
if key == "" {
key, _ = params["key"].(string)
}
if key == "" {
return nil, nil
}
val, _ := params["value"].(string)
contentType, _ := params["content_type"].(string)
if contentType == "" {
contentType = "text"
}
// If key == "tags", merge with existing tags so multiple rules append tags
if key == "tags" {
existingVariant, err := database.GetVariant(entry.ID, "tags")
if err == nil && existingVariant != nil && existingVariant.TextValue != "" {
existingTags := strings.Split(existingVariant.TextValue, ",")
newTags := strings.Split(val, ",")
tagSet := make(map[string]bool)
var merged []string
for _, t := range append(existingTags, newTags...) {
trimmed := strings.TrimSpace(t)
if trimmed != "" && !tagSet[strings.ToLower(trimmed)] {
tagSet[strings.ToLower(trimmed)] = true
merged = append(merged, trimmed)
}
}
val = strings.Join(merged, ", ")
}
}
variant := models.EntryVariant{
EntryID: entry.ID,
VariantKey: key,
ContentType: contentType,
TextValue: val,
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
}
return &worker.ActionResult{
ResultVariant: &variant,
}, nil
}
}
internal/ai/client.go
+356
View File
@@ -0,0 +1,356 @@
package ai
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
)
type Config struct {
Enabled bool `json:"enabled"`
Provider string `json:"provider"`
BaseURL string `json:"base_url"`
APIKey string `json:"api_key"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
}
type PromptTemplate struct {
ID string `json:"id"`
Name string `json:"name"`
Description string `json:"description,omitempty"`
Prompt string `json:"prompt"`
IsDefault bool `json:"is_default,omitempty"`
}
type TestResult struct {
Success bool `json:"success"`
Message string `json:"message"`
LatencyMs int64 `json:"latency_ms"`
Model string `json:"model"`
Duration time.Duration `json:"duration"`
}
type AgentDecision struct {
Reasoning string `json:"reasoning"`
Actions []models.RuleAction `json:"actions"`
}
type Client struct {
httpClient *http.Client
}
func NewClient(timeout time.Duration) *Client {
if timeout <= 0 {
timeout = 45 * time.Second
}
return &Client{
httpClient: &http.Client{
Timeout: timeout,
},
}
}
// DefaultPromptTemplates returns built-in prompt templates.
func DefaultPromptTemplates() []PromptTemplate {
return []PromptTemplate{
{
ID: "exec_summary",
Name: "3-Bullet Executive Brief",
Description: "3 concise bullet points followed by a single key takeaway.",
Prompt: "You are an expert technical editor. Summarize the following article in exactly 3 concise bullet points, followed by a bold 'Key Takeaway:' sentence.\n\nTitle: {{title}}\nFeed: {{feed_title}}\nContent:\n{{content}}",
IsDefault: true,
},
{
ID: "tech_deep_dive",
Name: "Technical Deep Dive",
Description: "Architecture, key technical concepts, and implementation highlights.",
Prompt: "Analyze the following technical post. Highlight:\n1. Core Architecture & Concepts\n2. Key Innovations / Techniques\n3. Practical Implications for Engineers\n\nTitle: {{title}}\nContent:\n{{content}}",
IsDefault: false,
},
{
ID: "news_general",
Name: "General News & Context",
Description: "Accessible summary with necessary background context for general readers.",
Prompt: "Summarize this news article for a general audience. Explain what happened, why it matters, and provide any helpful context in 2-3 short paragraphs.\n\nTitle: {{title}}\nContent:\n{{content}}",
IsDefault: false,
},
}
}
type chatMessage struct {
Role string `json:"role"`
Content string `json:"content"`
}
type chatCompletionRequest struct {
Model string `json:"model"`
Messages []chatMessage `json:"messages"`
Temperature float64 `json:"temperature,omitempty"`
MaxTokens int `json:"max_tokens,omitempty"`
}
type chatCompletionResponse struct {
Choices []struct {
Message struct {
Role string `json:"role"`
Content string `json:"content"`
} `json:"message"`
FinishReason string `json:"finish_reason"`
} `json:"choices"`
Error *struct {
Message string `json:"message"`
Type string `json:"type"`
} `json:"error,omitempty"`
}
func (c *Client) callChatCompletion(ctx context.Context, cfg Config, req chatCompletionRequest) (string, error) {
baseURL := strings.TrimRight(cfg.BaseURL, "/")
if baseURL == "" {
baseURL = "https://api.openai.com/v1"
}
endpoint := baseURL + "/chat/completions"
bodyBytes, err := json.Marshal(req)
if err != nil {
return "", err
}
httpReq, err := http.NewRequestWithContext(ctx, "POST", endpoint, bytes.NewReader(bodyBytes))
if err != nil {
return "", err
}
httpReq.Header.Set("Content-Type", "application/json")
if cfg.APIKey != "" {
httpReq.Header.Set("Authorization", "Bearer "+cfg.APIKey)
}
resp, err := c.httpClient.Do(httpReq)
if err != nil {
return "", fmt.Errorf("HTTP request failed: %w", err)
}
defer func() {
_ = resp.Body.Close()
}()
respBody, err := io.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("failed to read response body: %w", err)
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return "", fmt.Errorf("API error (status %d): %s", resp.StatusCode, string(respBody))
}
var chatResp chatCompletionResponse
if err := json.Unmarshal(respBody, &chatResp); err != nil {
return "", fmt.Errorf("failed to decode response JSON: %w", err)
}
if chatResp.Error != nil && chatResp.Error.Message != "" {
return "", fmt.Errorf("LLM error: %s", chatResp.Error.Message)
}
if len(chatResp.Choices) == 0 {
return "", fmt.Errorf("LLM returned no choices")
}
return chatResp.Choices[0].Message.Content, nil
}
// TestConnection pings the LLM with a strictly capped request (max_tokens: 15) to verify credentials.
func (c *Client) TestConnection(ctx context.Context, cfg Config) (*TestResult, error) {
start := time.Now()
req := chatCompletionRequest{
Model: cfg.Model,
Messages: []chatMessage{
{Role: "user", Content: "Ping test. Reply with 'OK' in exactly two words."},
},
Temperature: 0.1,
MaxTokens: 15, // Strictly capped to prevent token exhaustion
}
reply, err := c.callChatCompletion(ctx, cfg, req)
duration := time.Since(start)
if err != nil {
return &TestResult{
Success: false,
Message: err.Error(),
LatencyMs: duration.Milliseconds(),
Model: cfg.Model,
Duration: duration,
}, err
}
return &TestResult{
Success: true,
Message: strings.TrimSpace(reply),
LatencyMs: duration.Milliseconds(),
Model: cfg.Model,
Duration: duration,
}, nil
}
// GenerateEnrichment interpolates the prompt template and queries the LLM to generate variant content.
func (c *Client) GenerateEnrichment(ctx context.Context, cfg Config, promptTemplate string, entry models.FeedEntry) (string, error) {
if promptTemplate == "" {
promptTemplate = DefaultPromptTemplates()[0].Prompt
}
content := entry.Content
if entry.RawContent != "" && len(entry.RawContent) > len(content) {
content = entry.RawContent
}
if entry.Variants != nil {
if v, ok := entry.Variants["readability_full_text"]; ok && len(v.TextValue) > len(content) {
content = v.TextValue
}
}
// Truncate content to avoid exceeding token windows (e.g. max ~24,000 characters)
if len(content) > 24000 {
content = content[:24000] + "... [truncated]"
}
interpolated := promptTemplate
interpolated = strings.ReplaceAll(interpolated, "{{title}}", entry.Title)
interpolated = strings.ReplaceAll(interpolated, "{{content}}", content)
interpolated = strings.ReplaceAll(interpolated, "{{feed_title}}", entry.FeedTitle)
interpolated = strings.ReplaceAll(interpolated, "{{author}}", entry.Author)
interpolated = strings.ReplaceAll(interpolated, "{{url}}", entry.URL)
interpolated = strings.ReplaceAll(interpolated, "{{published_at}}", entry.PublishedAt.Format("2006-01-02 15:04:05"))
maxTokens := cfg.MaxTokens
if maxTokens <= 0 {
maxTokens = 1000
}
temp := cfg.Temperature
if temp <= 0 {
temp = 0.3
}
req := chatCompletionRequest{
Model: cfg.Model,
Messages: []chatMessage{
{Role: "user", Content: interpolated},
},
Temperature: temp,
MaxTokens: maxTokens,
}
return c.callChatCompletion(ctx, cfg, req)
}
// GenerateAgentDecision asks the LLM to evaluate the article and propose actions from allowedActions only.
func (c *Client) GenerateAgentDecision(ctx context.Context, cfg Config, policyPrompt string, allowedActions []string, entry models.FeedEntry) (*AgentDecision, error) {
allowedListStr := strings.Join(allowedActions, ", ")
content := entry.Content
if entry.RawContent != "" && len(entry.RawContent) > len(content) {
content = entry.RawContent
}
if entry.Variants != nil {
if v, ok := entry.Variants["readability_full_text"]; ok && len(v.TextValue) > len(content) {
content = v.TextValue
}
}
if len(content) > 16000 {
content = content[:16000] + "... [truncated]"
}
systemInstruction := fmt.Sprintf(`You are an autonomous RSS triage agent.
Your goal is to evaluate the provided article and propose actions strictly from the Allowed Action Pool.
Allowed Action Pool:
%s
Rules:
1. You may ONLY propose actions that exist in the Allowed Action Pool.
2. Output ONLY a valid JSON object matching this schema:
{
"reasoning": "brief explanation of your decision",
"actions": [
{"type": "action_type", "params": {"key": "value"}}
]
}
If no action is warranted, return {"reasoning": "...", "actions": []}.
Do not wrap in markdown backticks or other text. Return raw JSON only.`, allowedListStr)
userContent := fmt.Sprintf("Evaluation Policy:\n%s\n\nArticle Title: %s\nFeed: %s\nContent:\n%s", policyPrompt, entry.Title, entry.FeedTitle, content)
req := chatCompletionRequest{
Model: cfg.Model,
Messages: []chatMessage{
{Role: "system", Content: systemInstruction},
{Role: "user", Content: userContent},
},
Temperature: 0.1,
MaxTokens: 800,
}
rawJSON, err := c.callChatCompletion(ctx, cfg, req)
if err != nil {
return nil, err
}
// Clean code fence formatting if returned
rawJSON = strings.TrimSpace(rawJSON)
rawJSON = strings.TrimPrefix(rawJSON, "```json")
rawJSON = strings.TrimPrefix(rawJSON, "```")
rawJSON = strings.TrimSuffix(rawJSON, "```")
rawJSON = strings.TrimSpace(rawJSON)
var parsed struct {
Reasoning string `json:"reasoning"`
Actions []struct {
Type string `json:"type"`
Params map[string]interface{} `json:"params,omitempty"`
} `json:"actions"`
}
if err := json.Unmarshal([]byte(rawJSON), &parsed); err != nil {
return nil, fmt.Errorf("failed to parse AI decision JSON: %w (raw: %s)", err, rawJSON)
}
// Strict Guardrail: Validate all actions against allowedActions
allowedSet := make(map[string]bool)
for _, a := range allowedActions {
allowedSet[strings.ToLower(strings.TrimSpace(a))] = true
}
var validatedActions []models.RuleAction
for _, act := range parsed.Actions {
actTypeLower := strings.ToLower(strings.TrimSpace(act.Type))
// Check basic action type or composite key (e.g. set_variant:tags:xxx)
isAllowed := allowedSet[actTypeLower]
if !isAllowed && actTypeLower == "set_variant" && act.Params != nil {
vKey := fmt.Sprintf("%v", act.Params["variant_key"])
vVal := fmt.Sprintf("%v", act.Params["value"])
compositeKey := strings.ToLower(fmt.Sprintf("set_variant:%s:%s", vKey, vVal))
isAllowed = allowedSet[compositeKey] || allowedSet["set_variant"]
}
if isAllowed {
validatedActions = append(validatedActions, models.RuleAction{
Type: act.Type,
Params: act.Params,
})
}
}
return &AgentDecision{
Reasoning: parsed.Reasoning,
Actions: validatedActions,
}, nil
}
internal/ai/client_test.go
+212
View File
@@ -0,0 +1,212 @@
package ai
import (
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"testing"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
)
type roundTripFunc func(req *http.Request) *http.Response
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return f(req), nil
}
func TestTestConnection(t *testing.T) {
client := &Client{
httpClient: &http.Client{
Transport: roundTripFunc(func(req *http.Request) *http.Response {
if req.Header.Get("Authorization") != "Bearer test-key" {
return &http.Response{
StatusCode: http.StatusUnauthorized,
Body: io.NopCloser(bytes.NewBufferString(`{"error": {"message": "unauthorized"}}`)),
Header: make(http.Header),
}
}
resp := chatCompletionResponse{
Choices: []struct {
Message struct {
Role string `json:"role"`
Content string `json:"content"`
} `json:"message"`
FinishReason string `json:"finish_reason"`
}{
{
Message: struct {
Role string `json:"role"`
Content string `json:"content"`
}{
Role: "assistant",
Content: "OK connection",
},
FinishReason: "stop",
},
},
}
data, _ := json.Marshal(resp)
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(bytes.NewReader(data)),
Header: make(http.Header),
}
}),
},
}
cfg := Config{
Enabled: true,
Provider: "custom",
BaseURL: "https://mock.api",
APIKey: "test-key",
Model: "test-model",
}
res, err := client.TestConnection(context.Background(), cfg)
if err != nil {
t.Fatalf("TestConnection failed: %v", err)
}
if !res.Success {
t.Fatalf("Expected success true, got false")
}
if res.Message != "OK connection" {
t.Fatalf("Expected 'OK connection', got %s", res.Message)
}
}
func TestGenerateEnrichment(t *testing.T) {
client := &Client{
httpClient: &http.Client{
Transport: roundTripFunc(func(req *http.Request) *http.Response {
resp := chatCompletionResponse{
Choices: []struct {
Message struct {
Role string `json:"role"`
Content string `json:"content"`
} `json:"message"`
FinishReason string `json:"finish_reason"`
}{
{
Message: struct {
Role string `json:"role"`
Content string `json:"content"`
}{
Role: "assistant",
Content: "- Point 1\n- Point 2\n- Point 3\n\n**Key Takeaway**: Great post.",
},
FinishReason: "stop",
},
},
}
data, _ := json.Marshal(resp)
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(bytes.NewReader(data)),
Header: make(http.Header),
}
}),
},
}
cfg := Config{
Enabled: true,
BaseURL: "https://mock.api",
Model: "gpt-4o-mini",
MaxTokens: 500,
}
entry := models.FeedEntry{
ID: "e-1",
Title: "Kubernetes & GPU Optimization",
FeedTitle: "Cloud Native Blog",
Content: "Detailed technical content about GPU sharing in Kubernetes.",
PublishedAt: time.Now(),
}
summary, err := client.GenerateEnrichment(context.Background(), cfg, "Summarize: {{title}}\n{{content}}", entry)
if err != nil {
t.Fatalf("GenerateEnrichment failed: %v", err)
}
if summary == "" {
t.Fatalf("Expected summary string, got empty")
}
}
func TestGenerateAgentDecision(t *testing.T) {
client := &Client{
httpClient: &http.Client{
Transport: roundTripFunc(func(req *http.Request) *http.Response {
resp := chatCompletionResponse{
Choices: []struct {
Message struct {
Role string `json:"role"`
Content string `json:"content"`
} `json:"message"`
FinishReason string `json:"finish_reason"`
}{
{
Message: struct {
Role string `json:"role"`
Content string `json:"content"`
}{
Role: "assistant",
Content: `{
"reasoning": "High priority kubernetes security patch",
"actions": [
{"type": "mark_starred"},
{"type": "set_variant", "params": {"variant_key": "tags", "value": "security-alert"}},
{"type": "malicious_unallowed_action"}
]
}`,
},
FinishReason: "stop",
},
},
}
data, _ := json.Marshal(resp)
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(bytes.NewReader(data)),
Header: make(http.Header),
}
}),
},
}
cfg := Config{
Enabled: true,
BaseURL: "https://mock.api",
Model: "gpt-4o-mini",
}
entry := models.FeedEntry{
ID: "e-1",
Title: "Critical CVE in Kubelet",
FeedTitle: "Kubernetes Security",
Content: "Remote code execution flaw patched in v1.36.",
}
allowed := []string{"mark_starred", "set_variant:tags:security-alert"}
decision, err := client.GenerateAgentDecision(context.Background(), cfg, "Evaluate urgency", allowed, entry)
if err != nil {
t.Fatalf("GenerateAgentDecision failed: %v", err)
}
if decision.Reasoning == "" {
t.Fatalf("Expected reasoning, got empty")
}
// Verify strict validation: malicious_unallowed_action was stripped
if len(decision.Actions) != 2 {
t.Fatalf("Expected 2 validated actions, got %d", len(decision.Actions))
}
if decision.Actions[0].Type != "mark_starred" {
t.Fatalf("Expected mark_starred, got %s", decision.Actions[0].Type)
}
}
internal/api/handlers.go
+450 -9
View File
@@ -9,30 +9,41 @@ import (
"net/http"
"strconv"
"strings"
"sync"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/ai"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/auth"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/crawler"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/opml"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/parser"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/rules"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/worker"
"github.com/go-chi/chi/v5"
)
type Handler struct {
db *db.DB
auth *auth.AuthService
crawler *crawler.Crawler
parser *parser.Parser
db *db.DB
auth *auth.AuthService
crawler *crawler.Crawler
parser *parser.Parser
pool *worker.Pool
aiClient *ai.Client
testRateMu sync.Mutex
testPings []time.Time
}
func NewHandler(database *db.DB, authService *auth.AuthService, cr *crawler.Crawler) *Handler {
func NewHandler(database *db.DB, authService *auth.AuthService, cr *crawler.Crawler, pool *worker.Pool) *Handler {
return &Handler{
db: database,
auth: authService,
crawler: cr,
parser: parser.NewParser(),
db: database,
auth: authService,
crawler: cr,
parser: parser.NewParser(),
pool: pool,
aiClient: ai.NewClient(30 * time.Second),
testPings: []time.Time{},
}
}
@@ -101,6 +112,27 @@ func (h *Handler) GetMe(w http.ResponseWriter, r *http.Request) {
h.writeJSON(w, http.StatusOK, user)
}
func (h *Handler) UpdatePassword(w http.ResponseWriter, r *http.Request) {
claims, ok := r.Context().Value(auth.UserContextKey).(*auth.Claims)
if !ok {
h.writeError(w, http.StatusUnauthorized, "unauthorized")
return
}
var req models.ChangePasswordRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
h.writeError(w, http.StatusBadRequest, "invalid request payload")
return
}
if err := h.auth.UpdatePassword(claims.UserID, req.OldPassword, req.NewPassword); err != nil {
h.writeError(w, http.StatusBadRequest, err.Error())
return
}
h.writeJSON(w, http.StatusOK, map[string]string{"message": "password updated successfully"})
}
// Stats Handler
func (h *Handler) GetStats(w http.ResponseWriter, r *http.Request) {
stats, err := h.db.GetStats()
@@ -526,3 +558,412 @@ func (h *Handler) ViewSharedEntry(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
_, _ = w.Write([]byte(html))
}
// GetEntry returns a single entry by ID with its loaded variants
func (h *Handler) GetEntry(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
entry, err := h.db.GetEntry(id)
if err != nil {
if err == sql.ErrNoRows {
h.writeError(w, http.StatusNotFound, "entry not found")
return
}
h.writeError(w, http.StatusInternalServerError, err.Error())
return
}
h.writeJSON(w, http.StatusOK, entry)
}
// ListEntryVariants returns all variants for a given entry ID
func (h *Handler) ListEntryVariants(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
variants, err := h.db.ListVariantsForEntry(id)
if err != nil {
h.writeError(w, http.StatusInternalServerError, err.Error())
return
}
h.writeJSON(w, http.StatusOK, variants)
}
// GetVariantBlobStream streams the audio/binary file for a variant with HTTP 206 Partial Content support
func (h *Handler) GetVariantBlobStream(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
key := chi.URLParam(r, "key")
variant, err := h.db.GetVariant(id, key)
if err != nil {
h.writeError(w, http.StatusNotFound, "variant not found")
return
}
if variant.BlobPath == "" {
h.writeError(w, http.StatusNotFound, "variant has no binary media blob")
return
}
http.ServeFile(w, r, variant.BlobPath)
}
// ScrapeEntry manually triggers readability full text scraping and saves to entry_variants
func (h *Handler) ScrapeEntry(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
entry, err := h.db.GetEntry(id)
if err != nil {
h.writeError(w, http.StatusNotFound, "entry not found")
return
}
if h.pool == nil {
h.writeError(w, http.StatusInternalServerError, "worker pool not configured")
return
}
res, err := h.pool.ExecuteDirect(r.Context(), "readability_scrape", *entry, nil)
if err != nil {
h.writeError(w, http.StatusInternalServerError, fmt.Sprintf("scraping failed: %v", err))
return
}
if res != nil && res.ResultVariant != nil {
_ = h.db.SetVariant(*res.ResultVariant)
}
// Reload entry with newly saved variant
updatedEntry, err := h.db.GetEntry(id)
if err != nil {
h.writeError(w, http.StatusInternalServerError, err.Error())
return
}
h.writeJSON(w, http.StatusOK, updatedEntry)
}
// Rules API Handlers
func (h *Handler) ListRules(w http.ResponseWriter, r *http.Request) {
rules, err := h.db.ListRules()
if err != nil {
h.writeError(w, http.StatusInternalServerError, err.Error())
return
}
if rules == nil {
rules = []models.Rule{}
}
h.writeJSON(w, http.StatusOK, rules)
}
type CreateRuleRequest struct {
Name string `json:"name"`
Trigger string `json:"trigger"`
MatchMode string `json:"match_mode"`
Conditions []models.RuleCondition `json:"conditions"`
Actions []models.RuleAction `json:"actions"`
}
func (h *Handler) CreateRule(w http.ResponseWriter, r *http.Request) {
var body CreateRuleRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
h.writeError(w, http.StatusBadRequest, "invalid request body")
return
}
if strings.TrimSpace(body.Name) == "" {
h.writeError(w, http.StatusBadRequest, "rule name is required")
return
}
if strings.TrimSpace(body.Trigger) == "" {
body.Trigger = "on_post_created"
}
if err := rules.ValidateRuleSafety(models.Rule{
Trigger: body.Trigger,
Conditions: body.Conditions,
Actions: body.Actions,
}); err != nil {
h.writeError(w, http.StatusBadRequest, err.Error())
return
}
rule, err := h.db.CreateRule(body.Name, body.Trigger, body.MatchMode, body.Conditions, body.Actions)
if err != nil {
h.writeError(w, http.StatusInternalServerError, err.Error())
return
}
h.writeJSON(w, http.StatusCreated, rule)
}
type UpdateRuleRequest struct {
Name string `json:"name"`
Trigger string `json:"trigger"`
MatchMode string `json:"match_mode"`
Conditions []models.RuleCondition `json:"conditions"`
Actions []models.RuleAction `json:"actions"`
IsActive bool `json:"is_active"`
}
func (h *Handler) UpdateRule(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
var body UpdateRuleRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
h.writeError(w, http.StatusBadRequest, "invalid request body")
return
}
if err := rules.ValidateRuleSafety(models.Rule{
Trigger: body.Trigger,
Conditions: body.Conditions,
Actions: body.Actions,
}); err != nil {
h.writeError(w, http.StatusBadRequest, err.Error())
return
}
rule, err := h.db.UpdateRule(id, body.Name, body.Trigger, body.MatchMode, body.Conditions, body.Actions, body.IsActive)
if err != nil {
h.writeError(w, http.StatusInternalServerError, err.Error())
return
}
h.writeJSON(w, http.StatusOK, rule)
}
func (h *Handler) DeleteRule(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
if err := h.db.DeleteRule(id); err != nil {
h.writeError(w, http.StatusInternalServerError, err.Error())
return
}
h.writeJSON(w, http.StatusOK, map[string]string{"message": "rule deleted"})
}
func (h *Handler) ApplyRule(w http.ResponseWriter, r *http.Request) {
id := chi.URLParam(r, "id")
rule, err := h.db.GetRule(id)
if err != nil {
h.writeError(w, http.StatusNotFound, "rule not found")
return
}
evaluator := rules.NewEvaluator()
matchedCount := 0
queuedJobs := 0
// Paginate through ALL entries in batches of 500
batchSize := 500
offset := 0
for {
entries, total, err := h.db.ListEntries(db.EntryFilter{
View: "all",
Limit: batchSize,
Offset: offset,
})
if err != nil {
h.writeError(w, http.StatusInternalServerError, "failed to query entries: "+err.Error())
return
}
for _, entry := range entries {
if evaluator.Evaluate(*rule, entry) {
matchedCount++
for _, action := range rule.Actions {
h.pool.Dispatch(r.Context(), action, entry, 0)
queuedJobs++
}
}
}
offset += batchSize
if offset >= total || len(entries) == 0 {
break
}
}
h.writeJSON(w, http.StatusOK, map[string]interface{}{
"message": fmt.Sprintf("Rule applied to %d matching post(s). %d action(s) queued.", matchedCount, queuedJobs),
"matched_count": matchedCount,
"queued_jobs": queuedJobs,
})
}
// AI Settings Handlers
type aiSettingsResponse struct {
Enabled bool `json:"enabled"`
Provider string `json:"provider"`
BaseURL string `json:"base_url"`
APIKey string `json:"api_key"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
Prompts []ai.PromptTemplate `json:"prompts"`
}
func maskAPIKey(key string) string {
if len(key) == 0 {
return ""
}
if len(key) <= 6 {
return "••••••"
}
return key[:3] + "••••••••" + key[len(key)-4:]
}
func (h *Handler) GetAISettings(w http.ResponseWriter, r *http.Request) {
settings, err := h.db.GetSettingsMap()
if err != nil {
h.writeError(w, http.StatusInternalServerError, "failed to load settings: "+err.Error())
return
}
enabled := settings["ai_enabled"] == "true" || settings["ai_enabled"] == "1"
provider := settings["ai_provider"]
if provider == "" {
provider = "openai"
}
baseURL := settings["ai_base_url"]
if baseURL == "" {
baseURL = "https://api.openai.com/v1"
}
model := settings["ai_model"]
if model == "" {
model = "gpt-4o-mini"
}
temp, _ := strconv.ParseFloat(settings["ai_temperature"], 64)
if temp <= 0 {
temp = 0.3
}
maxTokens, _ := strconv.Atoi(settings["ai_max_tokens"])
if maxTokens <= 0 {
maxTokens = 1000
}
var prompts []ai.PromptTemplate
if promptsJSON := settings["ai_prompts"]; promptsJSON != "" {
_ = json.Unmarshal([]byte(promptsJSON), &prompts)
}
if len(prompts) == 0 {
prompts = ai.DefaultPromptTemplates()
}
h.writeJSON(w, http.StatusOK, aiSettingsResponse{
Enabled: enabled,
Provider: provider,
BaseURL: baseURL,
APIKey: maskAPIKey(settings["ai_api_key"]),
Model: model,
Temperature: temp,
MaxTokens: maxTokens,
Prompts: prompts,
})
}
func (h *Handler) UpdateAISettings(w http.ResponseWriter, r *http.Request) {
var req struct {
Enabled *bool `json:"enabled"`
Provider string `json:"provider"`
BaseURL string `json:"base_url"`
APIKey string `json:"api_key"`
Model string `json:"model"`
Temperature float64 `json:"temperature"`
MaxTokens int `json:"max_tokens"`
Prompts []ai.PromptTemplate `json:"prompts"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
h.writeError(w, http.StatusBadRequest, "invalid JSON request")
return
}
if req.Enabled != nil {
val := "false"
if *req.Enabled {
val = "true"
}
_ = h.db.SetSetting("ai_enabled", val)
}
if req.Provider != "" {
_ = h.db.SetSetting("ai_provider", req.Provider)
}
if req.BaseURL != "" {
_ = h.db.SetSetting("ai_base_url", req.BaseURL)
}
if req.Model != "" {
_ = h.db.SetSetting("ai_model", req.Model)
}
if req.Temperature > 0 {
_ = h.db.SetSetting("ai_temperature", fmt.Sprintf("%.2f", req.Temperature))
}
if req.MaxTokens > 0 {
_ = h.db.SetSetting("ai_max_tokens", strconv.Itoa(req.MaxTokens))
}
// Update API key only if not masked with bullets
if req.APIKey != "" && !strings.Contains(req.APIKey, "••••") {
_ = h.db.SetSetting("ai_api_key", strings.TrimSpace(req.APIKey))
}
if len(req.Prompts) > 0 {
promptsBytes, _ := json.Marshal(req.Prompts)
_ = h.db.SetSetting("ai_prompts", string(promptsBytes))
}
h.GetAISettings(w, r)
}
// TestAIConnection checks LLM connectivity with strict rate limiting & token caps
func (h *Handler) TestAIConnection(w http.ResponseWriter, r *http.Request) {
// Rate limiting: Max 5 test pings per minute to prevent token exhaustion
h.testRateMu.Lock()
now := time.Now()
var recentPings []time.Time
for _, t := range h.testPings {
if now.Sub(t) < time.Minute {
recentPings = append(recentPings, t)
}
}
if len(recentPings) >= 5 {
h.testRateMu.Unlock()
h.writeError(w, http.StatusTooManyRequests, "Rate limit exceeded. Maximum 5 connection tests per minute.")
return
}
h.testPings = append(recentPings, now)
h.testRateMu.Unlock()
var req struct {
Provider string `json:"provider"`
BaseURL string `json:"base_url"`
APIKey string `json:"api_key"`
Model string `json:"model"`
}
_ = json.NewDecoder(r.Body).Decode(&req)
settings, _ := h.db.GetSettingsMap()
apiKey := req.APIKey
if apiKey == "" || strings.Contains(apiKey, "••••") {
apiKey = settings["ai_api_key"]
}
baseURL := req.BaseURL
if baseURL == "" {
baseURL = settings["ai_base_url"]
}
if baseURL == "" {
baseURL = "https://api.openai.com/v1"
}
model := req.Model
if model == "" {
model = settings["ai_model"]
}
if model == "" {
model = "gpt-4o-mini"
}
cfg := ai.Config{
Enabled: true,
Provider: req.Provider,
BaseURL: baseURL,
APIKey: apiKey,
Model: model,
}
res, err := h.aiClient.TestConnection(r.Context(), cfg)
if err != nil {
h.writeJSON(w, http.StatusOK, res)
return
}
h.writeJSON(w, http.StatusOK, res)
}
internal/api/server.go
+17
View File
@@ -44,6 +44,7 @@ func NewRouter(h *Handler, authService *auth.AuthService, staticDir string) http
r.Use(authService.Middleware)
r.Get("/auth/me", h.GetMe)
r.Put("/auth/password", h.UpdatePassword)
r.Get("/stats", h.GetStats)
// Categories
@@ -61,9 +62,25 @@ func NewRouter(h *Handler, authService *auth.AuthService, staticDir string) http
// Entries
r.Get("/entries", h.ListEntries)
r.Get("/entries/{id}", h.GetEntry)
r.Put("/entries/{id}/read", h.UpdateEntryRead)
r.Put("/entries/{id}/star", h.UpdateEntryStar)
r.Post("/entries/mark-all-read", h.MarkAllRead)
r.Post("/entries/{id}/scrape", h.ScrapeEntry)
r.Get("/entries/{id}/variants", h.ListEntryVariants)
r.Get("/entries/{id}/variants/{key}/stream", h.GetVariantBlobStream)
// Rules
r.Get("/rules", h.ListRules)
r.Post("/rules", h.CreateRule)
r.Put("/rules/{id}", h.UpdateRule)
r.Delete("/rules/{id}", h.DeleteRule)
r.Post("/rules/{id}/apply", h.ApplyRule)
// AI Settings
r.Get("/settings/ai", h.GetAISettings)
r.Put("/settings/ai", h.UpdateAISettings)
r.Post("/settings/ai/test", h.TestAIConnection)
// OPML
r.Post("/opml/import", h.ImportOPML)
internal/auth/auth.go
+23
View File
@@ -84,6 +84,29 @@ func (s *AuthService) Authenticate(username, password string) (string, error) {
return tokenString, nil
}
func (s *AuthService) UpdatePassword(userID, oldPassword, newPassword string) error {
var currentHash string
err := s.db.QueryRow("SELECT password_hash FROM users WHERE id = ?", userID).Scan(&currentHash)
if err != nil {
return errors.New("user not found")
}
if err := bcrypt.CompareHashAndPassword([]byte(currentHash), []byte(oldPassword)); err != nil {
return errors.New("incorrect current password")
}
if len(strings.TrimSpace(newPassword)) < 6 {
return errors.New("new password must be at least 6 characters long")
}
newHash, err := bcrypt.GenerateFromPassword([]byte(newPassword), bcrypt.DefaultCost)
if err != nil {
return fmt.Errorf("failed to hash new password: %w", err)
}
return s.db.UpdateUserPassword(userID, string(newHash))
}
func (s *AuthService) ValidateToken(tokenString string) (*Claims, error) {
token, err := jwt.ParseWithClaims(tokenString, &Claims{}, func(token *jwt.Token) (interface{}, error) {
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
internal/auth/auth_test.go
+87
View File
@@ -0,0 +1,87 @@
package auth
import (
"os"
"path/filepath"
"testing"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/config"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
)
func TestAuthService(t *testing.T) {
tempDir, err := os.MkdirTemp("", "rsspipe-auth-test-*")
if err != nil {
t.Fatalf("failed to create temp dir: %v", err)
}
defer func() {
_ = os.RemoveAll(tempDir)
}()
dbPath := filepath.Join(tempDir, "auth_test.db")
database, err := db.InitDB(dbPath)
if err != nil {
t.Fatalf("InitDB failed: %v", err)
}
defer func() {
_ = database.Close()
}()
cfg := &config.Config{
AdminUsername: "admin",
AdminPassword: "initial_password_123",
JWTSecret: "super-secret-jwt-key-for-test",
}
authService := NewAuthService(database, cfg)
// 1. Bootstrap Admin
if err := authService.BootstrapAdmin(); err != nil {
t.Fatalf("BootstrapAdmin failed: %v", err)
}
// 2. Authenticate with initial password
token, err := authService.Authenticate("admin", "initial_password_123")
if err != nil {
t.Fatalf("Authenticate failed: %v", err)
}
if token == "" {
t.Fatalf("expected non-empty token")
}
// 3. Validate Token
claims, err := authService.ValidateToken(token)
if err != nil {
t.Fatalf("ValidateToken failed: %v", err)
}
if claims.Username != "admin" {
t.Errorf("expected username admin, got %s", claims.Username)
}
// 4. Update Password
if err := authService.UpdatePassword(claims.UserID, "wrong_old_pwd", "new_secure_pwd_456"); err == nil {
t.Errorf("expected error with incorrect old password")
}
if err := authService.UpdatePassword(claims.UserID, "initial_password_123", "short"); err == nil {
t.Errorf("expected error with too short new password")
}
if err := authService.UpdatePassword(claims.UserID, "initial_password_123", "new_secure_pwd_456"); err != nil {
t.Fatalf("UpdatePassword failed: %v", err)
}
// 5. Authenticate with new password
newToken, err := authService.Authenticate("admin", "new_secure_pwd_456")
if err != nil {
t.Fatalf("Authenticate with new password failed: %v", err)
}
if newToken == "" {
t.Fatalf("expected non-empty token for new password")
}
// 6. Old password should now fail
if _, err := authService.Authenticate("admin", "initial_password_123"); err == nil {
t.Errorf("expected failure when authenticating with old password")
}
}
internal/crawler/crawler.go
+43 -14
View File
@@ -3,35 +3,41 @@ package crawler
import (
"context"
"fmt"
"log"
"sync"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/events"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/parser"
"go.uber.org/zap"
)
type Crawler struct {
db *db.DB
parser *parser.Parser
bus *events.EventBus
intervalMinutes int
workerCount int
isRefreshing bool
mu sync.Mutex
log *zap.Logger
}
func NewCrawler(database *db.DB, intervalMinutes, workers int) *Crawler {
func NewCrawler(database *db.DB, bus *events.EventBus, intervalMinutes, workers int) *Crawler {
return &Crawler{
db: database,
parser: parser.NewParser(),
bus: bus,
intervalMinutes: intervalMinutes,
workerCount: workers,
log: logger.Named("crawler"),
}
}
func (c *Crawler) Start(ctx context.Context) {
log.Printf("==> Starting RSS background crawler scheduler (Workers: %d)\n", c.workerCount)
c.log.Info("Starting RSS background crawler scheduler", zap.Int("worker_count", c.workerCount))
// Run initial fetch asynchronously after 2 seconds startup
go func() {
@@ -72,7 +78,7 @@ func (c *Crawler) RefreshDue(ctx context.Context) {
feeds, err := c.db.ListFeeds()
if err != nil {
log.Printf("Crawler error listing feeds: %v\n", err)
c.log.Error("Crawler error listing feeds", zap.Error(err))
return
}
@@ -93,7 +99,7 @@ func (c *Crawler) RefreshDue(ctx context.Context) {
return
}
log.Printf("Crawler: Refreshing %d due feeds (out of %d total)...\n", len(dueFeeds), len(feeds))
c.log.Info("Crawler refreshing due feeds", zap.Int("due_count", len(dueFeeds)), zap.Int("total_count", len(feeds)))
c.runWorkerPool(ctx, dueFeeds)
}
@@ -102,7 +108,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) {
c.mu.Lock()
if c.isRefreshing {
c.mu.Unlock()
log.Println("Crawler: RefreshAll already in progress, skipping")
c.log.Debug("Crawler RefreshAll already in progress, skipping")
return
}
c.isRefreshing = true
@@ -116,7 +122,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) {
feeds, err := c.db.ListFeeds()
if err != nil {
log.Printf("Crawler error listing feeds: %v\n", err)
c.log.Error("Crawler error listing feeds", zap.Error(err))
return
}
@@ -124,7 +130,7 @@ func (c *Crawler) RefreshAll(ctx context.Context) {
return
}
log.Printf("Crawler: Refreshing all %d feeds...\n", len(feeds))
c.log.Info("Crawler refreshing all feeds", zap.Int("count", len(feeds)))
c.runWorkerPool(ctx, feeds)
}
@@ -148,7 +154,7 @@ func (c *Crawler) runWorkerPool(ctx context.Context, feeds []models.Feed) {
close(jobs)
wg.Wait()
log.Printf("Crawler: Completed refresh of %d feeds.\n", len(feeds))
c.log.Info("Crawler completed refresh batch", zap.Int("feed_count", len(feeds)))
}
func (c *Crawler) RefreshSingle(ctx context.Context, feedID string) error {
@@ -169,7 +175,11 @@ func (c *Crawler) fetchSingleFeed(ctx context.Context, f models.Feed) error {
res, err := c.parser.FetchFeed(feedCtx, f.FeedURL, f.ETag, f.LastModified)
if err != nil {
log.Printf("Crawler error fetching [%s]: %v\n", f.Title, err)
c.log.Warn("Crawler error fetching feed",
zap.String("feed_title", f.Title),
zap.String("feed_url", f.FeedURL),
zap.Error(err),
)
_ = c.db.UpdateFeedFetchStatus(f.ID, f.ETag, f.LastModified, err.Error())
return err
}
@@ -180,11 +190,30 @@ func (c *Crawler) fetchSingleFeed(ctx context.Context, f models.Feed) error {
}
// Upsert entries
inserted, err := c.db.UpsertFeedEntries(f.ID, res.Entries)
newEntries, err := c.db.UpsertFeedEntries(f.ID, res.Entries)
if err != nil {
log.Printf("Crawler error upserting entries for [%s]: %v\n", f.Title, err)
} else if inserted > 0 {
log.Printf("Crawler [%s]: inserted %d new entries\n", f.Title, inserted)
c.log.Error("Crawler error upserting entries",
zap.String("feed_title", f.Title),
zap.Error(err),
)
} else if len(newEntries) > 0 {
c.log.Info("Crawler inserted new entries",
zap.String("feed_title", f.Title),
zap.Int("new_entries", len(newEntries)),
)
// Publish on_post_created events for newly inserted entries
if c.bus != nil {
for _, entry := range newEntries {
entry.FeedTitle = f.Title
entry.FeedSiteURL = f.SiteURL
c.bus.Publish(ctx, events.Event{
Type: events.EventPostCreated,
Entry: entry,
Depth: 0,
})
}
}
}
_ = c.db.UpdateFeedFetchStatus(f.ID, res.ETag, res.LastModified, "")
internal/db/db.go
+511 -81
View File
@@ -114,24 +114,63 @@ func (d *DB) migrate() error {
CREATE INDEX IF NOT EXISTS idx_entries_is_starred ON feed_entries(is_starred);
CREATE INDEX IF NOT EXISTS idx_entries_share_uuid ON feed_entries(share_uuid);
CREATE TABLE IF NOT EXISTS entry_variants (
entry_id TEXT NOT NULL,
variant_key TEXT NOT NULL,
content_type TEXT NOT NULL,
text_value TEXT,
blob_path TEXT,
metadata TEXT DEFAULT '{}',
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL,
PRIMARY KEY (entry_id, variant_key),
FOREIGN KEY(entry_id) REFERENCES feed_entries(id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_entry_variants_key ON entry_variants(variant_key);
CREATE TABLE IF NOT EXISTS rules (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
rule_type TEXT NOT NULL,
is_active BOOLEAN DEFAULT 1,
trigger_type TEXT NOT NULL,
trigger_config TEXT DEFAULT '{}',
trigger TEXT NOT NULL,
conditions TEXT DEFAULT '[]',
match_mode TEXT DEFAULT 'all',
actions TEXT DEFAULT '[]',
is_active BOOLEAN DEFAULT 1,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL
);
CREATE TABLE IF NOT EXISTS system_settings (
key TEXT PRIMARY KEY,
value TEXT NOT NULL,
updated_at DATETIME NOT NULL
);
`
if _, err := d.Exec(schema); err != nil {
return err
}
// Migration for timeout_seconds column if older DB exists
_, _ = d.Exec("ALTER TABLE feeds ADD COLUMN timeout_seconds INTEGER DEFAULT 10")
// Migration for rules table schema if older table exists
var testCol string
err := d.QueryRow("SELECT trigger FROM rules LIMIT 1").Scan(&testCol)
if err != nil && err != sql.ErrNoRows {
_, _ = d.Exec("DROP TABLE IF EXISTS rules")
_, _ = d.Exec(`
CREATE TABLE IF NOT EXISTS rules (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
trigger TEXT NOT NULL,
conditions TEXT DEFAULT '[]',
match_mode TEXT DEFAULT 'all',
actions TEXT DEFAULT '[]',
is_active BOOLEAN DEFAULT 1,
created_at DATETIME NOT NULL,
updated_at DATETIME NOT NULL
);
`)
}
return nil
}
@@ -173,6 +212,15 @@ func (d *DB) CreateUser(username, passwordHash string) (*models.User, error) {
return u, nil
}
func (d *DB) UpdateUserPassword(userID, passwordHash string) error {
now := time.Now().UTC()
_, err := d.Exec(
"UPDATE users SET password_hash = ?, updated_at = ? WHERE id = ?",
passwordHash, now, userID,
)
return err
}
// Category operations
func (d *DB) ListCategories() ([]models.Category, error) {
query := `
@@ -465,51 +513,67 @@ func (d *DB) DeleteFeed(id string) error {
}
// Feed Entries operations
func (d *DB) UpsertFeedEntries(feedID string, entries []models.FeedEntry) (int, error) {
func (d *DB) UpsertFeedEntries(feedID string, entries []models.FeedEntry) ([]models.FeedEntry, error) {
if len(entries) == 0 {
return 0, nil
return nil, nil
}
stmt, err := d.Prepare(`
var newEntries []models.FeedEntry
now := time.Now().UTC()
insertStmt, err := d.Prepare(`
INSERT INTO feed_entries (
id, feed_id, guid, url, title, author, content, raw_content, published_at, is_read, is_starred, metadata, created_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?, ?)
ON CONFLICT(feed_id, guid) DO UPDATE SET
title = excluded.title,
content = CASE WHEN feed_entries.content = '' THEN excluded.content ELSE feed_entries.content END,
url = excluded.url,
author = excluded.author
`)
if err != nil {
return 0, err
return nil, err
}
defer func() {
_ = stmt.Close()
_ = insertStmt.Close()
}()
insertedCount := 0
for _, e := range entries {
id := uuid.New().String()
now := time.Now().UTC()
pubAt := e.PublishedAt
if pubAt.IsZero() {
pubAt = now
}
metaBytes, _ := json.Marshal(e.Metadata)
if len(metaBytes) == 0 {
metaBytes = []byte("{}")
}
updateStmt, err := d.Prepare(`
UPDATE feed_entries
SET title = ?, url = ?, author = ?, content = ?, raw_content = ?
WHERE feed_id = ? AND guid = ?
`)
if err != nil {
return nil, err
}
defer func() {
_ = updateStmt.Close()
}()
res, err := stmt.Exec(
id, feedID, e.GUID, e.URL, e.Title, e.Author, e.Content, e.RawContent, pubAt, string(metaBytes), now,
)
if err == nil {
if n, _ := res.RowsAffected(); n > 0 {
insertedCount++
for _, e := range entries {
var existingID string
err := d.QueryRow("SELECT id FROM feed_entries WHERE feed_id = ? AND guid = ?", feedID, e.GUID).Scan(&existingID)
switch err {
case sql.ErrNoRows:
e.ID = uuid.New().String()
e.FeedID = feedID
pubAt := e.PublishedAt
if pubAt.IsZero() {
pubAt = now
}
metaBytes, _ := json.Marshal(e.Metadata)
if len(metaBytes) == 0 {
metaBytes = []byte("{}")
}
_, err := insertStmt.Exec(
e.ID, feedID, e.GUID, e.URL, e.Title, e.Author, e.Content, e.RawContent, pubAt, string(metaBytes), now,
)
if err == nil {
e.CreatedAt = now
newEntries = append(newEntries, e)
}
case nil:
_, _ = updateStmt.Exec(e.Title, e.URL, e.Author, e.Content, e.RawContent, feedID, e.GUID)
}
}
return insertedCount, nil
return newEntries, nil
}
type EntryFilter struct {
@@ -521,67 +585,98 @@ type EntryFilter struct {
Offset int
}
func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) {
where := []string{"1=1"}
args := []interface{}{}
func (d *DB) GetEntry(id string) (*models.FeedEntry, error) {
query := `
SELECT
e.id, e.feed_id, f.title as feed_title, COALESCE(f.site_url, '') as feed_site_url,
e.guid, e.url, e.title, e.author, e.content, e.raw_content,
e.published_at, e.is_read, e.is_starred, e.share_uuid, e.metadata, e.created_at
FROM feed_entries e
JOIN feeds f ON f.id = e.feed_id
WHERE e.id = ?
`
var e models.FeedEntry
var author, rawContent, shareUUID, metaStr sql.NullString
err := d.QueryRow(query, id).Scan(
&e.ID, &e.FeedID, &e.FeedTitle, &e.FeedSiteURL,
&e.GUID, &e.URL, &e.Title, &author, &e.Content, &rawContent,
&e.PublishedAt, &e.IsRead, &e.IsStarred, &shareUUID, &metaStr, &e.CreatedAt,
)
if err != nil {
return nil, err
}
if author.Valid {
e.Author = author.String
}
if rawContent.Valid {
e.RawContent = rawContent.String
}
if shareUUID.Valid {
s := shareUUID.String
e.ShareUUID = &s
}
e.Metadata = make(map[string]interface{})
if metaStr.Valid && metaStr.String != "" {
_ = json.Unmarshal([]byte(metaStr.String), &e.Metadata)
}
switch f.View {
// Load variants
variants, _ := d.ListVariantsForEntry(e.ID)
e.Variants = variants
return &e, nil
}
func (d *DB) ListEntries(filter EntryFilter) ([]models.FeedEntry, int, error) {
var whereClauses []string
var args []interface{}
switch filter.View {
case "inbox":
where = append(where, "e.is_read = 0")
whereClauses = append(whereClauses, "e.is_read = 0")
case "starred":
where = append(where, "e.is_starred = 1")
whereClauses = append(whereClauses, "e.is_starred = 1")
}
if f.FeedID != "" {
where = append(where, "e.feed_id = ?")
args = append(args, f.FeedID)
if filter.FeedID != "" {
whereClauses = append(whereClauses, "e.feed_id = ?")
args = append(args, filter.FeedID)
}
if f.CategoryID != "" {
where = append(where, "f.category_id = ?")
args = append(args, f.CategoryID)
if filter.CategoryID != "" {
whereClauses = append(whereClauses, "f.category_id = ?")
args = append(args, filter.CategoryID)
}
if f.Search != "" {
searchTerms := strings.Fields(f.Search)
for _, term := range searchTerms {
lower := strings.ToLower(term)
if strings.HasPrefix(lower, "is:unread") {
where = append(where, "e.is_read = 0")
} else if strings.HasPrefix(lower, "is:read") {
where = append(where, "e.is_read = 1")
} else if strings.HasPrefix(lower, "is:starred") {
where = append(where, "e.is_starred = 1")
} else if strings.HasPrefix(lower, "feed:") {
feedVal := strings.TrimPrefix(term, "feed:")
where = append(where, "(f.title LIKE ? OR f.feed_url LIKE ?)")
args = append(args, "%"+feedVal+"%", "%"+feedVal+"%")
} else {
where = append(where, "(e.title LIKE ? OR e.content LIKE ? OR e.author LIKE ?)")
args = append(args, "%"+term+"%", "%"+term+"%", "%"+term+"%")
}
}
if filter.Search != "" {
whereClauses = append(whereClauses, "(e.title LIKE ? OR e.content LIKE ? OR e.author LIKE ?)")
pattern := "%" + filter.Search + "%"
args = append(args, pattern, pattern, pattern)
}
whereClause := strings.Join(where, " AND ")
whereSQL := ""
if len(whereClauses) > 0 {
whereSQL = "WHERE " + strings.Join(whereClauses, " AND ")
}
// Count total
countQuery := fmt.Sprintf(`
SELECT COUNT(*)
SELECT COUNT(e.id)
FROM feed_entries e
JOIN feeds f ON f.id = e.feed_id
WHERE %s
`, whereClause)
%s
`, whereSQL)
var total int
if err := d.QueryRow(countQuery, args...).Scan(&total); err != nil {
err := d.QueryRow(countQuery, args...).Scan(&total)
if err != nil {
return nil, 0, err
}
limit := f.Limit
if limit <= 0 || limit > 100 {
limit = 50
limit := filter.Limit
if limit <= 0 || limit > 500 {
limit = 100
}
offset := f.Offset
offset := filter.Offset
if offset < 0 {
offset = 0
}
@@ -593,10 +688,10 @@ func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) {
e.published_at, e.is_read, e.is_starred, e.share_uuid, e.metadata, e.created_at
FROM feed_entries e
JOIN feeds f ON f.id = e.feed_id
WHERE %s
ORDER BY e.published_at DESC
%s
ORDER BY e.published_at DESC, e.created_at DESC
LIMIT ? OFFSET ?
`, whereClause)
`, whereSQL)
queryArgs := append(args, limit, offset)
rows, err := d.Query(query, queryArgs...)
@@ -610,13 +705,13 @@ func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) {
var entries []models.FeedEntry
for rows.Next() {
var e models.FeedEntry
var author, rawContent, metaStr sql.NullString
var shareUUID sql.NullString
if err := rows.Scan(
var author, rawContent, shareUUID, metaStr sql.NullString
err := rows.Scan(
&e.ID, &e.FeedID, &e.FeedTitle, &e.FeedSiteURL,
&e.GUID, &e.URL, &e.Title, &author, &e.Content, &rawContent,
&e.PublishedAt, &e.IsRead, &e.IsStarred, &shareUUID, &metaStr, &e.CreatedAt,
); err != nil {
)
if err != nil {
return nil, 0, err
}
if author.Valid {
@@ -636,6 +731,22 @@ func (d *DB) ListEntries(f EntryFilter) ([]models.FeedEntry, int, error) {
entries = append(entries, e)
}
// Batch-load variants for all returned entries
if len(entries) > 0 {
ids := make([]string, len(entries))
for i, e := range entries {
ids[i] = e.ID
}
variantMap, err := d.batchLoadVariants(ids)
if err == nil {
for i := range entries {
if v, ok := variantMap[entries[i].ID]; ok {
entries[i].Variants = v
}
}
}
}
return entries, total, nil
}
@@ -666,6 +777,281 @@ func (d *DB) MarkAllRead(feedID, categoryID *string) error {
return err
}
// Entry Variants CRUD
func (d *DB) GetVariant(entryID, key string) (*models.EntryVariant, error) {
query := `
SELECT entry_id, variant_key, content_type, COALESCE(text_value, ''), COALESCE(blob_path, ''), metadata, created_at, updated_at
FROM entry_variants
WHERE entry_id = ? AND variant_key = ?
`
var v models.EntryVariant
var metaStr sql.NullString
err := d.QueryRow(query, entryID, key).Scan(
&v.EntryID, &v.VariantKey, &v.ContentType, &v.TextValue, &v.BlobPath, &metaStr, &v.CreatedAt, &v.UpdatedAt,
)
if err != nil {
return nil, err
}
v.Metadata = make(map[string]interface{})
if metaStr.Valid && metaStr.String != "" {
_ = json.Unmarshal([]byte(metaStr.String), &v.Metadata)
}
return &v, nil
}
func (d *DB) ListVariantsForEntry(entryID string) (map[string]models.EntryVariant, error) {
query := `
SELECT entry_id, variant_key, content_type, COALESCE(text_value, ''), COALESCE(blob_path, ''), metadata, created_at, updated_at
FROM entry_variants
WHERE entry_id = ?
`
rows, err := d.Query(query, entryID)
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
variants := make(map[string]models.EntryVariant)
for rows.Next() {
var v models.EntryVariant
var metaStr sql.NullString
err := rows.Scan(
&v.EntryID, &v.VariantKey, &v.ContentType, &v.TextValue, &v.BlobPath, &metaStr, &v.CreatedAt, &v.UpdatedAt,
)
if err != nil {
return nil, err
}
v.Metadata = make(map[string]interface{})
if metaStr.Valid && metaStr.String != "" {
_ = json.Unmarshal([]byte(metaStr.String), &v.Metadata)
}
variants[v.VariantKey] = v
}
return variants, nil
}
// batchLoadVariants loads lightweight variants (tags, ai_summary keys, etc.) for multiple entries.
// Excludes large text variants like readability_full_text for performance.
func (d *DB) batchLoadVariants(entryIDs []string) (map[string]map[string]models.EntryVariant, error) {
if len(entryIDs) == 0 {
return nil, nil
}
placeholders := make([]string, len(entryIDs))
args := make([]interface{}, len(entryIDs))
for i, id := range entryIDs {
placeholders[i] = "?"
args[i] = id
}
query := `
SELECT entry_id, variant_key, content_type,
CASE WHEN variant_key IN ('readability_full_text') THEN '' ELSE COALESCE(text_value, '') END,
COALESCE(blob_path, ''), metadata, created_at, updated_at
FROM entry_variants
WHERE entry_id IN (` + strings.Join(placeholders, ",") + `)
`
rows, err := d.Query(query, args...)
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
result := make(map[string]map[string]models.EntryVariant)
for rows.Next() {
var v models.EntryVariant
var metaStr sql.NullString
err := rows.Scan(
&v.EntryID, &v.VariantKey, &v.ContentType, &v.TextValue, &v.BlobPath, &metaStr, &v.CreatedAt, &v.UpdatedAt,
)
if err != nil {
return nil, err
}
v.Metadata = make(map[string]interface{})
if metaStr.Valid && metaStr.String != "" {
_ = json.Unmarshal([]byte(metaStr.String), &v.Metadata)
}
if result[v.EntryID] == nil {
result[v.EntryID] = make(map[string]models.EntryVariant)
}
result[v.EntryID][v.VariantKey] = v
}
return result, nil
}
func (d *DB) SetVariant(v models.EntryVariant) error {
now := time.Now().UTC()
metaBytes, _ := json.Marshal(v.Metadata)
if len(metaBytes) == 0 {
metaBytes = []byte("{}")
}
query := `
INSERT INTO entry_variants (
entry_id, variant_key, content_type, text_value, blob_path, metadata, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(entry_id, variant_key) DO UPDATE SET
content_type = excluded.content_type,
text_value = excluded.text_value,
blob_path = excluded.blob_path,
metadata = excluded.metadata,
updated_at = excluded.updated_at
`
_, err := d.Exec(query, v.EntryID, v.VariantKey, v.ContentType, v.TextValue, v.BlobPath, string(metaBytes), now, now)
return err
}
func (d *DB) DeleteVariant(entryID, key string) error {
_, err := d.Exec("DELETE FROM entry_variants WHERE entry_id = ? AND variant_key = ?", entryID, key)
return err
}
// Rules CRUD
func (d *DB) ListRules() ([]models.Rule, error) {
query := `
SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at
FROM rules
ORDER BY created_at ASC
`
rows, err := d.Query(query)
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
var rules []models.Rule
for rows.Next() {
var r models.Rule
var condsStr, actionsStr string
err := rows.Scan(
&r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt,
)
if err != nil {
return nil, err
}
_ = json.Unmarshal([]byte(condsStr), &r.Conditions)
_ = json.Unmarshal([]byte(actionsStr), &r.Actions)
rules = append(rules, r)
}
return rules, nil
}
func (d *DB) GetActiveRules(trigger string) ([]models.Rule, error) {
query := `
SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at
FROM rules
WHERE is_active = 1 AND (trigger = ? OR trigger = '*')
ORDER BY created_at ASC
`
rows, err := d.Query(query, trigger)
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
var rules []models.Rule
for rows.Next() {
var r models.Rule
var condsStr, actionsStr string
err := rows.Scan(
&r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt,
)
if err != nil {
return nil, err
}
_ = json.Unmarshal([]byte(condsStr), &r.Conditions)
_ = json.Unmarshal([]byte(actionsStr), &r.Actions)
rules = append(rules, r)
}
return rules, nil
}
func (d *DB) GetRule(id string) (*models.Rule, error) {
query := `
SELECT id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at
FROM rules
WHERE id = ?
`
var r models.Rule
var condsStr, actionsStr string
err := d.QueryRow(query, id).Scan(
&r.ID, &r.Name, &r.Trigger, &condsStr, &r.MatchMode, &actionsStr, &r.IsActive, &r.CreatedAt, &r.UpdatedAt,
)
if err != nil {
return nil, err
}
_ = json.Unmarshal([]byte(condsStr), &r.Conditions)
_ = json.Unmarshal([]byte(actionsStr), &r.Actions)
return &r, nil
}
func (d *DB) CreateRule(name, trigger, matchMode string, conditions []models.RuleCondition, actions []models.RuleAction) (*models.Rule, error) {
id := uuid.New().String()
now := time.Now().UTC()
if matchMode == "" {
matchMode = "all"
}
condsBytes, _ := json.Marshal(conditions)
actionsBytes, _ := json.Marshal(actions)
query := `
INSERT INTO rules (
id, name, trigger, conditions, match_mode, actions, is_active, created_at, updated_at
) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?)
`
_, err := d.Exec(query, id, name, trigger, string(condsBytes), matchMode, string(actionsBytes), now, now)
if err != nil {
return nil, err
}
return &models.Rule{
ID: id,
Name: name,
Trigger: trigger,
Conditions: conditions,
MatchMode: matchMode,
Actions: actions,
IsActive: true,
CreatedAt: now,
UpdatedAt: now,
}, nil
}
func (d *DB) UpdateRule(id string, name, trigger, matchMode string, conditions []models.RuleCondition, actions []models.RuleAction, isActive bool) (*models.Rule, error) {
now := time.Now().UTC()
if matchMode == "" {
matchMode = "all"
}
condsBytes, _ := json.Marshal(conditions)
actionsBytes, _ := json.Marshal(actions)
query := `
UPDATE rules
SET name = ?, trigger = ?, conditions = ?, match_mode = ?, actions = ?, is_active = ?, updated_at = ?
WHERE id = ?
`
res, err := d.Exec(query, name, trigger, string(condsBytes), matchMode, string(actionsBytes), isActive, now, id)
if err != nil {
return nil, err
}
if n, _ := res.RowsAffected(); n == 0 {
return nil, fmt.Errorf("rule not found")
}
return d.GetRule(id)
}
func (d *DB) DeleteRule(id string) error {
_, err := d.Exec("DELETE FROM rules WHERE id = ?", id)
return err
}
func (d *DB) GetStats() (*models.Stats, error) {
var s models.Stats
err := d.QueryRow("SELECT COUNT(*) FROM feeds").Scan(&s.TotalFeeds)
@@ -684,5 +1070,49 @@ func (d *DB) GetStats() (*models.Stats, error) {
if err != nil {
return nil, err
}
_ = d.QueryRow("SELECT COUNT(*) FROM rules").Scan(&s.TotalRules)
return &s, nil
}
// System Settings CRUD
func (d *DB) GetSetting(key string) (string, error) {
var val string
err := d.QueryRow("SELECT value FROM system_settings WHERE key = ?", key).Scan(&val)
if err != nil {
if err == sql.ErrNoRows {
return "", nil
}
return "", err
}
return val, nil
}
func (d *DB) SetSetting(key, value string) error {
now := time.Now().UTC()
_, err := d.Exec(`
INSERT INTO system_settings (key, value, updated_at)
VALUES (?, ?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at
`, key, value, now)
return err
}
func (d *DB) GetSettingsMap() (map[string]string, error) {
rows, err := d.Query("SELECT key, value FROM system_settings")
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
res := make(map[string]string)
for rows.Next() {
var k, v string
if err := rows.Scan(&k, &v); err != nil {
return nil, err
}
res[k] = v
}
return res, nil
}
internal/db/db_test.go
+74 -4
View File
@@ -72,8 +72,8 @@ func TestDatabaseOperations(t *testing.T) {
if err != nil {
t.Fatalf("UpsertFeedEntries failed: %v", err)
}
if inserted != 2 {
t.Fatalf("expected 2 inserted entries, got %d", inserted)
if len(inserted) != 2 {
t.Fatalf("expected 2 inserted entries, got %d", len(inserted))
}
// 5. List entries
@@ -94,12 +94,82 @@ func TestDatabaseOperations(t *testing.T) {
t.Fatalf("UpdateEntryStarred failed: %v", err)
}
// 7. Verify Stats
// 7. Test Variant CRUD
variant := models.EntryVariant{
EntryID: firstID,
VariantKey: "readability_full_text",
ContentType: "html",
TextValue: "<div>Clean full text content</div>",
Metadata: map[string]interface{}{
"scraped_at": time.Now().UTC().Format(time.RFC3339),
"length": 100,
},
}
if err := database.SetVariant(variant); err != nil {
t.Fatalf("SetVariant failed: %v", err)
}
fetchedVariant, err := database.GetVariant(firstID, "readability_full_text")
if err != nil {
t.Fatalf("GetVariant failed: %v", err)
}
if fetchedVariant.TextValue != "<div>Clean full text content</div>" {
t.Fatalf("expected variant text, got %s", fetchedVariant.TextValue)
}
// Test GetEntry with variants
entryWithVariants, err := database.GetEntry(firstID)
if err != nil {
t.Fatalf("GetEntry failed: %v", err)
}
if len(entryWithVariants.Variants) != 1 {
t.Fatalf("expected 1 variant in GetEntry, got %d", len(entryWithVariants.Variants))
}
// 8. Test Rules CRUD
rule, err := database.CreateRule(
"Auto Scrape Rule",
"on_post_created",
"all",
[]models.RuleCondition{
{Field: "feed_id", Operator: "eq", Value: feed.ID},
},
[]models.RuleAction{
{Type: "readability_scrape"},
},
)
if err != nil {
t.Fatalf("CreateRule failed: %v", err)
}
if rule.Name != "Auto Scrape Rule" {
t.Errorf("expected rule name Auto Scrape Rule, got %s", rule.Name)
}
activeRules, err := database.GetActiveRules("on_post_created")
if err != nil {
t.Fatalf("GetActiveRules failed: %v", err)
}
if len(activeRules) != 1 {
t.Fatalf("expected 1 active rule, got %d", len(activeRules))
}
// 9. Verify Stats
stats, err := database.GetStats()
if err != nil {
t.Fatalf("GetStats failed: %v", err)
}
if stats.TotalFeeds != 1 || stats.TotalUnread != 1 || stats.TotalStarred != 1 || stats.TotalCategories != 1 {
if stats.TotalFeeds != 1 || stats.TotalUnread != 1 || stats.TotalStarred != 1 || stats.TotalRules != 1 {
t.Errorf("unexpected stats: %+v", stats)
}
// 8. Test UpdateUserPassword
newHash := "$2a$10$updatedhashvalue"
if err := database.UpdateUserPassword(u.ID, newHash); err != nil {
t.Fatalf("UpdateUserPassword failed: %v", err)
}
updatedUser, err := database.GetUserByUsername("admin")
if err != nil {
t.Fatalf("GetUserByUsername failed: %v", err)
}
if updatedUser.PasswordHash != newHash {
t.Errorf("expected hash %s, got %s", newHash, updatedUser.PasswordHash)
}
}
internal/events/bus.go
+92
View File
@@ -0,0 +1,92 @@
package events
import (
"context"
"sync"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"go.uber.org/zap"
)
type EventType string
const (
EventPostCreated EventType = "on_post_created"
EventPostUpdated EventType = "on_post_updated"
EventVariantUpdated EventType = "on_variant_updated"
EventManualTrigger EventType = "manual"
)
type Event struct {
Type EventType
Entry models.FeedEntry
VariantKey string // populated when Type == EventVariantUpdated
Depth int
DirectAttrs map[string]interface{}
}
type EventHandler func(ctx context.Context, evt Event) error
type EventBus struct {
mu sync.RWMutex
subscribers map[EventType][]EventHandler
log *zap.Logger
maxDepth int
}
func NewEventBus() *EventBus {
return &EventBus{
subscribers: make(map[EventType][]EventHandler),
log: logger.Named("eventbus"),
maxDepth: 5,
}
}
func (b *EventBus) Subscribe(eventType EventType, handler EventHandler) {
b.mu.Lock()
defer b.mu.Unlock()
b.subscribers[eventType] = append(b.subscribers[eventType], handler)
}
func (b *EventBus) Publish(ctx context.Context, evt Event) {
if evt.Depth > b.maxDepth {
b.log.Warn("Event dropped: recursion depth limit exceeded",
zap.String("type", string(evt.Type)),
zap.String("entry_id", evt.Entry.ID),
zap.Int("depth", evt.Depth),
)
return
}
b.mu.RLock()
handlers := append([]EventHandler{}, b.subscribers[evt.Type]...)
wildcardHandlers := append([]EventHandler{}, b.subscribers["*"]...)
b.mu.RUnlock()
b.log.Debug("Event published",
zap.String("type", string(evt.Type)),
zap.String("entry_id", evt.Entry.ID),
zap.String("variant_key", evt.VariantKey),
zap.Int("depth", evt.Depth),
zap.Int("subscriber_count", len(handlers)+len(wildcardHandlers)),
)
allHandlers := append(handlers, wildcardHandlers...)
for _, h := range allHandlers {
go func(fn EventHandler) {
defer func() {
if r := recover(); r != nil {
b.log.Error("Event handler panic recovered", zap.Any("panic", r))
}
}()
if err := fn(ctx, evt); err != nil {
b.log.Error("Event handler error",
zap.String("type", string(evt.Type)),
zap.String("entry_id", evt.Entry.ID),
zap.Error(err),
)
}
}(h)
}
}
internal/logger/logger.go
+74
View File
@@ -0,0 +1,74 @@
package logger
import (
"os"
"strings"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
var globalLogger *zap.Logger
var globalSugared *zap.SugaredLogger
func init() {
InitLogger(os.Getenv("LOG_LEVEL"))
}
// InitLogger initializes the global Zap logger with proper formatting and log levels.
func InitLogger(level string) *zap.Logger {
zapLevel := zapcore.InfoLevel
switch strings.ToLower(level) {
case "debug":
zapLevel = zapcore.DebugLevel
case "warn", "warning":
zapLevel = zapcore.WarnLevel
case "error":
zapLevel = zapcore.ErrorLevel
}
encoderConfig := zapcore.EncoderConfig{
TimeKey: "time",
LevelKey: "level",
NameKey: "logger",
CallerKey: "caller",
MessageKey: "msg",
StacktraceKey: "stacktrace",
LineEnding: zapcore.DefaultLineEnding,
EncodeLevel: zapcore.CapitalColorLevelEncoder,
EncodeTime: zapcore.TimeEncoderOfLayout("15:04:05.000"),
EncodeDuration: zapcore.StringDurationEncoder,
EncodeCaller: zapcore.ShortCallerEncoder,
}
core := zapcore.NewCore(
zapcore.NewConsoleEncoder(encoderConfig),
zapcore.AddSync(os.Stdout),
zapLevel,
)
globalLogger = zap.New(core, zap.AddCaller())
globalSugared = globalLogger.Sugar()
return globalLogger
}
// L returns the global zap logger instance.
func L() *zap.Logger {
if globalLogger == nil {
InitLogger("info")
}
return globalLogger
}
// S returns the global sugared logger instance.
func S() *zap.SugaredLogger {
if globalSugared == nil {
InitLogger("info")
}
return globalSugared
}
// Named creates a child logger with a component name tag.
func Named(name string) *zap.Logger {
return L().Named(name)
}
internal/logger/logger_test.go
+15
View File
@@ -0,0 +1,15 @@
package logger
import (
"testing"
)
func TestLoggerInit(t *testing.T) {
log := InitLogger("debug")
if log == nil {
t.Fatal("expected non-nil logger")
}
L().Info("Test structured logging message")
S().Infof("Test sugared logging message with %s", "parameter")
Named("test-component").Debug("Test component debug log")
}
internal/models/models.go
+62 -15
View File
@@ -45,22 +45,62 @@ type Feed struct {
// FeedEntry represents an individual article or digest entry
type FeedEntry struct {
ID string `json:"id"`
FeedID string `json:"feed_id"`
FeedTitle string `json:"feed_title"`
FeedSiteURL string `json:"feed_site_url,omitempty"`
GUID string `json:"guid"`
URL string `json:"url"`
Title string `json:"title"`
Author string `json:"author,omitempty"`
Content string `json:"content"`
RawContent string `json:"raw_content,omitempty"`
PublishedAt time.Time `json:"published_at"`
IsRead bool `json:"is_read"`
IsStarred bool `json:"is_starred"`
ShareUUID *string `json:"share_uuid,omitempty"`
Metadata map[string]interface{} `json:"metadata"`
ID string `json:"id"`
FeedID string `json:"feed_id"`
FeedTitle string `json:"feed_title"`
FeedSiteURL string `json:"feed_site_url,omitempty"`
GUID string `json:"guid"`
URL string `json:"url"`
Title string `json:"title"`
Author string `json:"author,omitempty"`
Content string `json:"content"` // Immutable original RSS content
RawContent string `json:"raw_content,omitempty"`
PublishedAt time.Time `json:"published_at"`
IsRead bool `json:"is_read"`
IsStarred bool `json:"is_starred"`
ShareUUID *string `json:"share_uuid,omitempty"`
Variants map[string]EntryVariant `json:"variants,omitempty"`
Metadata map[string]interface{} `json:"metadata,omitempty"`
CreatedAt time.Time `json:"created_at"`
}
// EntryVariant represents an extensible enrichment result (readability, keywords, ai_summary, tts)
type EntryVariant struct {
EntryID string `json:"entry_id"`
VariantKey string `json:"variant_key"`
ContentType string `json:"content_type"` // 'text', 'html', 'json', 'yaml', 'audio/mp3', 'binary'
TextValue string `json:"text_value,omitempty"`
BlobPath string `json:"blob_path,omitempty"`
StreamURL string `json:"stream_url,omitempty"`
Metadata map[string]interface{} `json:"metadata,omitempty"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// Rule represents a filter/automation hook rule
type Rule struct {
ID string `json:"id"`
Name string `json:"name"`
Trigger string `json:"trigger"` // 'on_post_created', 'on_post_updated', 'on_variant_updated', 'manual'
Conditions []RuleCondition `json:"conditions"`
MatchMode string `json:"match_mode"` // 'all' (AND) or 'any' (OR)
Actions []RuleAction `json:"actions"`
IsActive bool `json:"is_active"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
}
// RuleCondition represents a single condition predicate
type RuleCondition struct {
Field string `json:"field"` // 'feed_id', 'title', 'author', 'variants.keywords', 'variants.readability_full_text'
Operator string `json:"operator"` // 'eq', 'neq', 'in', 'contains', 'matches'
Value interface{} `json:"value"`
}
// RuleAction represents an action triggered by a rule
type RuleAction struct {
Type string `json:"type"` // 'readability_scrape', 'mark_read', 'mark_starred', 'extract_keywords', 'ai_summary', 'tts'
Params map[string]interface{} `json:"params,omitempty"`
}
// Stats represents dashboard aggregate counts
@@ -69,6 +109,7 @@ type Stats struct {
TotalUnread int `json:"total_unread"`
TotalStarred int `json:"total_starred"`
TotalCategories int `json:"total_categories"`
TotalRules int `json:"total_rules"`
}
// LoginRequest is the body for POST /api/auth/login
@@ -116,3 +157,9 @@ type MarkAllReadRequest struct {
CategoryID *string `json:"category_id,omitempty"`
Before *string `json:"before,omitempty"`
}
// ChangePasswordRequest is the body for PUT /api/auth/password
type ChangePasswordRequest struct {
OldPassword string `json:"old_password"`
NewPassword string `json:"new_password"`
}
internal/parser/parser.go
+97 -5
View File
@@ -2,7 +2,9 @@ package parser
import (
"context"
"encoding/json"
"fmt"
"html"
"io"
"net/http"
"strings"
@@ -104,10 +106,7 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri
pubDate = item.UpdatedParsed.UTC()
}
content := item.Content
if strings.TrimSpace(content) == "" {
content = item.Description
}
content := sanitizeAndCombineContent(item.Content, item.Description)
author := ""
if item.Author != nil {
@@ -120,7 +119,7 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri
Title: item.Title,
Author: author,
Content: content,
RawContent: item.Description,
RawContent: content,
PublishedAt: pubDate,
Metadata: make(map[string]interface{}),
}
@@ -129,3 +128,96 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri
return result, nil
}
type draftBlock struct {
Key string `json:"key"`
Text string `json:"text"`
Type string `json:"type"`
}
type draftEntityData struct {
URL string `json:"url"`
Src string `json:"src"`
Caption string `json:"caption"`
}
type draftEntity struct {
Type string `json:"type"`
Data draftEntityData `json:"data"`
}
type draftPayload struct {
Blocks []draftBlock `json:"blocks"`
EntityMap map[string]draftEntity `json:"entityMap"`
}
func parseDraftJSText(raw string) (string, bool) {
unescaped := html.UnescapeString(raw)
idx := strings.Index(unescaped, `{"entityMap"`)
if idx == -1 {
idx = strings.Index(unescaped, `{"blocks"`)
}
if idx == -1 {
return "", false
}
prefix := strings.TrimSpace(unescaped[:idx])
jsonPart := strings.TrimSpace(unescaped[idx:])
var payload draftPayload
if err := json.Unmarshal([]byte(jsonPart), &payload); err != nil {
return "", false
}
if len(payload.Blocks) == 0 && len(payload.EntityMap) == 0 {
return "", false
}
var sb strings.Builder
if prefix != "" {
lines := strings.Split(prefix, "\n")
for _, line := range lines {
l := strings.TrimSpace(line)
if l != "" {
sb.WriteString("<p>")
sb.WriteString(html.EscapeString(l))
sb.WriteString("</p>\n")
}
}
}
for _, block := range payload.Blocks {
txt := strings.TrimSpace(block.Text)
if txt != "" {
sb.WriteString("<p>")
sb.WriteString(html.EscapeString(txt))
sb.WriteString("</p>\n")
}
}
for _, ent := range payload.EntityMap {
if strings.EqualFold(ent.Type, "IMAGE") && ent.Data.Src != "" {
fmt.Fprintf(&sb, `<p><img src="%s" alt="%s" /></p>`+"\n", html.EscapeString(ent.Data.Src), html.EscapeString(ent.Data.Caption))
}
}
return sb.String(), true
}
func sanitizeAndCombineContent(content, description string) string {
if parsedDesc, ok := parseDraftJSText(description); ok {
if strings.TrimSpace(content) != "" {
return content + "<br/>\n" + parsedDesc
}
return parsedDesc
}
if parsedContent, ok := parseDraftJSText(content); ok {
return parsedContent
}
if strings.TrimSpace(content) != "" {
return content
}
return description
}
internal/parser/parser_test.go
+27
View File
@@ -0,0 +1,27 @@
package parser
import (
"strings"
"testing"
)
func TestParseDraftJSText(t *testing.T) {
desc := `就这样吧评论: 奥德赛 (https://movie.douban.com/subject/36808876/)
评价: 推荐
{"entityMap": {}, "blocks": [{"key": "1", "text": "终于看完了奥德赛，非常精彩！", "type": "unstyled"}]}`
content := `<img src="https://img9.doubanio.com/poster.jpg" /><a href="#">就这样吧</a>评论: <a href="#">奥德赛</a><br/>评价: 推荐`
combined := sanitizeAndCombineContent(content, desc)
if !strings.Contains(combined, "终于看完了奥德赛，非常精彩！") {
t.Fatalf("expected parsed text in output, got: %s", combined)
}
if !strings.Contains(combined, "<p>终于看完了奥德赛，非常精彩！</p>") {
t.Fatalf("expected formatted paragraph in output, got: %s", combined)
}
if !strings.Contains(combined, "img9.doubanio.com/poster.jpg") {
t.Fatalf("expected poster in output, got: %s", combined)
}
}
internal/rules/engine.go
+82
View File
@@ -0,0 +1,82 @@
package rules
import (
"context"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/events"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"go.uber.org/zap"
)
type WorkerDispatcher interface {
Dispatch(ctx context.Context, action models.RuleAction, entry models.FeedEntry, depth int)
}
type Engine struct {
db *db.DB
bus *events.EventBus
evaluator *Evaluator
dispatcher WorkerDispatcher
log *zap.Logger
}
func NewEngine(database *db.DB, bus *events.EventBus, dispatcher WorkerDispatcher) *Engine {
return &Engine{
db: database,
bus: bus,
evaluator: NewEvaluator(),
dispatcher: dispatcher,
log: logger.Named("rules-engine"),
}
}
// Start registers event bus subscriptions to evaluate rules reactively.
func (e *Engine) Start(ctx context.Context) {
e.bus.Subscribe(events.EventPostCreated, e.handleEvent)
e.bus.Subscribe(events.EventPostUpdated, e.handleEvent)
e.bus.Subscribe(events.EventVariantUpdated, e.handleEvent)
e.bus.Subscribe(events.EventManualTrigger, e.handleEvent)
e.log.Info("Rules engine started, listening for events")
}
func (e *Engine) handleEvent(ctx context.Context, evt events.Event) error {
trigger := string(evt.Type)
rules, err := e.db.GetActiveRules(trigger)
if err != nil {
e.log.Error("Failed to fetch active rules", zap.String("trigger", trigger), zap.Error(err))
return err
}
if len(rules) == 0 {
return nil
}
// Fetch full entry with latest variants if entry ID is present
entry := evt.Entry
if entry.ID != "" {
if full, err := e.db.GetEntry(entry.ID); err == nil && full != nil {
entry = *full
}
}
for _, r := range rules {
if e.evaluator.Evaluate(r, entry) {
e.log.Info("Rule matched",
zap.String("rule_id", r.ID),
zap.String("rule_name", r.Name),
zap.String("entry_id", entry.ID),
zap.Int("actions_count", len(r.Actions)),
)
for _, action := range r.Actions {
if e.dispatcher != nil {
e.dispatcher.Dispatch(ctx, action, entry, evt.Depth+1)
}
}
}
}
return nil
}
internal/rules/evaluator.go
+302
View File
@@ -0,0 +1,302 @@
package rules
import (
"fmt"
"regexp"
"strconv"
"strings"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
)
type Evaluator struct{}
func NewEvaluator() *Evaluator {
return &Evaluator{}
}
// Evaluate checks whether a FeedEntry satisfies a Rule's conditions.
func (e *Evaluator) Evaluate(rule models.Rule, entry models.FeedEntry) bool {
if len(rule.Conditions) == 0 {
return true
}
matchMode := strings.ToLower(rule.MatchMode)
if matchMode == "" || matchMode == "all" {
for _, cond := range rule.Conditions {
if !e.evaluateCondition(cond, entry) {
return false
}
}
return true
}
// matchMode == "any"
for _, cond := range rule.Conditions {
if e.evaluateCondition(cond, entry) {
return true
}
}
return false
}
func (e *Evaluator) evaluateCondition(cond models.RuleCondition, entry models.FeedEntry) bool {
actualValue := e.resolveField(cond.Field, entry)
return e.compare(actualValue, cond.Operator, cond.Value)
}
func (e *Evaluator) resolveField(field string, entry models.FeedEntry) interface{} {
switch strings.ToLower(field) {
case "feed_id":
return entry.FeedID
case "feed_title":
return entry.FeedTitle
case "title":
return entry.Title
case "author":
return entry.Author
case "url":
return entry.URL
case "content", "raw_content", "rss_content":
var parts []string
if entry.Title != "" {
parts = append(parts, entry.Title)
}
if entry.Content != "" {
parts = append(parts, entry.Content)
}
if entry.RawContent != "" && entry.RawContent != entry.Content {
parts = append(parts, entry.RawContent)
}
if entry.Variants != nil {
if v, ok := entry.Variants["readability_full_text"]; ok && v.TextValue != "" {
parts = append(parts, v.TextValue)
}
}
return strings.Join(parts, "\n")
case "is_read":
return entry.IsRead
case "is_starred":
return entry.IsStarred
case "published_at":
return entry.PublishedAt
case "created_at":
return entry.CreatedAt
}
// Variant field resolution: "variants.keywords", "variants.readability_full_text", "variants.tags"
if strings.HasPrefix(field, "variants.") {
variantKey := strings.TrimPrefix(field, "variants.")
if entry.Variants != nil {
if v, ok := entry.Variants[variantKey]; ok {
return v.TextValue
}
}
}
return nil
}
func (e *Evaluator) compare(actual interface{}, op string, expected interface{}) bool {
opLower := strings.ToLower(strings.TrimSpace(op))
// Handle exists / not_exists operators
if opLower == "exists" {
if actual == nil {
return false
}
if s, ok := actual.(string); ok && s == "" {
return false
}
return true
}
if opLower == "not_exists" {
if actual == nil {
return true
}
if s, ok := actual.(string); ok && s == "" {
return true
}
return false
}
if actual == nil {
return opLower == "neq" || opLower == "!=" || opLower == "not_in" || opLower == "not_contains"
}
// Handle Time/Timestamp comparisons
if actualTime, ok := actual.(time.Time); ok {
return e.compareTime(actualTime, opLower, expected)
}
actualStr := fmt.Sprintf("%v", actual)
expectedStr := fmt.Sprintf("%v", expected)
switch opLower {
case "eq", "==":
return strings.EqualFold(actualStr, expectedStr)
case "neq", "!=":
return !strings.EqualFold(actualStr, expectedStr)
case "contains":
expectedClean := strings.Trim(expectedStr, "\"'` \t\r\n")
return strings.Contains(strings.ToLower(actualStr), strings.ToLower(expectedClean))
case "not_contains":
expectedClean := strings.Trim(expectedStr, "\"'` \t\r\n")
return !strings.Contains(strings.ToLower(actualStr), strings.ToLower(expectedClean))
case "starts_with":
expectedClean := strings.Trim(expectedStr, "\"'` \t\r\n")
return strings.HasPrefix(strings.ToLower(actualStr), strings.ToLower(expectedClean))
case "ends_with":
expectedClean := strings.Trim(expectedStr, "\"'` \t\r\n")
return strings.HasSuffix(strings.ToLower(actualStr), strings.ToLower(expectedClean))
case "contains_any":
tokens := parseStringList(expected)
for _, token := range tokens {
if strings.Contains(strings.ToLower(actualStr), strings.ToLower(token)) {
return true
}
}
return false
case "in":
tokens := parseStringList(expected)
for _, item := range tokens {
if strings.EqualFold(actualStr, item) {
return true
}
}
return false
case "not_in":
tokens := parseStringList(expected)
for _, item := range tokens {
if strings.EqualFold(actualStr, item) {
return false
}
}
return true
case "matches", "regex":
re, err := regexp.Compile(expectedStr)
if err != nil {
return false
}
return re.MatchString(actualStr)
default:
return false
}
}
func (e *Evaluator) compareTime(t time.Time, op string, expected interface{}) bool {
now := time.Now()
switch op {
case "within_last_hours":
hours, err := toFloat64(expected)
if err != nil {
return false
}
cutoff := now.Add(-time.Duration(hours * float64(time.Hour)))
return t.After(cutoff)
case "older_than_days":
days, err := toFloat64(expected)
if err != nil {
return false
}
cutoff := now.Add(-time.Duration(days * 24 * float64(time.Hour)))
return t.Before(cutoff)
case ">=", "after":
targetTime, err := parseTime(expected)
if err != nil {
return false
}
return t.After(targetTime) || t.Equal(targetTime)
case "<=", "before":
targetTime, err := parseTime(expected)
if err != nil {
return false
}
return t.Before(targetTime) || t.Equal(targetTime)
}
return false
}
func parseStringList(val interface{}) []string {
var result []string
clean := func(s string) string {
s = strings.TrimSpace(s)
s = strings.Trim(s, "\"'`")
return strings.TrimSpace(s)
}
switch v := val.(type) {
case []interface{}:
for _, item := range v {
itemStr := clean(fmt.Sprintf("%v", item))
if itemStr != "" {
result = append(result, itemStr)
}
}
case []string:
for _, item := range v {
itemStr := clean(item)
if itemStr != "" {
result = append(result, itemStr)
}
}
case string:
parts := strings.Split(v, ",")
for _, part := range parts {
trimmed := clean(part)
if trimmed != "" {
result = append(result, trimmed)
}
}
}
return result
}
func toFloat64(val interface{}) (float64, error) {
switch v := val.(type) {
case float64:
return v, nil
case int:
return float64(v), nil
case int64:
return float64(v), nil
case string:
return strconv.ParseFloat(strings.TrimSpace(v), 64)
}
return 0, fmt.Errorf("cannot convert %v to float64", val)
}
func parseTime(val interface{}) (time.Time, error) {
if t, ok := val.(time.Time); ok {
return t, nil
}
s := fmt.Sprintf("%v", val)
return time.Parse(time.RFC3339, s)
}
// ValidateRuleSafety checks for potential cyclic/endless loops at rule creation time.
func ValidateRuleSafety(rule models.Rule) error {
trigger := strings.ToLower(rule.Trigger)
// If trigger is on_variant_updated / variant_change, check for recursive set_variant on same key
if trigger == "on_variant_updated" || trigger == "variant_change" {
for _, action := range rule.Actions {
if action.Type == "set_variant" {
targetKey, _ := action.Params["variant_key"].(string)
if targetKey == "" {
targetKey, _ = action.Params["key"].(string)
}
// Check if trigger targets this exact key without safe conditions
for _, cond := range rule.Conditions {
if cond.Field == "variants."+targetKey && (cond.Operator == "exists" || cond.Operator == "eq") {
return fmt.Errorf("cyclic rule detected: rule triggers on variant '%s' and unconditionally writes to '%s'", targetKey, targetKey)
}
}
}
}
}
return nil
}
internal/rules/rules_test.go
+118
View File
@@ -0,0 +1,118 @@
package rules
import (
"testing"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
)
func TestRuleEvaluator(t *testing.T) {
eval := NewEvaluator()
entry := models.FeedEntry{
ID: "entry-1",
FeedID: "feed-hn",
FeedTitle: "Hacker News",
Title: "Announcing Go 1.24 Release",
Author: "Go Team",
Content: "<p>Go 1.24 brings new features...</p>",
Variants: map[string]models.EntryVariant{
"keywords": {
EntryID: "entry-1",
VariantKey: "keywords",
ContentType: "json",
TextValue: `["go", "compiler", "release"]`,
},
},
PublishedAt: time.Now().Add(-2 * time.Hour),
CreatedAt: time.Now().Add(-2 * time.Hour),
}
// Test 1: Title contains Go AND FeedID in list
rule1 := models.Rule{
ID: "r1",
Name: "Tech Feeds in list",
MatchMode: "all",
Conditions: []models.RuleCondition{
{Field: "feed_id", Operator: "in", Value: []string{"feed-hn", "feed-36kr", "feed-ft"}},
{Field: "title", Operator: "contains", Value: "Go 1.24"},
},
}
if !eval.Evaluate(rule1, entry) {
t.Error("expected rule1 to match entry")
}
// Test 2: not_in operator
rule2 := models.Rule{
ID: "r2",
Name: "Feed not in noise",
MatchMode: "all",
Conditions: []models.RuleCondition{
{Field: "feed_id", Operator: "not_in", Value: "feed-spam,feed-ads"},
},
}
if !eval.Evaluate(rule2, entry) {
t.Error("expected rule2 to match entry")
}
// Test 3: contains_any operator
rule3 := models.Rule{
ID: "r3",
Name: "Title match any keyword",
MatchMode: "all",
Conditions: []models.RuleCondition{
{Field: "title", Operator: "contains_any", Value: "Rust, Python, Go 1.24"},
},
}
if !eval.Evaluate(rule3, entry) {
t.Error("expected rule3 to match entry with contains_any")
}
// Test 4: exists operator on variants
rule4 := models.Rule{
ID: "r4",
Name: "Keywords exist",
MatchMode: "all",
Conditions: []models.RuleCondition{
{Field: "variants.keywords", Operator: "exists"},
{Field: "variants.nonexistent", Operator: "not_exists"},
},
}
if !eval.Evaluate(rule4, entry) {
t.Error("expected rule4 to evaluate variant existence correctly")
}
// Test 5: Timestamp within_last_hours
rule5 := models.Rule{
ID: "r5",
Name: "Recent article",
MatchMode: "all",
Conditions: []models.RuleCondition{
{Field: "published_at", Operator: "within_last_hours", Value: 24},
},
}
if !eval.Evaluate(rule5, entry) {
t.Error("expected rule5 to match article within last 24h")
}
// Test 6: Safety validator
unsafeRule := models.Rule{
Trigger: "variant_change",
Conditions: []models.RuleCondition{
{Field: "variants.tags", Operator: "exists"},
},
Actions: []models.RuleAction{
{
Type: "set_variant",
Params: map[string]interface{}{
"variant_key": "tags",
"value": "new-tag",
},
},
},
}
if err := ValidateRuleSafety(unsafeRule); err == nil {
t.Error("expected ValidateRuleSafety to detect cyclic loop")
}
}
internal/worker/pool.go
+191
View File
@@ -0,0 +1,191 @@
package worker
import (
"context"
"fmt"
"sync"
"time"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/db"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/events"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/logger"
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
"go.uber.org/zap"
)
type ActionResult struct {
ResultVariant *models.EntryVariant
MarkRead *bool
MarkStarred *bool
}
type ActionHandler func(ctx context.Context, entry models.FeedEntry, params map[string]interface{}) (*ActionResult, error)
type Task struct {
Action models.RuleAction
Entry models.FeedEntry
Depth int
}
type Pool struct {
db *db.DB
bus *events.EventBus
handlers map[string]ActionHandler
queue chan Task
workers int
wg sync.WaitGroup
log *zap.Logger
mu sync.RWMutex
}
func NewPool(database *db.DB, bus *events.EventBus, workers, queueSize int) *Pool {
if workers <= 0 {
workers = 5
}
if queueSize <= 0 {
queueSize = 100
}
return &Pool{
db: database,
bus: bus,
handlers: make(map[string]ActionHandler),
queue: make(chan Task, queueSize),
workers: workers,
log: logger.Named("worker-pool"),
}
}
func (p *Pool) RegisterHandler(actionType string, handler ActionHandler) {
p.mu.Lock()
defer p.mu.Unlock()
p.handlers[actionType] = handler
p.log.Info("Registered action handler", zap.String("action", actionType))
}
func (p *Pool) Start(ctx context.Context) {
p.log.Info("Starting worker pool", zap.Int("worker_count", p.workers))
for i := 0; i < p.workers; i++ {
p.wg.Add(1)
go p.worker(ctx, i)
}
}
func (p *Pool) Stop() {
close(p.queue)
p.wg.Wait()
p.log.Info("Worker pool stopped")
}
func (p *Pool) Dispatch(ctx context.Context, action models.RuleAction, entry models.FeedEntry, depth int) {
task := Task{Action: action, Entry: entry, Depth: depth}
select {
case <-ctx.Done():
return
case p.queue <- task:
return
default:
}
// If queue is full, wait up to 5 seconds for a slot before warning
select {
case <-ctx.Done():
return
case p.queue <- task:
case <-time.After(5 * time.Second):
p.log.Warn("Worker pool queue full, task dropped after timeout",
zap.String("action", action.Type),
zap.String("entry_id", entry.ID),
)
}
}
func (p *Pool) ExecuteDirect(ctx context.Context, actionType string, entry models.FeedEntry, params map[string]interface{}) (*ActionResult, error) {
p.mu.RLock()
handler, ok := p.handlers[actionType]
p.mu.RUnlock()
if !ok {
return nil, fmt.Errorf("unknown action handler: %s", actionType)
}
return handler(ctx, entry, params)
}
func (p *Pool) worker(ctx context.Context, id int) {
defer p.wg.Done()
for {
select {
case <-ctx.Done():
return
case task, ok := <-p.queue:
if !ok {
return
}
p.processTask(ctx, task)
}
}
}
func (p *Pool) processTask(ctx context.Context, task Task) {
p.mu.RLock()
handler, ok := p.handlers[task.Action.Type]
p.mu.RUnlock()
if !ok {
p.log.Warn("No handler registered for action", zap.String("type", task.Action.Type))
return
}
start := time.Now()
res, err := handler(ctx, task.Entry, task.Action.Params)
duration := time.Since(start)
if err != nil {
p.log.Error("Action execution failed",
zap.String("action", task.Action.Type),
zap.String("entry_id", task.Entry.ID),
zap.Duration("duration", duration),
zap.Error(err),
)
return
}
if res == nil {
return
}
// 1. Save Variant if returned
if res.ResultVariant != nil {
if err := p.db.SetVariant(*res.ResultVariant); err != nil {
p.log.Error("Failed to save entry variant",
zap.String("entry_id", task.Entry.ID),
zap.String("variant_key", res.ResultVariant.VariantKey),
zap.Error(err),
)
} else {
p.log.Info("Entry variant saved",
zap.String("entry_id", task.Entry.ID),
zap.String("variant_key", res.ResultVariant.VariantKey),
zap.Duration("duration", duration),
)
// Publish EventVariantUpdated to enable downstream chained triggers
if p.bus != nil {
p.bus.Publish(ctx, events.Event{
Type: events.EventVariantUpdated,
Entry: task.Entry,
VariantKey: res.ResultVariant.VariantKey,
Depth: task.Depth,
})
}
}
}
// 2. Apply Read/Star updates
if res.MarkRead != nil {
_ = p.db.UpdateEntryRead(task.Entry.ID, *res.MarkRead)
}
if res.MarkStarred != nil {
_ = p.db.UpdateEntryStarred(task.Entry.ID, *res.MarkStarred)
}
}
web/src/App.tsx
+143 -106
View File
@@ -43,6 +43,8 @@ export const App: React.FC = () => {
return localStorage.getItem('rsspipe_selected_entry_id');
});
const [activeArticle, setActiveArticle] = useState<FeedEntry | null>(null);
const [categories, setCategories] = useState<Category[]>([]);
const [feeds, setFeeds] = useState<Feed[]>([]);
const [entries, setEntries] = useState<FeedEntry[]>([]);
@@ -121,19 +123,11 @@ export const App: React.FC = () => {
search: filter.search,
limit: 100,
});
setEntries((prev) => {
if (filter.view === 'inbox' && selectedEntryId) {
const currentActive = prev.find((e) => e.id === selectedEntryId);
if (currentActive && !res.entries.some((e) => e.id === selectedEntryId)) {
return [currentActive, ...res.entries];
}
}
return res.entries;
});
setEntries(res.entries);
} catch (err) {
console.error('Failed to load entries:', err);
}
}, [currentUser, filter, selectedEntryId]);
}, [currentUser, filter]);
// Initial load
useEffect(() => {
@@ -143,62 +137,60 @@ export const App: React.FC = () => {
}
}, [currentUser, loadSidebarData, loadEntries]);
// Sync focused index with persisted selectedEntryId
// Sync focused index & active article when entries change without disrupting reading state
useEffect(() => {
if (entries.length > 0) {
if (selectedEntryId) {
const idx = entries.findIndex((e) => e.id === selectedEntryId);
if (idx >= 0) {
setFocusedIndex(idx);
if (activeArticle) {
const found = entries.find((e) => e.id === activeArticle.id);
if (found) {
setActiveArticle(found);
setFocusedIndex(entries.findIndex((e) => e.id === activeArticle.id));
return;
}
// Active article remains in reader canvas from prior feed, but list is unselected
setFocusedIndex(-1);
return;
}
if (selectedEntryId) {
const found = entries.find((e) => e.id === selectedEntryId);
if (found) {
setActiveArticle(found);
setFocusedIndex(entries.findIndex((e) => e.id === selectedEntryId));
return;
}
// If not found in current list, load directly so reading pane stays populated
api.getEntry(selectedEntryId).then((entry) => {
setActiveArticle(entry);
}).catch(() => {
// fallback to top
if (entries[0]) {
setFocusedIndex(0);
setActiveArticle(entries[0]);
setSelectedEntryId(entries[0].id);
localStorage.setItem('rsspipe_selected_entry_id', entries[0].id);
}
});
return;
}
setFocusedIndex(0);
if (entries[0]) {
setSelectedEntryId(entries[0].id);
localStorage.setItem('rsspipe_selected_entry_id', entries[0].id);
}
setActiveArticle(entries[0]);
setSelectedEntryId(entries[0].id);
localStorage.setItem('rsspipe_selected_entry_id', entries[0].id);
}
}, [entries, selectedEntryId]);
}, [entries]);
const handleSelectEntry = (index: number) => {
setFocusedIndex(index);
if (entries[index]) {
setActiveArticle(entries[index]);
setSelectedEntryId(entries[index].id);
localStorage.setItem('rsspipe_selected_entry_id', entries[index].id);
}
};
// Auto-Refresh & Background Polling (Every 30s + on focus / visibility change)
useEffect(() => {
if (!currentUser) return;
const interval = setInterval(() => {
loadSidebarData();
loadEntries();
}, 30000);
const handleWindowFocus = () => {
loadSidebarData();
loadEntries();
};
const handleVisibilityChange = () => {
if (document.visibilityState === 'visible') {
loadSidebarData();
loadEntries();
}
};
window.addEventListener('focus', handleWindowFocus);
document.addEventListener('visibilitychange', handleVisibilityChange);
return () => {
clearInterval(interval);
window.removeEventListener('focus', handleWindowFocus);
document.removeEventListener('visibilitychange', handleVisibilityChange);
};
}, [currentUser, loadSidebarData, loadEntries]);
// Refresh data only on initial mount or when user clicks refresh / changes filter
// Toggle Read Action with optimistic update
const handleToggleRead = async (entry: FeedEntry, e?: React.MouseEvent) => {
@@ -211,6 +203,9 @@ export const App: React.FC = () => {
item.id === entry.id ? { ...item, is_read: newStatus } : item
)
);
if (activeArticle && activeArticle.id === entry.id) {
setActiveArticle((prev) => (prev ? { ...prev, is_read: newStatus } : prev));
}
try {
await api.updateRead(entry.id, newStatus);
@@ -223,6 +218,9 @@ export const App: React.FC = () => {
item.id === entry.id ? { ...item, is_read: !newStatus } : item
)
);
if (activeArticle && activeArticle.id === entry.id) {
setActiveArticle((prev) => (prev ? { ...prev, is_read: !newStatus } : prev));
}
}
};
@@ -237,6 +235,9 @@ export const App: React.FC = () => {
item.id === entry.id ? { ...item, is_starred: newStatus } : item
)
);
if (activeArticle && activeArticle.id === entry.id) {
setActiveArticle((prev) => (prev ? { ...prev, is_starred: newStatus } : prev));
}
try {
await api.updateStar(entry.id, newStatus);
@@ -248,6 +249,9 @@ export const App: React.FC = () => {
item.id === entry.id ? { ...item, is_starred: !newStatus } : item
)
);
if (activeArticle && activeArticle.id === entry.id) {
setActiveArticle((prev) => (prev ? { ...prev, is_starred: !newStatus } : prev));
}
}
};
@@ -256,6 +260,9 @@ export const App: React.FC = () => {
try {
await api.markAllRead(filter.feedId, filter.categoryId);
setEntries((prev) => prev.map((item) => ({ ...item, is_read: true })));
if (activeArticle) {
setActiveArticle((prev) => (prev ? { ...prev, is_read: true } : prev));
}
loadSidebarData();
} catch (err) {
console.error('Failed to mark all read:', err);
@@ -309,63 +316,11 @@ export const App: React.FC = () => {
case 'j':
case 'J': // Next item
e.preventDefault();
setFocusedIndex((prev) => {
const next = Math.min(prev + 1, Math.max(0, entries.length - 1));
if (entries[next]) {
setSelectedEntryId(entries[next].id);
localStorage.setItem('rsspipe_selected_entry_id', entries[next].id);
}
return next;
});
break;
case 'k':
case 'K': // Previous item
e.preventDefault();
setFocusedIndex((prev) => {
const next = Math.max(prev - 1, 0);
if (entries[next]) {
setSelectedEntryId(entries[next].id);
localStorage.setItem('rsspipe_selected_entry_id', entries[next].id);
}
return next;
});
break;
case 'v':
case 'V': // Open original link
if (entries[focusedIndex]?.url) {
e.preventDefault();
window.open(entries[focusedIndex].url, '_blank');
}
break;
case 's':
case 'S': // Toggle Star
if (entries[focusedIndex]) {
e.preventDefault();
handleToggleStar(entries[focusedIndex]);
}
break;
case 'm':
case 'M': // Toggle Read / Unread
if (entries[focusedIndex]) {
e.preventDefault();
handleToggleRead(entries[focusedIndex]);
}
break;
case 'e':
case 'E': // Archive / Mark read and advance
if (entries[focusedIndex]) {
e.preventDefault();
if (!entries[focusedIndex].is_read) {
handleToggleRead(entries[focusedIndex]);
}
if (entries.length > 0) {
setFocusedIndex((prev) => {
const next = Math.min(prev + 1, Math.max(0, entries.length - 1));
const next = prev < 0 ? 0 : Math.min(prev + 1, entries.length - 1);
if (entries[next]) {
setActiveArticle(entries[next]);
setSelectedEntryId(entries[next].id);
localStorage.setItem('rsspipe_selected_entry_id', entries[next].id);
}
@@ -374,6 +329,79 @@ export const App: React.FC = () => {
}
break;
case 'k':
case 'K': // Previous item
e.preventDefault();
if (entries.length > 0) {
setFocusedIndex((prev) => {
const next = prev < 0 ? 0 : Math.max(prev - 1, 0);
if (entries[next]) {
setActiveArticle(entries[next]);
setSelectedEntryId(entries[next].id);
localStorage.setItem('rsspipe_selected_entry_id', entries[next].id);
}
return next;
});
}
break;
case 'v':
case 'V': // Open original link
{
const targetUrl = activeArticle?.url || entries[focusedIndex]?.url;
if (targetUrl) {
e.preventDefault();
window.open(targetUrl, '_blank');
}
}
break;
case 's':
case 'S': // Toggle Star
{
const target = activeArticle || entries[focusedIndex];
if (target) {
e.preventDefault();
handleToggleStar(target);
}
}
break;
case 'm':
case 'M': // Toggle Read / Unread
{
const target = activeArticle || entries[focusedIndex];
if (target) {
e.preventDefault();
handleToggleRead(target);
}
}
break;
case 'e':
case 'E': // Archive / Mark read and advance
{
const target = activeArticle || entries[focusedIndex];
if (target) {
e.preventDefault();
if (!target.is_read) {
handleToggleRead(target);
}
if (entries.length > 0) {
setFocusedIndex((prev) => {
const next = prev < 0 ? 0 : Math.min(prev + 1, entries.length - 1);
if (entries[next]) {
setActiveArticle(entries[next]);
setSelectedEntryId(entries[next].id);
localStorage.setItem('rsspipe_selected_entry_id', entries[next].id);
}
return next;
});
}
}
}
break;
case 'a':
case 'A': // Add feed modal
if (!e.shiftKey && !e.metaKey && !e.ctrlKey) {
@@ -421,7 +449,7 @@ export const App: React.FC = () => {
window.addEventListener('keydown', handleKeyDown);
return () => window.removeEventListener('keydown', handleKeyDown);
}, [entries, focusedIndex]);
}, [entries, focusedIndex, activeArticle]);
if (authLoading) {
return (
@@ -467,10 +495,15 @@ export const App: React.FC = () => {
stats={stats}
entries={entries}
selectedIndex={focusedIndex}
activeArticle={activeArticle}
onSelectEntry={handleSelectEntry}
onToggleRead={handleToggleRead}
onToggleStar={handleToggleStar}
onMarkAllRead={handleMarkAllRead}
onUpdateEntry={(updated) => {
setActiveArticle(updated);
setEntries((prev) => prev.map((e) => (e.id === updated.id ? updated : e)));
}}
sidebarCollapsed={sidebarCollapsed}
/>
) : viewMode === 'waterfall' ? (
@@ -481,6 +514,10 @@ export const App: React.FC = () => {
entries={entries}
onToggleRead={handleToggleRead}
onToggleStar={handleToggleStar}
onUpdateEntry={(updated) => {
setActiveArticle(updated);
setEntries((prev) => prev.map((e) => (e.id === updated.id ? updated : e)));
}}
/>
) : (
<SettingsView
web/src/components/SettingsView.tsx
+2312 -28
View File
File diff suppressed because it is too large Load Diff
web/src/components/ThreePaneView.tsx
+286 -31
View File
@@ -1,7 +1,8 @@
import React, { useEffect, useRef } from 'react';
import React, { useEffect, useRef, useState } from 'react';
import { Category, Feed, FeedEntry, FilterState, Stats } from '../types';
import { Sidebar } from './Sidebar';
import { formatRelativeDate, stripHtml, estimateReadingTime } from '../utils';
import { formatRelativeDate, stripHtml, estimateReadingTime, getEntryTags, renderMarkdownText } from '../utils';
import { api } from '../services/api';
import {
Star,
CheckCircle2,
@@ -10,6 +11,12 @@ import {
Clock,
User as UserIcon,
CheckCheck,
FileText,
Zap,
Sparkles,
Volume2,
Loader2,
Copy,
} from 'lucide-react';
interface ThreePaneViewProps {
@@ -20,10 +27,12 @@ interface ThreePaneViewProps {
stats: Stats | null;
entries: FeedEntry[];
selectedIndex: number;
activeArticle?: FeedEntry | null;
onSelectEntry: (index: number) => void;
onToggleRead: (entry: FeedEntry, e?: React.MouseEvent) => void;
onToggleStar: (entry: FeedEntry, e?: React.MouseEvent) => void;
onMarkAllRead: () => void;
onUpdateEntry?: (entry: FeedEntry) => void;
sidebarCollapsed: boolean;
}
@@ -35,16 +44,22 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
stats,
entries,
selectedIndex,
activeArticle,
onSelectEntry,
onToggleRead,
onToggleStar,
onMarkAllRead,
onUpdateEntry,
sidebarCollapsed,
}) => {
const cardRefs = useRef<(HTMLDivElement | null)[]>([]);
const readerPaneRef = useRef<HTMLDivElement>(null);
const summaryCardRef = useRef<HTMLDivElement>(null);
const [selectedVariantKey, setSelectedVariantKey] = useState<string>('original');
const [showAISummary, setShowAISummary] = useState<boolean>(true);
const [isScraping, setIsScraping] = useState<boolean>(false);
const selectedEntry = entries[selectedIndex] || entries[0];
const selectedEntry = activeArticle || entries[selectedIndex] || entries[0];
// Auto-scroll list to active card
useEffect(() => {
@@ -56,6 +71,13 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
}
}, [selectedIndex, entries.length]);
// Reset variant selection when article changes
useEffect(() => {
if (selectedEntry?.variants?.['readability_full_text']) {
// If full text is available, you can choose to stay on current or default to original
}
}, [selectedEntry?.id]);
// When switching selected article, reset reader scroll position to top and handle 2s timer for short content
useEffect(() => {
if (readerPaneRef.current) {
@@ -106,6 +128,22 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
}
};
const handleScrapeFullText = async () => {
if (!selectedEntry || isScraping) return;
setIsScraping(true);
try {
const updated = await api.scrapeEntry(selectedEntry.id);
if (onUpdateEntry) {
onUpdateEntry(updated);
}
setSelectedVariantKey('readability_full_text');
} catch (err: any) {
alert(`Scraping failed: ${err.message || err}`);
} finally {
setIsScraping(false);
}
};
const getListTitle = () => {
if (filter.search) return `Search: "${filter.search}"`;
if (filter.feedId) {
@@ -121,6 +159,18 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
return 'Inbox';
};
const hasReadability = Boolean(selectedEntry?.variants?.['readability_full_text']);
const hasAISummary = Boolean(selectedEntry?.variants?.['ai_summary']);
const hasTTS = Boolean(selectedEntry?.variants?.['tts'] || selectedEntry?.variants?.['podcast_tts']);
const getRenderedContent = () => {
if (!selectedEntry) return '';
if (selectedVariantKey === 'readability_full_text' && selectedEntry.variants?.['readability_full_text']) {
return selectedEntry.variants['readability_full_text'].text_value || '';
}
return selectedEntry.content || selectedEntry.raw_content || '';
};
return (
<div className="three-pane-container">
{/* Pane 1: Sidebar (Collapsible & Independently Scrollable) */}
@@ -189,6 +239,26 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
<h4 className="article-item-title">{entry.title || 'Untitled'}</h4>
{snippet && <p className="article-item-snippet">{snippet}</p>}
{(getEntryTags(entry.variants).length > 0 || Boolean(entry.variants?.['ai_summary'])) && (
<div style={{ display: 'flex', flexWrap: 'wrap', gap: 4, marginTop: 4 }}>
{getEntryTags(entry.variants).map((tag) => (
<span key={tag} className="variant-tag-chip">🏷️ {tag}</span>
))}
{entry.variants?.['ai_summary'] && (
<span
className="variant-tag-chip"
style={{
background: 'rgba(16, 185, 129, 0.08)',
color: '#34d399',
borderColor: 'rgba(16, 185, 129, 0.25)',
}}
>
✨ AI Summary
</span>
)}
</div>
)}
</div>
);
})
@@ -233,6 +303,108 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
>
<CheckCircle2 size={16} />
</button>
{/* Sliding Ball Capsule Toggle Button */}
<button
type="button"
className={`capsule-switch-btn ${
selectedVariantKey === 'readability_full_text' ? 'active' : ''
} ${isScraping ? 'scraping' : ''}`}
onClick={() => {
if (isScraping) return;
if (selectedVariantKey === 'readability_full_text') {
setSelectedVariantKey('original');
} else {
if (hasReadability) {
setSelectedVariantKey('readability_full_text');
} else {
handleScrapeFullText();
}
}
}}
title={
selectedVariantKey === 'readability_full_text'
? 'Switch to Original RSS content'
: hasReadability
? 'Switch to Clean Full Text'
: 'Scrape and view Clean Full Text'
}
>
<div className="capsule-switch-track">
<span className="capsule-switch-label">
{isScraping
? 'Scraping...'
: selectedVariantKey === 'readability_full_text'
? 'Full Text'
: 'Original'}
</span>
<div className="capsule-switch-thumb">
{isScraping ? (
<Loader2 size={12} className="spin" />
) : selectedVariantKey === 'readability_full_text' ? (
<Zap size={12} />
) : (
<FileText size={12} />
)}
</div>
</div>
</button>
{hasAISummary && (
<button
type="button"
className={`btn-secondary ${showAISummary ? 'active' : ''}`}
style={{
border: '1px solid',
borderColor: showAISummary ? 'rgba(16, 185, 129, 0.35)' : 'transparent',
borderRadius: '16px',
padding: '3px 9px',
fontSize: '0.72rem',
background: showAISummary ? 'rgba(16, 185, 129, 0.12)' : 'transparent',
color: showAISummary ? '#34d399' : 'var(--text-muted)',
boxShadow: showAISummary ? '0 1px 3px rgba(0,0,0,0.1)' : 'none',
marginLeft: '0.4rem',
display: 'inline-flex',
alignItems: 'center',
gap: 4,
}}
onClick={() => {
const next = !showAISummary;
setShowAISummary(next);
if (next) {
setTimeout(() => {
summaryCardRef.current?.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
}, 50);
}
}}
title={showAISummary ? 'Hide AI Summary' : 'Show AI Summary'}
>
<Sparkles size={12} style={{ color: showAISummary ? '#34d399' : undefined }} />
<span>Summary</span>
</button>
)}
{hasTTS && (
<button
type="button"
className={`btn-secondary ${selectedVariantKey === 'tts' ? 'active' : ''}`}
style={{
border: 'none',
borderRadius: '16px',
padding: '3px 9px',
fontSize: '0.72rem',
background: selectedVariantKey === 'tts' ? 'var(--card-bg)' : 'transparent',
color: selectedVariantKey === 'tts' ? '#10b981' : 'var(--text-muted)',
boxShadow: selectedVariantKey === 'tts' ? '0 1px 3px rgba(0,0,0,0.1)' : 'none',
marginLeft: '0.4rem',
}}
onClick={() => setSelectedVariantKey('tts')}
title="Audio Stream"
>
<Volume2 size={12} style={{ marginRight: 4 }} />
Audio
</button>
)}
</div>
<div style={{ display: 'flex', alignItems: 'center', gap: '0.5rem' }}>
@@ -262,41 +434,123 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
{/* Reading Content */}
<div className="reader-content-canvas">
<div className="reader-source-bar">
<span className="tag-badge">
{selectedEntry.feed_title}
</span>
<span>•</span>
<span style={{ display: 'flex', alignItems: 'center', gap: 4 }}>
<Clock size={12} />{' '}
{new Date(selectedEntry.published_at).toLocaleDateString(undefined, {
month: 'short',
day: 'numeric',
year: 'numeric',
})}
</span>
<span>•</span>
<span>{estimateReadingTime(selectedEntry.content || selectedEntry.raw_content)}</span>
{selectedEntry.author && (
<>
<span>•</span>
<span style={{ display: 'flex', alignItems: 'center', gap: 4 }}>
<UserIcon size={12} /> {selectedEntry.author}
</span>
</>
<div className="article-header">
<div className="article-header-meta">
<span className="article-feed-badge">
{selectedEntry.feed_title}
</span>
<span className="article-meta-divider">•</span>
<span className="article-meta-item">
<Clock size={12} />
{new Date(selectedEntry.published_at).toLocaleDateString(undefined, {
month: 'short',
day: 'numeric',
year: 'numeric',
})}
</span>
<span className="article-meta-divider">•</span>
<span className="article-meta-item">{estimateReadingTime(selectedEntry.content || selectedEntry.raw_content)}</span>
{selectedEntry.author && (
<>
<span className="article-meta-divider">•</span>
<span className="article-meta-item">
<UserIcon size={12} /> {selectedEntry.author}
</span>
</>
)}
</div>
<h1 className="reader-headline">
<a href={selectedEntry.url} target="_blank" rel="noopener noreferrer">
{selectedEntry.title}
</a>
</h1>
{getEntryTags(selectedEntry.variants).length > 0 && (
<div className="article-tags-row">
{getEntryTags(selectedEntry.variants).map((tag) => (
<span key={tag} className="variant-tag-chip">🏷️ {tag}</span>
))}
</div>
)}
</div>
<h1 className="reader-headline">
<a href={selectedEntry.url} target="_blank" rel="noopener noreferrer">
{selectedEntry.title}
</a>
</h1>
{/* Audio player if TTS variant is selected */}
{selectedVariantKey === 'tts' && (
<div style={{ margin: '1rem 0', padding: '1rem', background: 'var(--bg-tertiary)', borderRadius: '8px' }}>
<p style={{ margin: '0 0 0.5rem 0', fontSize: '0.85rem', fontWeight: 600 }}>🎧 Audio Stream:</p>
<audio
controls
style={{ width: '100%' }}
src={`/api/entries/${selectedEntry.id}/variants/tts/stream`}
/>
</div>
)}
{/* AI Summary Card if present and visible */}
{selectedEntry.variants?.ai_summary?.text_value && showAISummary && (
<div
ref={summaryCardRef}
style={{
background: 'rgba(16, 185, 129, 0.05)',
border: '1px solid rgba(16, 185, 129, 0.22)',
borderRadius: 10,
padding: '1rem 1.15rem',
marginBottom: '1.5rem',
}}
>
<div style={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', marginBottom: '0.6rem' }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 6, color: '#34d399', fontWeight: 600, fontSize: '0.82rem' }}>
<Sparkles size={15} />
<span>AI Summary & Takeaways</span>
{selectedEntry.variants.ai_summary.metadata?.model && (
<span style={{ fontSize: '0.68rem', padding: '1px 5px', borderRadius: 4, background: 'rgba(16, 185, 129, 0.1)', color: '#34d399', border: '1px solid rgba(16, 185, 129, 0.2)' }}>
{String(selectedEntry.variants.ai_summary.metadata.model)}
</span>
)}
</div>
<div style={{ display: 'flex', alignItems: 'center', gap: 4 }}>
<button
type="button"
className="btn-icon"
style={{ fontSize: '0.72rem', color: 'var(--text-muted)' }}
title="Copy Summary"
onClick={() => {
if (selectedEntry.variants?.ai_summary?.text_value) {
navigator.clipboard.writeText(selectedEntry.variants.ai_summary.text_value);
}
}}
>
<Copy size={13} />
</button>
<button
type="button"
className="btn-icon"
style={{ fontSize: '0.72rem', color: 'var(--text-muted)' }}
title="Hide Summary"
onClick={() => setShowAISummary(false)}
>
<span style={{ fontSize: '0.75rem', padding: '0 2px' }}>✕</span>
</button>
</div>
</div>
<div
style={{
fontSize: '0.88rem',
lineHeight: 1.6,
color: 'var(--text-secondary)',
}}
dangerouslySetInnerHTML={{
__html: renderMarkdownText(selectedEntry.variants.ai_summary.text_value),
}}
/>
</div>
)}
<div
className="reader-body"
dangerouslySetInnerHTML={{
__html: selectedEntry.content || selectedEntry.raw_content || '',
__html: getRenderedContent(),
}}
/>
</div>
@@ -312,3 +566,4 @@ export const ThreePaneView: React.FC<ThreePaneViewProps> = ({
</div>
);
};
web/src/components/WaterfallView.tsx
+282 -34
View File
@@ -1,6 +1,7 @@
import React, { useState, useRef } from 'react';
import React, { useState, useEffect, useRef } from 'react';
import { api } from '../services/api';
import { Category, FeedEntry, FilterState } from '../types';
import { extractFirstImage, formatRelativeDate, stripHtml, estimateReadingTime } from '../utils';
import { extractFirstImage, formatRelativeDate, stripHtml, estimateReadingTime, getEntryTags, renderMarkdownText } from '../utils';
import {
Star,
Inbox,
@@ -13,6 +14,12 @@ import {
CheckCircle2,
Clock,
User as UserIcon,
FileText,
Zap,
Loader2,
Sparkles,
Volume2,
Copy,
} from 'lucide-react';
interface WaterfallViewProps {
@@ -22,6 +29,7 @@ interface WaterfallViewProps {
entries: FeedEntry[];
onToggleRead: (entry: FeedEntry, e?: React.MouseEvent) => void;
onToggleStar: (entry: FeedEntry, e?: React.MouseEvent) => void;
onUpdateEntry?: (entry: FeedEntry) => void;
}
export const WaterfallView: React.FC<WaterfallViewProps> = ({
@@ -31,9 +39,46 @@ export const WaterfallView: React.FC<WaterfallViewProps> = ({
entries,
onToggleRead,
onToggleStar,
onUpdateEntry,
}) => {
const [selectedEntry, setSelectedEntry] = useState<FeedEntry | null>(null);
const [selectedVariantKey, setSelectedVariantKey] = useState<string>('original');
const [showAISummary, setShowAISummary] = useState<boolean>(true);
const [isScraping, setIsScraping] = useState<boolean>(false);
const readerScrollRef = useRef<HTMLDivElement>(null);
const summaryCardRef = useRef<HTMLDivElement>(null);
// Reset variant selection when selected article changes
useEffect(() => {
setSelectedVariantKey('original');
}, [selectedEntry?.id]);
const handleScrapeFullText = async () => {
if (!selectedEntry || isScraping) return;
setIsScraping(true);
try {
const updated = await api.scrapeEntry(selectedEntry.id);
setSelectedEntry(updated);
onUpdateEntry?.(updated);
setSelectedVariantKey('readability_full_text');
} catch (err: any) {
alert(`Scraping failed: ${err.message || err}`);
} finally {
setIsScraping(false);
}
};
const hasReadability = Boolean(selectedEntry?.variants?.['readability_full_text']);
const hasAISummary = Boolean(selectedEntry?.variants?.['ai_summary']);
const hasTTS = Boolean(selectedEntry?.variants?.['tts'] || selectedEntry?.variants?.['podcast_tts']);
const getRenderedContent = () => {
if (!selectedEntry) return '';
if (selectedVariantKey === 'readability_full_text' && selectedEntry.variants?.['readability_full_text']) {
return selectedEntry.variants['readability_full_text'].text_value || '';
}
return selectedEntry.content || selectedEntry.raw_content || '';
};
// When card is clicked, open in full-screen reader view and immediately mark as read
const handleCardClick = (entry: FeedEntry) => {
@@ -165,6 +210,26 @@ export const WaterfallView: React.FC<WaterfallViewProps> = ({
<h3 className="xhs-card-title">{entry.title}</h3>
{snippet && <p className="xhs-card-snippet">{snippet}</p>}
{(getEntryTags(entry.variants).length > 0 || Boolean(entry.variants?.['ai_summary'])) && (
<div style={{ display: 'flex', flexWrap: 'wrap', gap: 4, marginTop: 4, marginBottom: 4 }}>
{getEntryTags(entry.variants).map((tag) => (
<span key={tag} className="variant-tag-chip">🏷️ {tag}</span>
))}
{entry.variants?.['ai_summary'] && (
<span
className="variant-tag-chip"
style={{
background: 'rgba(16, 185, 129, 0.08)',
color: '#34d399',
borderColor: 'rgba(16, 185, 129, 0.25)',
}}
>
✨ AI Summary
</span>
)}
</div>
)}
{/* Footer */}
<div className="xhs-card-footer">
<div className="xhs-publisher-badge">
@@ -232,7 +297,107 @@ export const WaterfallView: React.FC<WaterfallViewProps> = ({
{selectedEntry.feed_title}
</span>
<div style={{ display: 'flex', alignItems: 'center', gap: 4 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 6 }}>
{/* Sliding Ball Capsule Toggle Button */}
<button
type="button"
className={`capsule-switch-btn ${
selectedVariantKey === 'readability_full_text' ? 'active' : ''
} ${isScraping ? 'scraping' : ''}`}
onClick={() => {
if (isScraping) return;
if (selectedVariantKey === 'readability_full_text') {
setSelectedVariantKey('original');
} else {
if (hasReadability) {
setSelectedVariantKey('readability_full_text');
} else {
handleScrapeFullText();
}
}
}}
title={
selectedVariantKey === 'readability_full_text'
? 'Switch to Original RSS content'
: hasReadability
? 'Switch to Clean Full Text'
: 'Scrape and view Clean Full Text'
}
>
<div className="capsule-switch-track">
<span className="capsule-switch-label">
{isScraping
? 'Scraping...'
: selectedVariantKey === 'readability_full_text'
? 'Full Text'
: 'Original'}
</span>
<div className="capsule-switch-thumb">
{isScraping ? (
<Loader2 size={12} className="spin" />
) : selectedVariantKey === 'readability_full_text' ? (
<Zap size={12} />
) : (
<FileText size={12} />
)}
</div>
</div>
</button>
{hasAISummary && (
<button
type="button"
className={`btn-secondary ${showAISummary ? 'active' : ''}`}
style={{
border: '1px solid',
borderColor: showAISummary ? 'rgba(16, 185, 129, 0.35)' : 'transparent',
borderRadius: '16px',
padding: '3px 8px',
fontSize: '0.7rem',
background: showAISummary ? 'rgba(16, 185, 129, 0.12)' : 'transparent',
color: showAISummary ? '#34d399' : 'var(--text-muted)',
boxShadow: showAISummary ? '0 1px 3px rgba(0,0,0,0.1)' : 'none',
display: 'inline-flex',
alignItems: 'center',
gap: 3,
}}
onClick={() => {
const next = !showAISummary;
setShowAISummary(next);
if (next) {
setTimeout(() => {
summaryCardRef.current?.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
}, 50);
}
}}
title={showAISummary ? 'Hide AI Summary' : 'Show AI Summary'}
>
<Sparkles size={12} style={{ color: showAISummary ? '#34d399' : undefined }} />
<span>Summary</span>
</button>
)}
{hasTTS && (
<button
type="button"
className={`btn-secondary ${selectedVariantKey === 'tts' ? 'active' : ''}`}
style={{
border: 'none',
borderRadius: '16px',
padding: '3px 8px',
fontSize: '0.7rem',
background: selectedVariantKey === 'tts' ? 'var(--card-bg)' : 'transparent',
color: selectedVariantKey === 'tts' ? '#10b981' : 'var(--text-muted)',
boxShadow: selectedVariantKey === 'tts' ? '0 1px 3px rgba(0,0,0,0.1)' : 'none',
}}
onClick={() => setSelectedVariantKey('tts')}
title="Audio Stream"
>
<Volume2 size={12} style={{ marginRight: 3 }} />
Audio
</button>
)}
<button
className="btn-icon"
style={{
@@ -283,41 +448,124 @@ export const WaterfallView: React.FC<WaterfallViewProps> = ({
{/* Reading Body */}
<div className="mobile-post-reader-content" ref={readerScrollRef}>
<div className="reader-content-canvas" style={{ padding: 0 }}>
<div className="reader-source-bar">
<span className="tag-badge">{selectedEntry.feed_title}</span>
<span>•</span>
<span style={{ display: 'flex', alignItems: 'center', gap: 4 }}>
<Clock size={12} />{' '}
{new Date(selectedEntry.published_at).toLocaleDateString(undefined, {
month: 'short',
day: 'numeric',
year: 'numeric',
})}
</span>
<span>•</span>
<span>{estimateReadingTime(selectedEntry.content || selectedEntry.raw_content)}</span>
{selectedEntry.author && (
<>
<span>•</span>
<span style={{ display: 'flex', alignItems: 'center', gap: 4 }}>
<UserIcon size={12} /> {selectedEntry.author}
</span>
</>
<div className="article-header">
<div className="article-header-meta">
<span className="article-feed-badge">
{selectedEntry.feed_title}
</span>
<span className="article-meta-divider">•</span>
<span className="article-meta-item">
<Clock size={12} />
{new Date(selectedEntry.published_at).toLocaleDateString(undefined, {
month: 'short',
day: 'numeric',
year: 'numeric',
})}
</span>
<span className="article-meta-divider">•</span>
<span className="article-meta-item">{estimateReadingTime(selectedEntry.content || selectedEntry.raw_content)}</span>
{selectedEntry.author && (
<>
<span className="article-meta-divider">•</span>
<span className="article-meta-item">
<UserIcon size={12} /> {selectedEntry.author}
</span>
</>
)}
</div>
<h1 className="reader-headline">
<a href={selectedEntry.url} target="_blank" rel="noopener noreferrer">
{selectedEntry.title}
</a>
</h1>
{getEntryTags(selectedEntry.variants).length > 0 && (
<div className="article-tags-row">
{getEntryTags(selectedEntry.variants).map((tag) => (
<span key={tag} className="variant-tag-chip">🏷️ {tag}</span>
))}
</div>
)}
</div>
<h1 className="reader-headline">
<a href={selectedEntry.url} target="_blank" rel="noopener noreferrer">
{selectedEntry.title}
</a>
</h1>
{/* AI Summary Card if present and visible */}
{selectedEntry.variants?.ai_summary?.text_value && showAISummary && (
<div
ref={summaryCardRef}
style={{
background: 'rgba(16, 185, 129, 0.05)',
border: '1px solid rgba(16, 185, 129, 0.22)',
borderRadius: 10,
padding: '1rem 1.15rem',
marginBottom: '1.5rem',
}}
>
<div style={{ display: 'flex', alignItems: 'center', justifyContent: 'space-between', marginBottom: '0.6rem' }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 6, color: '#34d399', fontWeight: 600, fontSize: '0.82rem' }}>
<Sparkles size={15} />
<span>AI Summary & Takeaways</span>
{selectedEntry.variants.ai_summary.metadata?.model && (
<span style={{ fontSize: '0.68rem', padding: '1px 5px', borderRadius: 4, background: 'rgba(16, 185, 129, 0.1)', color: '#34d399', border: '1px solid rgba(16, 185, 129, 0.2)' }}>
{String(selectedEntry.variants.ai_summary.metadata.model)}
</span>
)}
</div>
<div style={{ display: 'flex', alignItems: 'center', gap: 4 }}>
<button
type="button"
className="btn-icon"
style={{ fontSize: '0.72rem', color: 'var(--text-muted)' }}
title="Copy Summary"
onClick={() => {
if (selectedEntry.variants?.ai_summary?.text_value) {
navigator.clipboard.writeText(selectedEntry.variants.ai_summary.text_value);
}
}}
>
<Copy size={13} />
</button>
<button
type="button"
className="btn-icon"
style={{ fontSize: '0.72rem', color: 'var(--text-muted)' }}
title="Hide Summary"
onClick={() => setShowAISummary(false)}
>
<span style={{ fontSize: '0.75rem', padding: '0 2px' }}>✕</span>
</button>
</div>
</div>
<div
style={{
fontSize: '0.88rem',
lineHeight: 1.6,
color: 'var(--text-secondary)',
}}
dangerouslySetInnerHTML={{
__html: renderMarkdownText(selectedEntry.variants.ai_summary.text_value),
}}
/>
</div>
)}
<div
className="reader-body"
dangerouslySetInnerHTML={{
__html: selectedEntry.content || selectedEntry.raw_content || '',
}}
/>
{/* Dynamic Content Variant Rendering */}
{selectedVariantKey === 'tts' && hasTTS ? (
<div className="audio-player-container" style={{ margin: '1.5rem 0', padding: '1rem', background: 'var(--bg-card-hover)', borderRadius: '12px' }}>
<audio
controls
src={`/api/entries/${selectedEntry.id}/variants/tts/stream`}
style={{ width: '100%' }}
/>
</div>
) : (
<div
className="reader-body"
dangerouslySetInnerHTML={{
__html: getRenderedContent(),
}}
/>
)}
</div>
</div>
</div>
web/src/index.css
+186 -6
View File
@@ -266,6 +266,27 @@ html, body, #root {
border: 1px solid var(--tag-border);
}
.variant-tag-chip {
display: inline-flex;
align-items: center;
gap: 4px;
padding: 2px 8px;
border-radius: 9999px;
font-size: 0.7rem;
font-weight: 500;
background: rgba(16, 185, 129, 0.08);
color: #34d399;
border: 1px solid rgba(16, 185, 129, 0.22);
white-space: nowrap;
line-height: 1.45;
transition: all 0.15s ease;
}
.variant-tag-chip:hover {
background: rgba(16, 185, 129, 0.15);
border-color: rgba(16, 185, 129, 0.35);
}
/* Mobile Dropdown Menu */
.mobile-menu-dropdown {
position: absolute;
@@ -560,8 +581,8 @@ html, body, #root {
.reader-toolbar {
position: sticky;
top: 0;
height: 44px;
min-height: 44px;
height: 46px;
min-height: 46px;
background: rgba(255, 255, 255, 0.95);
backdrop-filter: blur(8px);
border-bottom: 1px solid var(--border);
@@ -572,6 +593,107 @@ html, body, #root {
z-index: 10;
}
/* Sliding Ball Capsule Toggle Switch */
.capsule-switch-btn {
display: inline-flex;
align-items: center;
border: none;
background: transparent;
padding: 0;
cursor: pointer;
outline: none;
user-select: none;
margin-left: 0.5rem;
}
.capsule-switch-track {
position: relative;
display: flex;
align-items: center;
width: 86px;
height: 26px;
border-radius: 9999px;
background-color: var(--bg-card-hover);
border: 1px solid var(--border-dark);
padding: 2px;
transition: background-color 0.25s cubic-bezier(0.4, 0, 0.2, 1),
border-color 0.25s cubic-bezier(0.4, 0, 0.2, 1),
box-shadow 0.25s ease;
box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.05);
}
.capsule-switch-btn:hover .capsule-switch-track {
border-color: var(--border-focus);
}
.capsule-switch-btn.active .capsule-switch-track {
background-color: #0284c7;
border-color: #0284c7;
}
.capsule-switch-btn.scraping .capsule-switch-track {
background-color: #0369a1;
border-color: #0369a1;
cursor: wait;
}
.capsule-switch-thumb {
position: absolute;
top: 2px;
left: 2px;
width: 20px;
height: 20px;
border-radius: 50%;
background-color: #ffffff;
color: #64748b;
display: flex;
align-items: center;
justify-content: center;
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.15), 0 1px 2px rgba(0, 0, 0, 0.1);
transition: transform 0.25s cubic-bezier(0.4, 0, 0.2, 1),
color 0.25s ease;
z-index: 2;
}
.capsule-switch-btn.active .capsule-switch-thumb {
transform: translateX(60px);
color: #0284c7;
}
.capsule-switch-label {
width: 100%;
font-size: 0.69rem;
font-weight: 600;
text-align: center;
transition: color 0.25s ease, padding 0.25s ease;
z-index: 1;
white-space: nowrap;
letter-spacing: -0.01em;
}
/* When thumb is on the left, label is padded to the right */
.capsule-switch-btn:not(.active) .capsule-switch-label {
padding-left: 17px;
padding-right: 2px;
color: var(--text-main);
}
/* When thumb is on the right, label is padded to the left */
.capsule-switch-btn.active .capsule-switch-label {
padding-right: 17px;
padding-left: 2px;
color: #ffffff;
}
@keyframes spin {
from { transform: rotate(0deg); }
to { transform: rotate(360deg); }
}
.spin {
animation: spin 1s linear infinite;
}
.reader-content-canvas {
max-width: 680px;
width: 100%;
@@ -579,13 +701,57 @@ html, body, #root {
padding: 2rem 1.5rem 5rem 1.5rem;
}
.reader-source-bar {
.article-header {
margin-bottom: 1.75rem;
padding-bottom: 1.25rem;
border-bottom: 1px solid var(--border);
}
.reader-source-bar,
.article-header-meta {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 0.5rem;
font-size: 0.8rem;
gap: 0.5rem 0.65rem;
font-size: 0.78rem;
color: var(--text-muted);
margin-bottom: 0.85rem;
line-height: 1.5;
}
.article-feed-badge {
display: inline-flex;
align-items: center;
gap: 5px;
padding: 3px 8px;
border-radius: 6px;
font-size: 0.74rem;
font-weight: 600;
background: var(--bg-card);
color: var(--text-main);
border: 1px solid var(--border);
}
.article-meta-item {
display: inline-flex;
align-items: center;
gap: 4px;
color: var(--text-muted);
font-size: 0.75rem;
}
.article-meta-divider {
color: var(--border-dark, #3f3f46);
font-size: 0.65rem;
opacity: 0.6;
}
.article-tags-row {
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 0.4rem;
margin-top: 0.85rem;
}
.reader-headline {
@@ -594,7 +760,7 @@ html, body, #root {
line-height: 1.28;
letter-spacing: -0.02em;
color: var(--text-main);
margin-bottom: 1.5rem;
margin-bottom: 0.5rem;
}
.reader-headline a {
@@ -1130,3 +1296,17 @@ html, body, #root {
font-size: 0.8rem;
margin-bottom: 1.1rem;
}
@keyframes spin {
from {
transform: rotate(0deg);
}
to {
transform: rotate(360deg);
}
}
.animate-spin {
animation: spin 1s linear infinite;
}
web/src/services/api.ts
+73
View File
@@ -64,6 +64,13 @@ export const api = {
return request<User>('/auth/me');
},
async changePassword(old_password: string, new_password: string): Promise<{ message: string }> {
return request<{ message: string }>('/auth/password', {
method: 'PUT',
body: JSON.stringify({ old_password, new_password }),
});
},
// Stats
async getStats(): Promise<Stats> {
return request<Stats>('/stats');
@@ -165,6 +172,51 @@ export const api = {
});
},
async getEntry(id: string): Promise<FeedEntry> {
return request<FeedEntry>(`/entries/${id}`);
},
async getVariants(id: string): Promise<Record<string, any>> {
return request<Record<string, any>>(`/entries/${id}/variants`);
},
async scrapeEntry(id: string): Promise<FeedEntry> {
return request<FeedEntry>(`/entries/${id}/scrape`, {
method: 'POST',
});
},
// Rules
async listRules(): Promise<any[]> {
return request<any[]>('/rules');
},
async createRule(rule: any): Promise<any> {
return request<any>('/rules', {
method: 'POST',
body: JSON.stringify(rule),
});
},
async updateRule(id: string, rule: any): Promise<any> {
return request<any>(`/rules/${id}`, {
method: 'PUT',
body: JSON.stringify(rule),
});
},
async deleteRule(id: string): Promise<void> {
return request<void>(`/rules/${id}`, {
method: 'DELETE',
});
},
async applyRule(id: string): Promise<{ message: string; matched_count: number; queued_jobs: number }> {
return request<{ message: string; matched_count: number; queued_jobs: number }>(`/rules/${id}/apply`, {
method: 'POST',
});
},
// OPML
async importOPML(file: File): Promise<{ imported_count: number; total_parsed: number }> {
const token = localStorage.getItem('rsspipe_jwt');
@@ -191,5 +243,26 @@ export const api = {
exportOPMLUrl(): string {
const token = localStorage.getItem('rsspipe_jwt');
return `${API_BASE}/opml/export?token=${token || ''}`;
},
// AI Settings
async getAISettings(): Promise<import('../types').AISettings> {
return request<import('../types').AISettings>('/settings/ai');
},
async updateAISettings(data: Partial<import('../types').AISettings>): Promise<import('../types').AISettings> {
return request<import('../types').AISettings>('/settings/ai', {
method: 'PUT',
body: JSON.stringify(data),
});
},
async testAIConnection(data?: Partial<import('../types').AISettings>): Promise<import('../types').AITestResult> {
return request<import('../types').AITestResult>('/settings/ai/test', {
method: 'POST',
body: JSON.stringify(data || {}),
});
}
};
web/src/types.ts
+66 -2
View File
@@ -32,6 +32,18 @@ export interface Feed {
created_at: string;
}
export interface EntryVariant {
entry_id: string;
variant_key: string;
content_type: string; // 'text', 'html', 'json', 'yaml', 'audio/mp3', 'binary'
text_value?: string;
blob_path?: string;
stream_url?: string;
metadata?: Record<string, any>;
created_at: string;
updated_at: string;
}
export interface FeedEntry {
id: string;
feed_id: string;
@@ -41,21 +53,46 @@ export interface FeedEntry {
url: string;
title: string;
author?: string;
content: string;
content: string; // Immutable original RSS content
raw_content?: string;
published_at: string;
is_read: boolean;
is_starred: boolean;
share_uuid?: string;
metadata: Record<string, any>;
variants?: Record<string, EntryVariant>;
metadata?: Record<string, any>;
created_at: string;
}
export interface RuleCondition {
field: string;
operator: string;
value: any;
}
export interface RuleAction {
type: string;
params?: Record<string, any>;
}
export interface Rule {
id: string;
name: string;
trigger: string;
conditions: RuleCondition[];
match_mode: string;
actions: RuleAction[];
is_active: boolean;
created_at: string;
updated_at: string;
}
export interface Stats {
total_feeds: number;
total_unread: number;
total_starred: number;
total_categories: number;
total_rules?: number;
}
export type ViewType = 'inbox' | 'starred' | 'all';
@@ -67,3 +104,30 @@ export interface FilterState {
feedId?: string;
search: string;
}
export interface PromptTemplate {
id: string;
name: string;
description?: string;
prompt: string;
is_default?: boolean;
}
export interface AISettings {
enabled: boolean;
provider: string; // 'openai' | 'deepseek' | 'gemini' | 'ollama' | 'custom'
base_url: string;
api_key: string;
model: string;
temperature: number;
max_tokens: number;
prompts: PromptTemplate[];
}
export interface AITestResult {
success: boolean;
message: string;
latency_ms: number;
model: string;
}
web/src/utils.ts
+64
View File
@@ -61,3 +61,67 @@ export function estimateReadingTime(content?: string): string {
const mins = Math.ceil(words / 200);
return `${mins} min read`;
}
/**
* Extract tags from an entry's variants map.
* Tags are stored as variant key "tags" with comma-separated text_value.
*/
export function getEntryTags(variants?: Record<string, { text_value?: string }>): string[] {
if (!variants) return [];
const tagVariant = variants['tags'];
if (!tagVariant?.text_value) return [];
return tagVariant.text_value
.split(',')
.map((t) => t.trim())
.filter(Boolean);
}
export function renderMarkdownText(markdown?: string): string {
if (!markdown) return '';
const lines = markdown.split('\n');
const output: string[] = [];
let inList = false;
for (let line of lines) {
const trimmed = line.trim();
if (!trimmed) {
if (inList) {
output.push('</ul>');
inList = false;
}
continue;
}
// Check if line is a bullet point: starts with "* ", "- ", "• "
const bulletMatch = trimmed.match(/^[\*\-•]\s+(.+)$/);
if (bulletMatch) {
if (!inList) {
output.push('<ul style="margin: 0.6rem 0; padding-left: 1.25rem; display: flex; flex-direction: column; gap: 0.35rem;">');
inList = true;
}
let content = bulletMatch[1];
// Format bold and code
content = content
.replace(/\*\*(.+?)\*\*/g, '<strong style="color: var(--text-main); font-weight: 600;">$1</strong>')
.replace(/`(.+?)`/g, '<code style="background: var(--bg-card); padding: 2px 4px; border-radius: 4px; font-size: 0.8em;">$1</code>');
output.push(`<li style="line-height: 1.55; color: var(--text-secondary);">${content}</li>`);
} else {
if (inList) {
output.push('</ul>');
inList = false;
}
let content = trimmed
.replace(/\*\*(.+?)\*\*/g, '<strong style="color: var(--text-main); font-weight: 600;">$1</strong>')
.replace(/`(.+?)`/g, '<code style="background: var(--bg-card); padding: 2px 4px; border-radius: 4px; font-size: 0.8em;">$1</code>');
output.push(`<p style="margin: 0.5rem 0; line-height: 1.6; color: var(--text-secondary);">${content}</p>`);
}
}
if (inList) {
output.push('</ul>');
}
return output.join('');
}