feat: implement rule engine, worker pool, event bus, and readability scraping with enhanced logging and content sanitization.

internal/parser/parser.go

@@ -2,7 +2,9 @@ package parser
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"html"
 	"io"
 	"net/http"
 	"strings"
@@ -104,10 +106,7 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri
 			pubDate = item.UpdatedParsed.UTC()
 		}
 
-		content := item.Content
-		if strings.TrimSpace(content) == "" {
-			content = item.Description
-		}
+		content := sanitizeAndCombineContent(item.Content, item.Description)
 
 		author := ""
 		if item.Author != nil {
@@ -120,7 +119,7 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri
 			Title:       item.Title,
 			Author:      author,
 			Content:     content,
-			RawContent:  item.Description,
+			RawContent:  content,
 			PublishedAt: pubDate,
 			Metadata:    make(map[string]interface{}),
 		}
@@ -129,3 +128,96 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri
 
 	return result, nil
 }
+
+type draftBlock struct {
+	Key  string `json:"key"`
+	Text string `json:"text"`
+	Type string `json:"type"`
+}
+
+type draftEntityData struct {
+	URL     string `json:"url"`
+	Src     string `json:"src"`
+	Caption string `json:"caption"`
+}
+
+type draftEntity struct {
+	Type string          `json:"type"`
+	Data draftEntityData `json:"data"`
+}
+
+type draftPayload struct {
+	Blocks    []draftBlock           `json:"blocks"`
+	EntityMap map[string]draftEntity `json:"entityMap"`
+}
+
+func parseDraftJSText(raw string) (string, bool) {
+	unescaped := html.UnescapeString(raw)
+	idx := strings.Index(unescaped, `{"entityMap"`)
+	if idx == -1 {
+		idx = strings.Index(unescaped, `{"blocks"`)
+	}
+	if idx == -1 {
+		return "", false
+	}
+
+	prefix := strings.TrimSpace(unescaped[:idx])
+	jsonPart := strings.TrimSpace(unescaped[idx:])
+
+	var payload draftPayload
+	if err := json.Unmarshal([]byte(jsonPart), &payload); err != nil {
+		return "", false
+	}
+
+	if len(payload.Blocks) == 0 && len(payload.EntityMap) == 0 {
+		return "", false
+	}
+
+	var sb strings.Builder
+	if prefix != "" {
+		lines := strings.Split(prefix, "\n")
+		for _, line := range lines {
+			l := strings.TrimSpace(line)
+			if l != "" {
+				sb.WriteString("<p>")
+				sb.WriteString(html.EscapeString(l))
+				sb.WriteString("</p>\n")
+			}
+		}
+	}
+
+	for _, block := range payload.Blocks {
+		txt := strings.TrimSpace(block.Text)
+		if txt != "" {
+			sb.WriteString("<p>")
+			sb.WriteString(html.EscapeString(txt))
+			sb.WriteString("</p>\n")
+		}
+	}
+
+	for _, ent := range payload.EntityMap {
+		if strings.EqualFold(ent.Type, "IMAGE") && ent.Data.Src != "" {
+			fmt.Fprintf(&sb, `<p><img src="%s" alt="%s" /></p>`+"\n", html.EscapeString(ent.Data.Src), html.EscapeString(ent.Data.Caption))
+		}
+	}
+
+	return sb.String(), true
+}
+
+func sanitizeAndCombineContent(content, description string) string {
+	if parsedDesc, ok := parseDraftJSText(description); ok {
+		if strings.TrimSpace(content) != "" {
+			return content + "<br/>\n" + parsedDesc
+		}
+		return parsedDesc
+	}
+
+	if parsedContent, ok := parseDraftJSText(content); ok {
+		return parsedContent
+	}
+
+	if strings.TrimSpace(content) != "" {
+		return content
+	}
+	return description
+}

internal/parser/parser_test.go

@@ -0,0 +1,27 @@
+package parser
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestParseDraftJSText(t *testing.T) {
+	desc := `就这样吧评论: 奥德赛 (https://movie.douban.com/subject/36808876/)
+评价: 推荐
+
+{"entityMap": {}, "blocks": [{"key": "1", "text": "终于看完了奥德赛，非常精彩！", "type": "unstyled"}]}`
+
+	content := `<img src="https://img9.doubanio.com/poster.jpg" /><a href="#">就这样吧</a>评论: <a href="#">奥德赛</a><br/>评价: 推荐`
+
+	combined := sanitizeAndCombineContent(content, desc)
+
+	if !strings.Contains(combined, "终于看完了奥德赛，非常精彩！") {
+		t.Fatalf("expected parsed text in output, got: %s", combined)
+	}
+	if !strings.Contains(combined, "<p>终于看完了奥德赛，非常精彩！</p>") {
+		t.Fatalf("expected formatted paragraph in output, got: %s", combined)
+	}
+	if !strings.Contains(combined, "img9.doubanio.com/poster.jpg") {
+		t.Fatalf("expected poster in output, got: %s", combined)
+	}
+}