feat: implement rule engine, worker pool, event bus, and readability scraping with enhanced logging and content sanitization.

internal/parser/parser.go

@@ -2,7 +2,9 @@ package parser
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"html"
 	"io"
 	"net/http"
 	"strings"
@@ -104,10 +106,7 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri
 			pubDate = item.UpdatedParsed.UTC()
 		}
 
-		content := item.Content
-		if strings.TrimSpace(content) == "" {
-			content = item.Description
-		}
+		content := sanitizeAndCombineContent(item.Content, item.Description)
 
 		author := ""
 		if item.Author != nil {
@@ -120,7 +119,7 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri
 			Title:       item.Title,
 			Author:      author,
 			Content:     content,
-			RawContent:  item.Description,
+			RawContent:  content,
 			PublishedAt: pubDate,
 			Metadata:    make(map[string]interface{}),
 		}
@@ -129,3 +128,96 @@ func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified stri
 
 	return result, nil
 }
+
+type draftBlock struct {
+	Key  string `json:"key"`
+	Text string `json:"text"`
+	Type string `json:"type"`
+}
+
+type draftEntityData struct {
+	URL     string `json:"url"`
+	Src     string `json:"src"`
+	Caption string `json:"caption"`
+}
+
+type draftEntity struct {
+	Type string          `json:"type"`
+	Data draftEntityData `json:"data"`
+}
+
+type draftPayload struct {
+	Blocks    []draftBlock           `json:"blocks"`
+	EntityMap map[string]draftEntity `json:"entityMap"`
+}
+
+func parseDraftJSText(raw string) (string, bool) {
+	unescaped := html.UnescapeString(raw)
+	idx := strings.Index(unescaped, `{"entityMap"`)
+	if idx == -1 {
+		idx = strings.Index(unescaped, `{"blocks"`)
+	}
+	if idx == -1 {
+		return "", false
+	}
+
+	prefix := strings.TrimSpace(unescaped[:idx])
+	jsonPart := strings.TrimSpace(unescaped[idx:])
+
+	var payload draftPayload
+	if err := json.Unmarshal([]byte(jsonPart), &payload); err != nil {
+		return "", false
+	}
+
+	if len(payload.Blocks) == 0 && len(payload.EntityMap) == 0 {
+		return "", false
+	}
+
+	var sb strings.Builder
+	if prefix != "" {
+		lines := strings.Split(prefix, "\n")
+		for _, line := range lines {
+			l := strings.TrimSpace(line)
+			if l != "" {
+				sb.WriteString("<p>")
+				sb.WriteString(html.EscapeString(l))
+				sb.WriteString("</p>\n")
+			}
+		}
+	}
+
+	for _, block := range payload.Blocks {
+		txt := strings.TrimSpace(block.Text)
+		if txt != "" {
+			sb.WriteString("<p>")
+			sb.WriteString(html.EscapeString(txt))
+			sb.WriteString("</p>\n")
+		}
+	}
+
+	for _, ent := range payload.EntityMap {
+		if strings.EqualFold(ent.Type, "IMAGE") && ent.Data.Src != "" {
+			fmt.Fprintf(&sb, `<p><img src="%s" alt="%s" /></p>`+"\n", html.EscapeString(ent.Data.Src), html.EscapeString(ent.Data.Caption))
+		}
+	}
+
+	return sb.String(), true
+}
+
+func sanitizeAndCombineContent(content, description string) string {
+	if parsedDesc, ok := parseDraftJSText(description); ok {
+		if strings.TrimSpace(content) != "" {
+			return content + "<br/>\n" + parsedDesc
+		}
+		return parsedDesc
+	}
+
+	if parsedContent, ok := parseDraftJSText(content); ok {
+		return parsedContent
+	}
+
+	if strings.TrimSpace(content) != "" {
+		return content
+	}
+	return description
+}