rsspipe/internal/parser/parser.go

package parser

import (
	"context"
	"encoding/json"
	"fmt"
	"html"
	"io"
	"net/http"
	"strings"
	"time"

	"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
	"github.com/mmcdole/gofeed"
)

type FetchResult struct {
	NotModified  bool
	ETag         string
	LastModified string
	Title        string
	SiteURL      string
	Entries      []models.FeedEntry
}

type Parser struct {
	client *http.Client
}

func NewParser() *Parser {
	return &Parser{
		client: &http.Client{
			Timeout: 8 * time.Second,
		},
	}
}

func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified string) (*FetchResult, error) {
	req, err := http.NewRequestWithContext(ctx, "GET", feedURL, nil)
	if err != nil {
		return nil, fmt.Errorf("failed to create request: %w", err)
	}

	req.Header.Set("User-Agent", "RSSPipe/1.0 (+https://git.pengzhan.dev/haopengzhan/rsspipe)")
	req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml, text/xml, application/json;q=0.9, */*;q=0.8")

	if etag != "" {
		req.Header.Set("If-None-Match", etag)
	}
	if lastModified != "" {
		req.Header.Set("If-Modified-Since", lastModified)
	}

	resp, err := p.client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("failed to fetch url: %w", err)
	}
	defer func() {
		_ = resp.Body.Close()
	}()

	if resp.StatusCode == http.StatusNotModified {
		return &FetchResult{NotModified: true}, nil
	}

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return nil, fmt.Errorf("server returned HTTP status %d", resp.StatusCode)
	}

	newETag := resp.Header.Get("ETag")
	newLastModified := resp.Header.Get("Last-Modified")

	bodyBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("failed to read response body: %w", err)
	}

	fp := gofeed.NewParser()
	feed, err := fp.ParseString(string(bodyBytes))
	if err != nil {
		return nil, fmt.Errorf("failed to parse feed content: %w", err)
	}

	result := &FetchResult{
		NotModified:  false,
		ETag:         newETag,
		LastModified: newLastModified,
		Title:        feed.Title,
		SiteURL:      feed.Link,
		Entries:      make([]models.FeedEntry, 0, len(feed.Items)),
	}

	for _, item := range feed.Items {
		guid := item.GUID
		if guid == "" {
			guid = item.Link
		}
		if guid == "" {
			guid = item.Title
		}

		pubDate := time.Now().UTC()
		if item.PublishedParsed != nil {
			pubDate = item.PublishedParsed.UTC()
		} else if item.UpdatedParsed != nil {
			pubDate = item.UpdatedParsed.UTC()
		}

		content := sanitizeAndCombineContent(item.Content, item.Description)

		author := ""
		if item.Author != nil {
			author = item.Author.Name
		}

		entry := models.FeedEntry{
			GUID:        guid,
			URL:         item.Link,
			Title:       item.Title,
			Author:      author,
			Content:     content,
			RawContent:  content,
			PublishedAt: pubDate,
			Metadata:    make(map[string]interface{}),
		}
		result.Entries = append(result.Entries, entry)
	}

	return result, nil
}

type draftBlock struct {
	Key  string `json:"key"`
	Text string `json:"text"`
	Type string `json:"type"`
}

type draftEntityData struct {
	URL     string `json:"url"`
	Src     string `json:"src"`
	Caption string `json:"caption"`
}

type draftEntity struct {
	Type string          `json:"type"`
	Data draftEntityData `json:"data"`
}

type draftPayload struct {
	Blocks    []draftBlock           `json:"blocks"`
	EntityMap map[string]draftEntity `json:"entityMap"`
}

func parseDraftJSText(raw string) (string, bool) {
	unescaped := html.UnescapeString(raw)
	idx := strings.Index(unescaped, `{"entityMap"`)
	if idx == -1 {
		idx = strings.Index(unescaped, `{"blocks"`)
	}
	if idx == -1 {
		return "", false
	}

	prefix := strings.TrimSpace(unescaped[:idx])
	jsonPart := strings.TrimSpace(unescaped[idx:])

	var payload draftPayload
	if err := json.Unmarshal([]byte(jsonPart), &payload); err != nil {
		return "", false
	}

	if len(payload.Blocks) == 0 && len(payload.EntityMap) == 0 {
		return "", false
	}

	var sb strings.Builder
	if prefix != "" {
		lines := strings.Split(prefix, "\n")
		for _, line := range lines {
			l := strings.TrimSpace(line)
			if l != "" {
				sb.WriteString("<p>")
				sb.WriteString(html.EscapeString(l))
				sb.WriteString("</p>\n")
			}
		}
	}

	for _, block := range payload.Blocks {
		txt := strings.TrimSpace(block.Text)
		if txt != "" {
			sb.WriteString("<p>")
			sb.WriteString(html.EscapeString(txt))
			sb.WriteString("</p>\n")
		}
	}

	for _, ent := range payload.EntityMap {
		if strings.EqualFold(ent.Type, "IMAGE") && ent.Data.Src != "" {
			fmt.Fprintf(&sb, `<p><img src="%s" alt="%s" /></p>`+"\n", html.EscapeString(ent.Data.Src), html.EscapeString(ent.Data.Caption))
		}
	}

	return sb.String(), true
}

func sanitizeAndCombineContent(content, description string) string {
	if parsedDesc, ok := parseDraftJSText(description); ok {
		if strings.TrimSpace(content) != "" {
			return content + "<br/>\n" + parsedDesc
		}
		return parsedDesc
	}

	if parsedContent, ok := parseDraftJSText(content); ok {
		return parsedContent
	}

	if strings.TrimSpace(content) != "" {
		return content
	}
	return description
}