package parser import ( "context" "encoding/json" "fmt" "html" "io" "net/http" "strings" "time" "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" "github.com/mmcdole/gofeed" ) type FetchResult struct { NotModified bool ETag string LastModified string Title string SiteURL string Entries []models.FeedEntry } type Parser struct { client *http.Client } func NewParser() *Parser { return &Parser{ client: &http.Client{ Timeout: 8 * time.Second, }, } } func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified string) (*FetchResult, error) { req, err := http.NewRequestWithContext(ctx, "GET", feedURL, nil) if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) } req.Header.Set("User-Agent", "RSSPipe/1.0 (+https://git.pengzhan.dev/haopengzhan/rsspipe)") req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml, text/xml, application/json;q=0.9, */*;q=0.8") if etag != "" { req.Header.Set("If-None-Match", etag) } if lastModified != "" { req.Header.Set("If-Modified-Since", lastModified) } resp, err := p.client.Do(req) if err != nil { return nil, fmt.Errorf("failed to fetch url: %w", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode == http.StatusNotModified { return &FetchResult{NotModified: true}, nil } if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil, fmt.Errorf("server returned HTTP status %d", resp.StatusCode) } newETag := resp.Header.Get("ETag") newLastModified := resp.Header.Get("Last-Modified") bodyBytes, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read response body: %w", err) } fp := gofeed.NewParser() feed, err := fp.ParseString(string(bodyBytes)) if err != nil { return nil, fmt.Errorf("failed to parse feed content: %w", err) } result := &FetchResult{ NotModified: false, ETag: newETag, LastModified: newLastModified, Title: feed.Title, SiteURL: feed.Link, Entries: make([]models.FeedEntry, 0, len(feed.Items)), } for _, item := range feed.Items { guid := item.GUID if guid == "" { guid = item.Link } if guid == "" { guid = item.Title } pubDate := time.Now().UTC() if item.PublishedParsed != nil { pubDate = item.PublishedParsed.UTC() } else if item.UpdatedParsed != nil { pubDate = item.UpdatedParsed.UTC() } content := sanitizeAndCombineContent(item.Content, item.Description) author := "" if item.Author != nil { author = item.Author.Name } entry := models.FeedEntry{ GUID: guid, URL: item.Link, Title: item.Title, Author: author, Content: content, RawContent: content, PublishedAt: pubDate, Metadata: make(map[string]interface{}), } result.Entries = append(result.Entries, entry) } return result, nil } type draftBlock struct { Key string `json:"key"` Text string `json:"text"` Type string `json:"type"` } type draftEntityData struct { URL string `json:"url"` Src string `json:"src"` Caption string `json:"caption"` } type draftEntity struct { Type string `json:"type"` Data draftEntityData `json:"data"` } type draftPayload struct { Blocks []draftBlock `json:"blocks"` EntityMap map[string]draftEntity `json:"entityMap"` } func parseDraftJSText(raw string) (string, bool) { unescaped := html.UnescapeString(raw) idx := strings.Index(unescaped, `{"entityMap"`) if idx == -1 { idx = strings.Index(unescaped, `{"blocks"`) } if idx == -1 { return "", false } prefix := strings.TrimSpace(unescaped[:idx]) jsonPart := strings.TrimSpace(unescaped[idx:]) var payload draftPayload if err := json.Unmarshal([]byte(jsonPart), &payload); err != nil { return "", false } if len(payload.Blocks) == 0 && len(payload.EntityMap) == 0 { return "", false } var sb strings.Builder if prefix != "" { lines := strings.Split(prefix, "\n") for _, line := range lines { l := strings.TrimSpace(line) if l != "" { sb.WriteString("
") sb.WriteString(html.EscapeString(l)) sb.WriteString("
\n") } } } for _, block := range payload.Blocks { txt := strings.TrimSpace(block.Text) if txt != "" { sb.WriteString("") sb.WriteString(html.EscapeString(txt)) sb.WriteString("
\n") } } for _, ent := range payload.EntityMap { if strings.EqualFold(ent.Type, "IMAGE") && ent.Data.Src != "" { fmt.Fprintf(&sb, `