package parser import ( "context" "fmt" "io" "net/http" "strings" "time" "git.pengzhan.dev/haopengzhan/rsspipe/internal/models" "github.com/mmcdole/gofeed" ) type FetchResult struct { NotModified bool ETag string LastModified string Title string SiteURL string Entries []models.FeedEntry } type Parser struct { client *http.Client } func NewParser() *Parser { return &Parser{ client: &http.Client{ Timeout: 8 * time.Second, }, } } func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified string) (*FetchResult, error) { req, err := http.NewRequestWithContext(ctx, "GET", feedURL, nil) if err != nil { return nil, fmt.Errorf("failed to create request: %w", err) } req.Header.Set("User-Agent", "RSSPipe/1.0 (+https://git.pengzhan.dev/haopengzhan/rsspipe)") req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml, text/xml, application/json;q=0.9, */*;q=0.8") if etag != "" { req.Header.Set("If-None-Match", etag) } if lastModified != "" { req.Header.Set("If-Modified-Since", lastModified) } resp, err := p.client.Do(req) if err != nil { return nil, fmt.Errorf("failed to fetch url: %w", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode == http.StatusNotModified { return &FetchResult{NotModified: true}, nil } if resp.StatusCode < 200 || resp.StatusCode >= 300 { return nil, fmt.Errorf("server returned HTTP status %d", resp.StatusCode) } newETag := resp.Header.Get("ETag") newLastModified := resp.Header.Get("Last-Modified") bodyBytes, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read response body: %w", err) } fp := gofeed.NewParser() feed, err := fp.ParseString(string(bodyBytes)) if err != nil { return nil, fmt.Errorf("failed to parse feed content: %w", err) } result := &FetchResult{ NotModified: false, ETag: newETag, LastModified: newLastModified, Title: feed.Title, SiteURL: feed.Link, Entries: make([]models.FeedEntry, 0, len(feed.Items)), } for _, item := range feed.Items { guid := item.GUID if guid == "" { guid = item.Link } if guid == "" { guid = item.Title } pubDate := time.Now().UTC() if item.PublishedParsed != nil { pubDate = item.PublishedParsed.UTC() } else if item.UpdatedParsed != nil { pubDate = item.UpdatedParsed.UTC() } content := item.Content if strings.TrimSpace(content) == "" { content = item.Description } author := "" if item.Author != nil { author = item.Author.Name } entry := models.FeedEntry{ GUID: guid, URL: item.Link, Title: item.Title, Author: author, Content: content, RawContent: item.Description, PublishedAt: pubDate, Metadata: make(map[string]interface{}), } result.Entries = append(result.Entries, entry) } return result, nil }