130 lines
2.9 KiB
Go
130 lines
2.9 KiB
Go
package parser
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"git.pengzhan.dev/haopengzhan/rsspipe/internal/models"
|
|
"github.com/mmcdole/gofeed"
|
|
)
|
|
|
|
type FetchResult struct {
|
|
NotModified bool
|
|
ETag string
|
|
LastModified string
|
|
Title string
|
|
SiteURL string
|
|
Entries []models.FeedEntry
|
|
}
|
|
|
|
type Parser struct {
|
|
client *http.Client
|
|
}
|
|
|
|
func NewParser() *Parser {
|
|
return &Parser{
|
|
client: &http.Client{
|
|
Timeout: 8 * time.Second,
|
|
},
|
|
}
|
|
}
|
|
|
|
func (p *Parser) FetchFeed(ctx context.Context, feedURL, etag, lastModified string) (*FetchResult, error) {
|
|
req, err := http.NewRequestWithContext(ctx, "GET", feedURL, nil)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to create request: %w", err)
|
|
}
|
|
|
|
req.Header.Set("User-Agent", "RSSPipe/1.0 (+https://git.pengzhan.dev/haopengzhan/rsspipe)")
|
|
req.Header.Set("Accept", "application/rss+xml, application/atom+xml, application/xml, text/xml, application/json;q=0.9, */*;q=0.8")
|
|
|
|
if etag != "" {
|
|
req.Header.Set("If-None-Match", etag)
|
|
}
|
|
if lastModified != "" {
|
|
req.Header.Set("If-Modified-Since", lastModified)
|
|
}
|
|
|
|
resp, err := p.client.Do(req)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to fetch url: %w", err)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode == http.StatusNotModified {
|
|
return &FetchResult{NotModified: true}, nil
|
|
}
|
|
|
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
|
return nil, fmt.Errorf("server returned HTTP status %d", resp.StatusCode)
|
|
}
|
|
|
|
newETag := resp.Header.Get("ETag")
|
|
newLastModified := resp.Header.Get("Last-Modified")
|
|
|
|
bodyBytes, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read response body: %w", err)
|
|
}
|
|
|
|
fp := gofeed.NewParser()
|
|
feed, err := fp.ParseString(string(bodyBytes))
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse feed content: %w", err)
|
|
}
|
|
|
|
result := &FetchResult{
|
|
NotModified: false,
|
|
ETag: newETag,
|
|
LastModified: newLastModified,
|
|
Title: feed.Title,
|
|
SiteURL: feed.Link,
|
|
Entries: make([]models.FeedEntry, 0, len(feed.Items)),
|
|
}
|
|
|
|
for _, item := range feed.Items {
|
|
guid := item.GUID
|
|
if guid == "" {
|
|
guid = item.Link
|
|
}
|
|
if guid == "" {
|
|
guid = item.Title
|
|
}
|
|
|
|
pubDate := time.Now().UTC()
|
|
if item.PublishedParsed != nil {
|
|
pubDate = item.PublishedParsed.UTC()
|
|
} else if item.UpdatedParsed != nil {
|
|
pubDate = item.UpdatedParsed.UTC()
|
|
}
|
|
|
|
content := item.Content
|
|
if strings.TrimSpace(content) == "" {
|
|
content = item.Description
|
|
}
|
|
|
|
author := ""
|
|
if item.Author != nil {
|
|
author = item.Author.Name
|
|
}
|
|
|
|
entry := models.FeedEntry{
|
|
GUID: guid,
|
|
URL: item.Link,
|
|
Title: item.Title,
|
|
Author: author,
|
|
Content: content,
|
|
RawContent: item.Description,
|
|
PublishedAt: pubDate,
|
|
Metadata: make(map[string]interface{}),
|
|
}
|
|
result.Entries = append(result.Entries, entry)
|
|
}
|
|
|
|
return result, nil
|
|
}
|