75 lines
2.6 KiB
Go
75 lines
2.6 KiB
Go
package crawler_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
"testing"
|
|
|
|
"git.pengzhan.dev/aimaren/internal/crawler"
|
|
)
|
|
|
|
const (
|
|
// The live URL to be used for the end-to-end test.
|
|
testURL = "https://www.hermes.com/us/en/category/women/bags-and-small-leather-goods/bags-and-clutches/"
|
|
)
|
|
|
|
// TestScrape_EndToEnd performs a live test against the Hermès website.
|
|
// NOTE: This test makes a real network request and may fail due to network issues,
|
|
// IP blocking, or changes on the live website.
|
|
func TestScrape_EndToEnd(t *testing.T) {
|
|
t.Log("🚀 Starting end-to-end test against:", testURL)
|
|
|
|
// Requirement 1 & 2: Get HTML content and generate bags.
|
|
// The Scrape function handles the HTTP request internally.
|
|
// If the response is not 200 OK, it will return an error.
|
|
bags, err := crawler.Scrape(context.WithValue(t.Context(), crawler.DebugContextKey, true), testURL)
|
|
if err != nil {
|
|
t.Fatalf("❌ Test Failed: The scrape function returned an error. This could be a network issue or a non-200 response from the server. Error: %v", err)
|
|
}
|
|
|
|
// Requirement 2: Fail if the result is empty.
|
|
if len(bags) == 0 {
|
|
t.Fatalf("❌ Test Failed: Scraper found 0 items. The website's HTML structure has likely changed completely, or the request was blocked.")
|
|
}
|
|
|
|
t.Logf("✅ Successfully scraped %d items. Performing data validation...", len(bags))
|
|
|
|
var availableCount, unavailableCount int
|
|
var sampleBags []string
|
|
|
|
for sku, bag := range bags {
|
|
// Requirement 4 (Others): Sanity check each parsed item.
|
|
if bag.Name == "" {
|
|
t.Errorf("❌ Test Failed: Bag with SKU %s has an empty Name.", sku)
|
|
}
|
|
if !strings.HasPrefix(bag.URL, "http") {
|
|
t.Errorf("❌ Test Failed: Bag with SKU %s has an invalid URL: %s", sku, bag.URL)
|
|
}
|
|
|
|
// Requirement 3: Count availability for health check.
|
|
if bag.Availability {
|
|
availableCount++
|
|
} else {
|
|
unavailableCount++
|
|
}
|
|
|
|
// Collect a few samples for logging.
|
|
if len(sampleBags) < 3 {
|
|
sampleBags = append(sampleBags, fmt.Sprintf(" - %s (Available: %t)", bag.Name, bag.Availability))
|
|
}
|
|
}
|
|
|
|
// Log statistics for review.
|
|
t.Logf("📊 Availability Stats: %d Available, %d Unavailable", availableCount, unavailableCount)
|
|
t.Logf("✨ Sample Items:\n%s", strings.Join(sampleBags, "\n"))
|
|
|
|
// Requirement 3: Warn if availability is homogenous.
|
|
if availableCount == 0 || unavailableCount == 0 {
|
|
// This is a warning, not a failure. It flags a potential issue with the availability logic.
|
|
t.Logf("⚠️ WARNING: All scraped bags have the same availability status. The 'unavailable' indicator on the website may have changed, causing our logic to be incorrect.")
|
|
}
|
|
|
|
t.Log("✅ End-to-end test completed successfully.")
|
|
}
|