package crawler_test import ( "context" "fmt" "strings" "testing" "git.pengzhan.dev/aimaren/internal/crawler" ) const ( // The live URL to be used for the end-to-end test. testURL = "https://www.hermes.com/us/en/category/women/bags-and-small-leather-goods/bags-and-clutches/" ) // TestScrape_EndToEnd performs a live test against the HermΓ¨s website. // NOTE: This test makes a real network request and may fail due to network issues, // IP blocking, or changes on the live website. func TestScrape_EndToEnd(t *testing.T) { t.Log("πŸš€ Starting end-to-end test against:", testURL) // Requirement 1 & 2: Get HTML content and generate bags. // The Scrape function handles the HTTP request internally. // If the response is not 200 OK, it will return an error. bags, err := crawler.Scrape(context.WithValue(t.Context(), crawler.DebugContextKey, true), testURL) if err != nil { t.Fatalf("❌ Test Failed: The scrape function returned an error. This could be a network issue or a non-200 response from the server. Error: %v", err) } // Requirement 2: Fail if the result is empty. if len(bags) == 0 { t.Fatalf("❌ Test Failed: Scraper found 0 items. The website's HTML structure has likely changed completely, or the request was blocked.") } t.Logf("βœ… Successfully scraped %d items. Performing data validation...", len(bags)) var availableCount, unavailableCount int var sampleBags []string for sku, bag := range bags { // Requirement 4 (Others): Sanity check each parsed item. if bag.Name == "" { t.Errorf("❌ Test Failed: Bag with SKU %s has an empty Name.", sku) } if !strings.HasPrefix(bag.URL, "http") { t.Errorf("❌ Test Failed: Bag with SKU %s has an invalid URL: %s", sku, bag.URL) } // Requirement 3: Count availability for health check. if bag.Availability { availableCount++ } else { unavailableCount++ } // Collect a few samples for logging. if len(sampleBags) < 3 { sampleBags = append(sampleBags, fmt.Sprintf(" - %s (Available: %t)", bag.Name, bag.Availability)) } } // Log statistics for review. t.Logf("πŸ“Š Availability Stats: %d Available, %d Unavailable", availableCount, unavailableCount) t.Logf("✨ Sample Items:\n%s", strings.Join(sampleBags, "\n")) // Requirement 3: Warn if availability is homogenous. if availableCount == 0 || unavailableCount == 0 { // This is a warning, not a failure. It flags a potential issue with the availability logic. t.Logf("⚠️ WARNING: All scraped bags have the same availability status. The 'unavailable' indicator on the website may have changed, causing our logic to be incorrect.") } t.Log("βœ… End-to-end test completed successfully.") }