feat: Inital commit

2025-07-26 05:58:59 +00:00
commit 753d1c60ea
1849 changed files with 830533 additions and 0 deletions
@@ -0,0 +1,74 @@
+package crawler_test
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+
+	"git.pengzhan.dev/aimaren/internal/crawler"
+)
+
+const (
+	// The live URL to be used for the end-to-end test.
+	testURL = "https://www.hermes.com/us/en/category/women/bags-and-small-leather-goods/bags-and-clutches/"
+)
+
+// TestScrape_EndToEnd performs a live test against the Hermès website.
+// NOTE: This test makes a real network request and may fail due to network issues,
+// IP blocking, or changes on the live website.
+func TestScrape_EndToEnd(t *testing.T) {
+	t.Log("🚀 Starting end-to-end test against:", testURL)
+
+	// Requirement 1 & 2: Get HTML content and generate bags.
+	// The Scrape function handles the HTTP request internally.
+	// If the response is not 200 OK, it will return an error.
+	bags, err := crawler.Scrape(context.WithValue(t.Context(), crawler.DebugContextKey, true), testURL)
+	if err != nil {
+		t.Fatalf("❌ Test Failed: The scrape function returned an error. This could be a network issue or a non-200 response from the server. Error: %v", err)
+	}
+
+	// Requirement 2: Fail if the result is empty.
+	if len(bags) == 0 {
+		t.Fatalf("❌ Test Failed: Scraper found 0 items. The website's HTML structure has likely changed completely, or the request was blocked.")
+	}
+
+	t.Logf("✅ Successfully scraped %d items. Performing data validation...", len(bags))
+
+	var availableCount, unavailableCount int
+	var sampleBags []string
+
+	for sku, bag := range bags {
+		// Requirement 4 (Others): Sanity check each parsed item.
+		if bag.Name == "" {
+			t.Errorf("❌ Test Failed: Bag with SKU %s has an empty Name.", sku)
+		}
+		if !strings.HasPrefix(bag.URL, "http") {
+			t.Errorf("❌ Test Failed: Bag with SKU %s has an invalid URL: %s", sku, bag.URL)
+		}
+
+		// Requirement 3: Count availability for health check.
+		if bag.Availability {
+			availableCount++
+		} else {
+			unavailableCount++
+		}
+
+		// Collect a few samples for logging.
+		if len(sampleBags) < 3 {
+			sampleBags = append(sampleBags, fmt.Sprintf("  - %s (Available: %t)", bag.Name, bag.Availability))
+		}
+	}
+
+	// Log statistics for review.
+	t.Logf("📊 Availability Stats: %d Available, %d Unavailable", availableCount, unavailableCount)
+	t.Logf("✨ Sample Items:\n%s", strings.Join(sampleBags, "\n"))
+
+	// Requirement 3: Warn if availability is homogenous.
+	if availableCount == 0 || unavailableCount == 0 {
+		// This is a warning, not a failure. It flags a potential issue with the availability logic.
+		t.Logf("⚠️ WARNING: All scraped bags have the same availability status. The 'unavailable' indicator on the website may have changed, causing our logic to be incorrect.")
+	}
+
+	t.Log("✅ End-to-end test completed successfully.")
+}