Wayfair Closeout Scraping System

#!/usr/bin/env python3
"""
Wayfair Closeout Section Scraper
Captures: Product Name, Price, Description
Outputs: Clean, deduplicated CSV file.
"""

import json
import logging
import os
import re
import sys
import time
from datetime import datetime
from bs4 import BeautifulSoup
import pandas as pd
from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError

# Load configuration
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
CONFIG_FILE = os.path.join(BASE_DIR, "config.json")

if os.path.exists(CONFIG_FILE):
with open(CONFIG_FILE, "r", encoding="utf-8") as f:
config = json.load(f)
else:
config = {
"start_url": "https://www.wayfair.com/daily-sales/closeout-sale",
"max_pages": 10,
"request_delay_seconds": 3,
"headless": True,
"output_dir": os.path.join(BASE_DIR, "output"),
"log_dir": os.path.join(BASE_DIR, "output", "logs"),
"timeout_ms": 60000,
}

OUTPUT_DIR = os.path.abspath(config.get("output_dir", "./output"))
LOG_DIR = os.path.abspath(config.get("log_dir", "./output/logs"))
os.makedirs(OUTPUT_DIR, exist_ok=True)
os.makedirs(LOG_DIR, exist_ok=True)

# Logger setup
log_filename = os.path.join(LOG_DIR, f"scraper_{datetime.now().strftime('%Y%m%d')}.log")
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(message)s",
handlers=[
logging.FileHandler(log_filename, encoding="utf-8"),
logging.StreamHandler(sys.stdout),
],
)

def sanitize_text(text: str) -> str:
"""Removes irregular whitespace, non-breaking spaces, and extra newlines."""
if not text:
return ""
text = re.sub(r"[\r\n\t]+", " ", text)
text = re.sub(r"\s{2,}", " ", text)
return text.strip()

def extract_item_details(card_soup: BeautifulSoup) -> dict:
"""Extracts Name, Price, and Description with fallback selectors."""
# 1. Product Name
name_el = (
card_soup.select_one('[data-hb-id="ProductCard-name"]')
or card_soup.select_one("h2")
or card_soup.select_one('[class*="ProductCard-title"]')
or card_soup.select_one('[class*="ProductCardName"]')
)
product_name = sanitize_text(name_el.get_text()) if name_el else ""

# 2. Current Price
price_el = (
card_soup.select_one('[data-hb-id="PriceDisplay"]')
or card_soup.select_one('span[class*="PriceDisplay"]')
or card_soup.select_one('[class*="pl-Price"]')
or card_soup.select_one('[data-hb-id="VisualHierarchy-Price"]')
)
price = sanitize_text(price_el.get_text()) if price_el else ""

# 3. Description
desc_el = (
card_soup.select_one('[data-hb-id="ProductCard-description"]')
or card_soup.select_one('[class*="ProductCardDescription"]')
or card_soup.select_one('[class*="CardReviewAndDescription"] p')
)
description = sanitize_text(desc_el.get_text()) if desc_el else ""

# Fallback for description if it's rendered as bulleted specifications
if not description:
bullets = card_soup.select("ul li")
if bullets:
description = "; ".join([sanitize_text(b.get_text()) for b in bullets])

return {
"Product Name": product_name,
"Price": price,
"Description": description,
}

def scrape_wayfair():
start_url = config.get("start_url")
max_pages = config.get("max_pages", 10)
delay = config.get("request_delay_seconds", 3)
headless = config.get("headless", True)
timeout_val = config.get("timeout_ms", 60000)

records = []
seen_products = set()

logging.info(f"Starting run on URL: {start_url}")

with sync_playwright() as pw:
browser = pw.chromium.launch(
headless=headless,
args=[
"--disable-blink-features=AutomationControlled",
"--no-sandbox",
"--disable-dev-shm-usage",
],
)

context = browser.new_context(
user_agent=(
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/124.0.0.0 Safari/537.36"
),
viewport={"width": 1440, "height": 900},
)

page = context.new_page()
current_page_idx = 1
current_url = start_url

while current_page_idx