Dynamic Web Scraper with Selenium
This guide demonstrates how to build a web scraper using Selenium. Unlike requests, Selenium controls a real web browser, allowing it to scrape data from websites that use JavaScript to render content (e.g., Single Page Applications).
Modules Used:
- selenium: To automate the browser interaction.
- json: To save the scraped data.
time: To handle simple delays during navigation.
Prerequisites
You need to install Selenium.
pip install seleniumThe Code
Save this as dynamic_scraper.py. This script scrapes quotes from a JavaScript-heavy sandbox site, handling pagination automatically.
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import json
import time
def scrape_quotes():
# 1. Setup Headless Chrome
# Headless mode runs the browser in the background without a UI window
chrome_options = Options()
chrome_options.add_argument("--headless=new")
print("Starting browser...")
driver = webdriver.Chrome(options=chrome_options)
base_url = "http://quotes.toscrape.com/js/"
driver.get(base_url)
all_quotes = []
try:
while True:
print(f"Scraping page: {driver.current_url}")
# 2. Wait for content to load
# Dynamic sites take time to render JS. We wait up to 10s for quotes to appear.
wait = WebDriverWait(driver, 10)
quote_elements = wait.until(
EC.presence_of_all_elements_located((By.CLASS_NAME, "quote"))
)
# 3. Extract Data
for quote in quote_elements:
text = quote.find_element(By.CLASS_NAME, "text").text
author = quote.find_element(By.CLASS_NAME, "author").text
all_quotes.append({"text": text, "author": author})
# 4. Handle Pagination
try:
# Look for the 'Next' button
next_btn = driver.find_element(By.CSS_SELECTOR, "li.next > a")
next_btn.click()
# Small pause to allow the click to register and page to start loading
time.sleep(1)
except:
print("No more pages found.")
break
finally:
# Always close the browser to free resources
driver.quit()
# 5. Save Results
with open("quotes.json", "w", encoding="utf-8") as f:
json.dump(all_quotes, f, indent=2, ensure_ascii=False)
print(f"Successfully saved {len(all_quotes)} quotes to quotes.json")
if __name__ == "__main__":
scrape_quotes()Usage
python dynamic_scraper.py