WordPress Broken Link Checker
This utility crawls a WordPress website (or any website) to identify broken links. It starts from a homepage, recursively finds internal pages, and checks the status code of every link found. It maintains a cache of checked URLs to avoid redundant requests and improve performance.
Modules Used:
- requests: To fetch pages and check link status codes.
- beautifulsoup4: To parse HTML and extract
<a>tags. - argparse: To handle command-line arguments.
urllib.parse: To handle URL joining and domain parsing.
Installation
pip install requests beautifulsoup4The Code
Save this as link_checker.py.
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import argparse
import sys
def check_link(url, checked_cache):
"""Checks the status code of a URL, using a cache to avoid duplicates."""
if url in checked_cache:
return checked_cache[url]
try:
headers = {'User-Agent': 'Mozilla/5.0 (compatible; BrokenLinkBot/1.0)'}
# stream=True prevents downloading the entire body for large files
r = requests.get(url, headers=headers, timeout=5, stream=True)
status = r.status_code
r.close()
except Exception:
status = 0 # Connection error
checked_cache[url] = status
return status
def crawl_site(start_url, max_pages=50):
domain = urlparse(start_url).netloc
queue = [start_url]
visited_pages = set()
checked_links = {} # Cache: {url: status_code}
broken_links = [] # List: [(source_page, broken_link, status)]
print(f"Starting scan of {start_url} (Limit: {max_pages} pages)...")
while queue and len(visited_pages) < max_pages:
current_page = queue.pop(0)
if current_page in visited_pages:
continue
print(f"Scanning page {len(visited_pages)+1}: {current_page}")
visited_pages.add(current_page)
try:
headers = {'User-Agent': 'Mozilla/5.0 (compatible; BrokenLinkBot/1.0)'}
response = requests.get(current_page, headers=headers, timeout=10)
# Skip if page load failed or not HTML
if response.status_code != 200 or 'text/html' not in response.headers.get('Content-Type', ''):
continue
soup = BeautifulSoup(response.text, 'html.parser')
links = soup.find_all('a', href=True)
for link in links:
href = link['href']
full_url = urljoin(current_page, href)
# Remove fragments (#section)
full_url = full_url.split('#')[0]
if not full_url.startswith(('http://', 'https://')):
continue
# Check status
status = check_link(full_url, checked_links)
if status >= 400 or status == 0:
print(f" [BROKEN] {full_url} (Status: {status})")
broken_links.append((current_page, full_url, status))
# If internal link and not visited, add to queue for crawling
if domain in urlparse(full_url).netloc:
if full_url not in visited_pages and full_url not in queue:
queue.append(full_url)
except Exception as e:
print(f" Error processing page: {e}")
print("\n" + "="*50)
print(f"Scan Complete. Found {len(broken_links)} broken links.")
print("="*50)
for source, link, status in broken_links:
print(f"Source: {source}\n -> Link: {link} (Status: {status})\n")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="WordPress Broken Link Checker")
parser.add_argument("url", help="The URL of the website to scan")
parser.add_argument("--limit", type=int, default=50, help="Max number of internal pages to crawl (default: 50)")
args = parser.parse_args()
crawl_site(args.url, args.limit)Usage
# Scan a site (defaults to 50 pages)
python link_checker.py https://my-wordpress-site.com
# Scan deeper (100 pages)
python link_checker.py https://my-wordpress-site.com --limit 100