# WordPress Broken Link Checker

This utility crawls a WordPress website (or any website) to identify broken links. It starts from a homepage, recursively finds internal pages, and checks the status code of every link found. It maintains a cache of checked URLs to avoid redundant requests and improve performance.

**Modules Used:**
*   [[programming/python/modules/requests-module|requests]]: To fetch pages and check link status codes.
*   [[programming/python/modules/beautifulsoup-module|beautifulsoup4]]: To parse HTML and extract `<a>` tags.
*   [[programming/python/modules/argparse-module|argparse]]: To handle command-line arguments.
*   `urllib.parse`: To handle URL joining and domain parsing.

## Installation

```bash
pip install requests beautifulsoup4
```

## The Code

Save this as `link_checker.py`.

```python
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import argparse
import sys

def check_link(url, checked_cache):
    """Checks the status code of a URL, using a cache to avoid duplicates."""
    if url in checked_cache:
        return checked_cache[url]
    
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (compatible; BrokenLinkBot/1.0)'}
        # stream=True prevents downloading the entire body for large files
        r = requests.get(url, headers=headers, timeout=5, stream=True)
        status = r.status_code
        r.close()
    except Exception:
        status = 0 # Connection error
    
    checked_cache[url] = status
    return status

def crawl_site(start_url, max_pages=50):
    domain = urlparse(start_url).netloc
    queue = [start_url]
    visited_pages = set()
    checked_links = {} # Cache: {url: status_code}
    broken_links = []  # List: [(source_page, broken_link, status)]

    print(f"Starting scan of {start_url} (Limit: {max_pages} pages)...")

    while queue and len(visited_pages) < max_pages:
        current_page = queue.pop(0)
        
        if current_page in visited_pages:
            continue
            
        print(f"Scanning page {len(visited_pages)+1}: {current_page}")
        visited_pages.add(current_page)

        try:
            headers = {'User-Agent': 'Mozilla/5.0 (compatible; BrokenLinkBot/1.0)'}
            response = requests.get(current_page, headers=headers, timeout=10)
            
            # Skip if page load failed or not HTML
            if response.status_code != 200 or 'text/html' not in response.headers.get('Content-Type', ''):
                continue

            soup = BeautifulSoup(response.text, 'html.parser')
            links = soup.find_all('a', href=True)
            
            for link in links:
                href = link['href']
                full_url = urljoin(current_page, href)
                # Remove fragments (#section)
                full_url = full_url.split('#')[0]
                
                if not full_url.startswith(('http://', 'https://')):
                    continue

                # Check status
                status = check_link(full_url, checked_links)
                
                if status >= 400 or status == 0:
                    print(f"  [BROKEN] {full_url} (Status: {status})")
                    broken_links.append((current_page, full_url, status))
                
                # If internal link and not visited, add to queue for crawling
                if domain in urlparse(full_url).netloc:
                    if full_url not in visited_pages and full_url not in queue:
                        queue.append(full_url)

        except Exception as e:
            print(f"  Error processing page: {e}")

    print("\n" + "="*50)
    print(f"Scan Complete. Found {len(broken_links)} broken links.")
    print("="*50)
    for source, link, status in broken_links:
        print(f"Source: {source}\n  -> Link: {link} (Status: {status})\n")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="WordPress Broken Link Checker")
    parser.add_argument("url", help="The URL of the website to scan")
    parser.add_argument("--limit", type=int, default=50, help="Max number of internal pages to crawl (default: 50)")
    
    args = parser.parse_args()
    
    crawl_site(args.url, args.limit)
```

## Usage

```bash
# Scan a site (defaults to 50 pages)
python link_checker.py https://my-wordpress-site.com

# Scan deeper (100 pages)
python link_checker.py https://my-wordpress-site.com --limit 100
```

[[programming/python/python]]