# Sitemap Generator

This utility crawls a website to generate a standard `sitemap.xml` file. It uses `requests` to fetch pages and `BeautifulSoup` to extract links, ensuring only internal links are included.

**Modules Used:**
*   [[programming/python/modules/requests-module|requests]]: To fetch HTML content from URLs.
*   [[programming/python/modules/beautifulsoup-module|beautifulsoup4]]: To parse HTML and find links.
*   `xml.etree.ElementTree`: To build the XML structure for the sitemap.
*   `urllib.parse`: To handle URL joining and parsing.

## Installation

```bash
pip install requests beautifulsoup4
```

## The Code

Save this as `sitemap_gen.py`.

```python
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import xml.etree.ElementTree as ET
import argparse
import sys

def generate_sitemap(start_url, output_file):
    # Ensure start_url has a scheme
    if not start_url.startswith('http'):
        start_url = 'https://' + start_url

    domain = urlparse(start_url).netloc
    visited = set()
    to_visit = [start_url]
    sitemap_urls = []

    print(f"Starting crawl of {start_url}...")

    while to_visit:
        current_url = to_visit.pop(0)
        
        # Normalize URL (remove trailing slash for consistency check, though sitemaps prefer consistency)
        if current_url.endswith('/'):
            current_url = current_url[:-1]
            
        if current_url in visited:
            continue
            
        visited.add(current_url)
        sitemap_urls.append(current_url)
        print(f"Crawled: {current_url}")

        try:
            response = requests.get(current_url, timeout=5)
            # Only parse HTML
            if 'text/html' not in response.headers.get('Content-Type', ''):
                continue
                
            soup = BeautifulSoup(response.text, 'html.parser')
            
            for link in soup.find_all('a', href=True):
                href = link['href']
                
                # Handle relative URLs
                full_url = urljoin(current_url, href)
                
                # Remove fragments (#section)
                clean_url = full_url.split('#')[0]
                
                # Parse to check domain
                parsed_url = urlparse(clean_url)
                
                # Only add internal links that haven't been visited
                if parsed_url.netloc == domain and clean_url not in visited and clean_url not in to_visit:
                    # Basic filter for non-page resources
                    if not any(clean_url.lower().endswith(ext) for ext in ['.png', '.jpg', '.pdf', '.zip']):
                         to_visit.append(clean_url)

        except Exception as e:
            print(f"Error fetching {current_url}: {e}")

    # Generate XML
    urlset = ET.Element("urlset", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")
    
    for page_url in sitemap_urls:
        url_elem = ET.SubElement(urlset, "url")
        loc_elem = ET.SubElement(url_elem, "loc")
        loc_elem.text = page_url

    tree = ET.ElementTree(urlset)
    
    # Indent for pretty printing (Python 3.9+)
    if sys.version_info >= (3, 9):
        ET.indent(tree, space="  ", level=0)
        
    tree.write(output_file, encoding='utf-8', xml_declaration=True)
    print(f"\nSuccess! Generated sitemap with {len(sitemap_urls)} URLs at '{output_file}'")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Sitemap Generator")
    parser.add_argument("url", help="Base URL to crawl (e.g., https://example.com)")
    parser.add_argument("-o", "--output", default="sitemap.xml", help="Output XML file")
    
    args = parser.parse_args()
    
    generate_sitemap(args.url, args.output)
```

## Usage

```bash
python sitemap_gen.py https://www.python.org
```

[[programming/python/python]]