Sitemap Generator
This utility crawls a website to generate a standard sitemap.xml file. It uses requests to fetch pages and BeautifulSoup to extract links, ensuring only internal links are included.
Modules Used:
- requests: To fetch HTML content from URLs.
- beautifulsoup4: To parse HTML and find links.
xml.etree.ElementTree: To build the XML structure for the sitemap.urllib.parse: To handle URL joining and parsing.
Installation
pip install requests beautifulsoup4The Code
Save this as sitemap_gen.py.
import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse
import xml.etree.ElementTree as ET
import argparse
import sys
def generate_sitemap(start_url, output_file):
# Ensure start_url has a scheme
if not start_url.startswith('http'):
start_url = 'https://' + start_url
domain = urlparse(start_url).netloc
visited = set()
to_visit = [start_url]
sitemap_urls = []
print(f"Starting crawl of {start_url}...")
while to_visit:
current_url = to_visit.pop(0)
# Normalize URL (remove trailing slash for consistency check, though sitemaps prefer consistency)
if current_url.endswith('/'):
current_url = current_url[:-1]
if current_url in visited:
continue
visited.add(current_url)
sitemap_urls.append(current_url)
print(f"Crawled: {current_url}")
try:
response = requests.get(current_url, timeout=5)
# Only parse HTML
if 'text/html' not in response.headers.get('Content-Type', ''):
continue
soup = BeautifulSoup(response.text, 'html.parser')
for link in soup.find_all('a', href=True):
href = link['href']
# Handle relative URLs
full_url = urljoin(current_url, href)
# Remove fragments (#section)
clean_url = full_url.split('#')[0]
# Parse to check domain
parsed_url = urlparse(clean_url)
# Only add internal links that haven't been visited
if parsed_url.netloc == domain and clean_url not in visited and clean_url not in to_visit:
# Basic filter for non-page resources
if not any(clean_url.lower().endswith(ext) for ext in ['.png', '.jpg', '.pdf', '.zip']):
to_visit.append(clean_url)
except Exception as e:
print(f"Error fetching {current_url}: {e}")
# Generate XML
urlset = ET.Element("urlset", xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")
for page_url in sitemap_urls:
url_elem = ET.SubElement(urlset, "url")
loc_elem = ET.SubElement(url_elem, "loc")
loc_elem.text = page_url
tree = ET.ElementTree(urlset)
# Indent for pretty printing (Python 3.9+)
if sys.version_info >= (3, 9):
ET.indent(tree, space=" ", level=0)
tree.write(output_file, encoding='utf-8', xml_declaration=True)
print(f"\nSuccess! Generated sitemap with {len(sitemap_urls)} URLs at '{output_file}'")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Sitemap Generator")
parser.add_argument("url", help="Base URL to crawl (e.g., https://example.com)")
parser.add_argument("-o", "--output", default="sitemap.xml", help="Output XML file")
args = parser.parse_args()
generate_sitemap(args.url, args.output)Usage
python sitemap_gen.py https://www.python.org