News Headline Scraper
This utility scrapes headlines and links from a news website (e.g., Drudge Report). It uses requests to fetch the HTML and BeautifulSoup to parse the content, filtering for valid headline links.
Modules Used:
- requests: To fetch the webpage.
- beautifulsoup4: To parse HTML and extract links.
- argparse: To handle command-line arguments.
Installation
pip install requests beautifulsoup4The Code
Save this as news_scraper.py.
import requests
from bs4 import BeautifulSoup
import argparse
import sys
from urllib.parse import urljoin
def scrape_headlines(url, keyword=None):
print(f"Scraping headlines from: {url}...")
try:
# User-Agent is often required to avoid 403 Forbidden errors
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}
response = requests.get(url, headers=headers, timeout=10)
response.raise_for_status()
soup = BeautifulSoup(response.text, 'html.parser')
# Find all links
links = soup.find_all('a')
headlines = []
for link in links:
text = link.get_text(strip=True)
href = link.get('href')
# Basic filtering:
# 1. Text must be present
# 2. If keyword is provided, text must contain it (case-insensitive)
# 3. Filter out very short links (navigation items)
if text and href and len(text) > 10:
if keyword and keyword.lower() not in text.lower():
continue
# Resolve relative URLs
full_url = urljoin(url, href)
headlines.append((text, full_url))
if not headlines:
print("No headlines found.")
return
print(f"\nFound {len(headlines)} headlines:\n")
for i, (title, link) in enumerate(headlines, 1):
print(f"{i}. {title}")
print(f" {link}")
print("-" * 40)
except requests.exceptions.RequestException as e:
print(f"Error fetching URL: {e}")
except Exception as e:
print(f"An error occurred: {e}")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="News Headline Scraper")
parser.add_argument("url", nargs="?", default="https://www.drudgereport.com", help="News website URL (default: Drudge Report)")
parser.add_argument("--search", help="Filter headlines by keyword")
args = parser.parse_args()
url = args.url
if not url.startswith("http"):
url = "https://" + url
scrape_headlines(url, args.search)Usage
# Scrape Drudge Report (default)
python news_scraper.py
# Scrape another site
python news_scraper.py https://news.ycombinator.com
# Scrape and filter for specific topic
python news_scraper.py --search "tech"