import requests
def web_crawler(start_url, max_pages=100):
    pages_to_visit = [start_url]
    visited_pages = set()
    page_count = 0

    while pages_to_visit and page_count < max_pages:
        current_url = pages_to_visit.pop(0)

        # Avoid re-visiting already processed URLs
        if current_url in visited_pages:
            continue

        try:
            response = requests.get(current_url)
            response.raise_for_status()  # Raise HTTPError for bad responses (4xx or 5xx)
            page_content = response.text

            # Mark the current URL as visited to avoid loops
            visited_pages.add(current_url)
            page_count += 1

            # Extract all the links from the current webpage
            new_urls = extract_links(page_content, current_url)

            # Add discovered links for future exploration
            for new_url in new_urls:
                if new_url not in visited_pages and new_url not in pages_to_visit:
                    pages_to_visit.append(new_url)

        except requests.exceptions.RequestException as e:
            print(f"Error crawling {current_url}: {e}")

    return visited_pages

def extract_links(page_content, base_url):
    from bs4 import BeautifulSoup
    soup = BeautifulSoup(page_content, 'html.parser')
    links = []
    for anchor in soup.find_all('a', href=True):
        link = anchor['href']
        absolute_url = get_absolute_url(base_url, link)
        if absolute_url:
            links.append(absolute_url)
    return links

def get_absolute_url(base_url, url):
    from urllib.parse import urljoin, urlparse
    absolute_url = urljoin(base_url, url)
    parsed_url = urlparse(absolute_url)
    if parsed_url.scheme not in ('http', 'https'):
        return None
    return absolute_url

import requests
from bs4 import BeautifulSoup
from urllib.parse import urljoin, urlparse

def web_crawler(seed_url, max_urls=100):
    visited_urls = set()
    urls_to_visit = [seed_url]
    crawled_urls_count = 0

    while urls_to_visit and crawled_urls_count < max_urls:
        current_url = urls_to_visit.pop(0)

        # Avoid re-crawling already processed URLs.
        if current_url in visited_urls:
            continue

        try:
            response = requests.get(current_url)
            response.raise_for_status()

            soup = BeautifulSoup(response.text, 'html.parser')

            # This extracts all the links from the current webpage.
            for anchor in soup.find_all('a', href=True):
                absolute_url = urljoin(current_url, anchor['href'])

                # Normalize URLs to prevent duplicates.
                parsed_url = urlparse(absolute_url)
                normalized_url = parsed_url.scheme + "://" + parsed_url.netloc + parsed_url.path

                # Avoid re-crawling already processed URLs.
                if normalized_url not in visited_urls:
                    urls_to_visit.append(normalized_url)

        except requests.exceptions.RequestException as e:
            print(f"Error crawling {current_url}: {e}")

        visited_urls.add(current_url)
        crawled_urls_count += 1

    return visited_urls