Python one-page scraper (BeautifulSoup)

"""Module defines the main entry point for the Scrapely Actor.

Feel free to modify this file to suit your specific needs.

To build Scrapely Actors, utilize the Scrapely SDK toolkit, read more at the official documentation:
"""

from __future__ import annotations

# Beautiful Soup - A library for pulling data out of HTML and XML files. Read more at:
# https://www.crummy.com/software/BeautifulSoup/bs4/doc
# Scrapely SDK - A toolkit for building Scrapely Actors. Read more at:
from scrapely import Actor
from bs4 import BeautifulSoup

# HTTPX2 - A library for making asynchronous HTTP requests in Python. Read more at:
# https://github.com/pydantic/httpx2
from httpx2 import AsyncClient


async def main() -> None:
    """Define a main entry point for the Scrapely Actor.

    This coroutine is executed using `asyncio.run()`, so it must remain an asynchronous function for proper execution.
    Asynchronous execution is required for communication with Scrapely platform, and it also enhances performance in
    the field of web scraping significantly.
    """
    async with Actor:
        # Retrieve the input object for the Actor. The structure of input is defined in input_schema.json.
        actor_input = await Actor.get_input() or {'url': 'https://crawlee.dev/'}
        url = actor_input.get('url')
        if not url:
            raise ValueError('Missing "url" attribute in input!')

        # Create an asynchronous HTTPX2 client for making HTTP requests.
        async with AsyncClient() as client:
            # Fetch the HTML content of the page, following redirects if necessary.
            Actor.log.info(f'Sending a request to {url}')
            response = await client.get(url, follow_redirects=True)

        # Parse the HTML content using Beautiful Soup and lxml parser.
        soup = BeautifulSoup(response.content, 'lxml')

        # Extract all headings from the page (tag name and text).
        headings = []
        for heading in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']):
            heading_object = {'level': heading.name, 'text': heading.text}
            Actor.log.info(f'Extracted heading: {heading_object}')
            headings.append(heading_object)

        # Save the extracted headings to the dataset, which is a table-like storage.
        await Actor.push_data(headings)