Beautifulsoup Parsing

from bs4 import BeautifulSoupimport requests
# From stringhtml = '<html><body><h1>Hello</h1></body></html>'soup = BeautifulSoup(html, 'lxml')
# From filewith open('page.html', 'r', encoding='utf-8') as f:    soup = BeautifulSoup(f, 'lxml')
# From URLresponse = requests.get('https://example.com')soup = BeautifulSoup(response.content, 'lxml')

# lxml - Fast, lenient (recommended)soup = BeautifulSoup(html, 'lxml')
# html.parser - Built-in, no dependenciessoup = BeautifulSoup(html, 'html.parser')
# html5lib - Most lenient, slowestsoup = BeautifulSoup(html, 'html5lib')
# lxml-xml - For XML documentssoup = BeautifulSoup(xml, 'lxml-xml')

# First matching elementsoup.find('h1')
# All matching elementssoup.find_all('p')
# Shorthandsoup.h1  # Same as soup.find('h1')

# By classsoup.find('div', class_='article')soup.find_all('div', class_='article')
# By IDsoup.find(id='main-content')
# By any attributesoup.find('a', href='https://example.com')soup.find_all('input', attrs={'type': 'text', 'name': 'email'})
# By data attributessoup.find('div', attrs={'data-id': '123'})

# Single elementsoup.select_one('div.article > h2')
# Multiple elementssoup.select('div.article h2')
# Complex selectorssoup.select('a[href^="https://"]')  # Starts withsoup.select('a[href$=".pdf"]')      # Ends withsoup.select('a[href*="example"]')   # Containssoup.select('li:nth-child(2)')soup.select('h1, h2, h3')           # Multiple

import re
# By regexsoup.find_all('a', href=re.compile(r'^https://'))
# By functiondef has_data_attr(tag):    return tag.has_attr('data-id')
soup.find_all(has_data_attr)
# String matchingsoup.find_all(string='exact text')soup.find_all(string=re.compile('pattern'))

# Get textelement.textelement.get_text()
# Get text with separatorelement.get_text(separator=' ')
# Get stripped textelement.get_text(strip=True)
# Get strings (generator)for string in element.stripped_strings:    print(string)

# Get attributeelement['href']element.get('href')  # Returns None if missingelement.get('href', 'default')  # With default
# Get all attributeselement.attrs  # Returns dict
# Check attribute existselement.has_attr('class')

# Inner HTMLstr(element)
# Just the tagelement.name
# Prettified HTMLelement.prettify()

element.parentelement.parents  # Generator of all ancestors
# Find specific ancestorfor parent in element.parents:    if parent.name == 'div' and 'article' in parent.get('class', []):        break

element.children      # Direct children (generator)list(element.children)
element.contents      # Direct children (list)element.descendants   # All descendants (generator)
# Find in childrenelement.find('span')  # Searches descendants

element.next_siblingelement.previous_sibling
element.next_siblings      # Generatorelement.previous_siblings  # Generator
# Next/previous element (skips whitespace)element.next_elementelement.previous_element

def safe_text(element, selector, default=''):    """Safely extract text from element."""    found = element.select_one(selector)    return found.get_text(strip=True) if found else default
def safe_attr(element, selector, attr, default=None):    """Safely extract attribute from element."""    found = element.select_one(selector)    return found.get(attr, default) if found else default

def extract_table(table):    """Extract table data as list of dictionaries."""    headers = [th.get_text(strip=True) for th in table.select('th')]
    rows = []    for tr in table.select('tbody tr'):        cells = [td.get_text(strip=True) for td in tr.select('td')]        if cells:            rows.append(dict(zip(headers, cells)))
    return rows

def extract_items(soup, selector, extractor):    """Extract multiple items using a custom extractor function."""    return [extractor(item) for item in soup.select(selector)]
# Usagedef extract_product(item):    return {        'name': safe_text(item, '.name'),        'price': safe_text(item, '.price'),        'url': safe_attr(item, 'a', 'href')    }
products = extract_items(soup, '.product', extract_product)

from urllib.parse import urljoin
def resolve_url(base_url, relative_url):    """Convert relative URL to absolute."""    if not relative_url:        return None    return urljoin(base_url, relative_url)
# Usagebase_url = 'https://example.com/products/'for link in soup.select('a'):    href = link.get('href')    absolute_url = resolve_url(base_url, href)    print(absolute_url)

# lxml parser is lenient with malformed HTMLsoup = BeautifulSoup(malformed_html, 'lxml')
# For very broken HTML, use html5libsoup = BeautifulSoup(very_broken_html, 'html5lib')
# Handle encoding issuesresponse = requests.get(url)response.encoding = response.apparent_encodingsoup = BeautifulSoup(response.text, 'lxml')

import requestsfrom bs4 import BeautifulSoupfrom urllib.parse import urljoinimport time
class ProductScraper:    def __init__(self, base_url):        self.base_url = base_url        self.session = requests.Session()        self.session.headers.update({            'User-Agent': 'Mozilla/5.0 (compatible; MyScraper/1.0)'        })
    def fetch_page(self, url):        """Fetch and parse a page."""        response = self.session.get(url, timeout=30)        response.raise_for_status()        return BeautifulSoup(response.content, 'lxml')
    def extract_product(self, item):        """Extract product data from a card element."""        return {            'name': self._safe_text(item, '.product-title'),            'price': self._parse_price(item.select_one('.price')),            'rating': self._safe_attr(item, '.rating', 'data-rating'),            'image': self._resolve(self._safe_attr(item, 'img', 'src')),            'url': self._resolve(self._safe_attr(item, 'a', 'href')),            'in_stock': not item.select_one('.out-of-stock')        }
    def scrape_products(self, url):        """Scrape all products from a page."""        soup = self.fetch_page(url)        items = soup.select('.product-card')        return [self.extract_product(item) for item in items]
    def _safe_text(self, element, selector, default=''):        found = element.select_one(selector)        return found.get_text(strip=True) if found else default
    def _safe_attr(self, element, selector, attr, default=None):        found = element.select_one(selector)        return found.get(attr, default) if found else default
    def _parse_price(self, element):        if not element:            return None        text = element.get_text(strip=True)        try:            return float(text.replace('$', '').replace(',', ''))        except ValueError:            return None
    def _resolve(self, url):        return urljoin(self.base_url, url) if url else None
# Usagescraper = ProductScraper('https://example.com')products = scraper.scrape_products('https://example.com/products')for product in products:    print(product)

# Use SoupStrainer to parse only needed elementsfrom bs4 import SoupStrainer
only_articles = SoupStrainer('article')soup = BeautifulSoup(html, 'lxml', parse_only=only_articles)
# Use lxml parser for speedsoup = BeautifulSoup(html, 'lxml')  # Fastest
# Decompose unneeded elementsfor script in soup.find_all('script'):    script.decompose()
# Use generators for memory efficiencyfor item in soup.select('.item'):    yield extract_data(item)