通用爬虫工具

import requests
from bs4 import BeautifulSoup

# 定义一个请求函数
def test_get(url: str):
	# 定义一个请求头
	headers = {
	            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
	        }
	# 设置超时和重试
	timeout = 20
	retries = 5
	for _ in range(retries):
	    response = requests.get(url, headers=headers, timeout=timeout, verify=False) #不效验证书
	    response.raise_for_status() # 如果请求失败，抛出异常
	    return response

response = test_get("https://www.baidu.com")
# 取源码元素
soup = BeautifulSoup(response.text, "html.parser")
# 找到指定的div元素
target_div = soup.select_one("#__next > main > div")
# 提取纯文本
if target_div:
    # 使用separator=' ' 并设置strip=False保留换行符
    text = target_div.get_text(separator="\n", strip=False)
    # 清理多余的空行
    text = "\n".join(
        [line.strip() for line in text.split("\n") if line.strip()]
    )
else:
    text = None

from playwright.sync_api import sync_playwright
import trafilatura
from time import sleep

# 用 Playwright 抓取网页内容（包括 JS 渲染）
def playwright_bot(url: str, headless_mode=True) -> str:
    with sync_playwright() as p:
        try:
            # 启动浏览器，设置不同的参数
            browser_args = []
            browser = p.firefox.launch( #使用火狐浏览器，无头模式下更好
                headless=headless_mode, # 默认无头模式，第二次传入有头模式
                args=browser_args,
                timeout=60000  # 浏览器启动超时
                )
            # 创建浏览器上下文
            context = browser.new_context(
                viewport={'width': 1920, 'height': 1080}, # 设置浏览器窗口大小
                java_script_enabled=True, # 启用JavaScript
                ignore_https_errors=True, # 忽略HTTPS错误
                user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', # 设置用户代理
            )
            page = context.new_page()

            # 设置请求头
            headers = {
                'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
                'Accept-Language': 'zh-CN,zh;q=0.9,zh-TW;q=0.8,en-US;q=0.7,ja;q=0.6,ko;q=0.5',
                'Accept-Encoding': 'gzip, deflate, br',
                'Connection': 'keep-alive',
                'Upgrade-Insecure-Requests': '1'
            }
            page.set_extra_http_headers(headers)

            # 设置超时时间,并等待页面加载完成
            page.goto(url, timeout=60000, wait_until='domcontentloaded')
            # 尝试等待网络空闲
            try:
                page.wait_for_load_state('networkidle', timeout=10000)
            except Exception as e:
                print(f"等待网络空闲超时，但会继续: {e}")
                # 网络不空闲也没关系，我们可以加一个短暂延时
                sleep(5)
            page_content = page.content()
            browser.close()
            return page_content
        except Exception as e:
            print(f"{url}\\n抓取失败: {e}")
            return ""

# 用 trafilatura 抽取网页正文
def trafilatura_bot(html: str) -> str:
    result = trafilatura.extract(html)
    return result

# 通用爬虫
def General_spider(url: str) -> str:
    html = playwright_bot(url)
    if not html:
        print("无头模式抓取失败，尝试有头模式")
        # 如果无头模式抓取失败，尝试有头模式
        html = playwright_bot(url, headless_mode=False)
    
    # 如果有内容则提取
    if html:
        text = trafilatura_bot(html)
        return text
    else:
        return ""
        
# 测试
if __name__ == "__main__":
    print(General_spider("https://www.mckinsey.com"))

from goose3 import Goose
url = 'http://baidu.com'
g = Goose()
article = g.extract(url=url)
#article = g.extract(raw_html=html, url=url) # 如果使用playwright可手动传递html，待验证
article.title #标题
article.meta_description #摘要
article.cleaned_text #正文
article.top_image.src # 最相关图片
article.movies #视频

{
  "partner_id": "web",
  "param": {
    "pageSize": 20,
    "pageEvent": 0,
    "siteId": 1,
    "type": 1,
    "platformId": 2
  }
}

import os
from fundus import PublisherCollection, Crawler, Requires

# 使用代理
os.environ['HTTPS_PROXY'] = 'http://172.27.176.1:10810'

# 取美国信息
crawler = Crawler(PublisherCollection.us)

def get_fundus_news(num:int=2) -> list:
    news = crawler.crawl(max_articles=num)
	#news = crawler.crawl(max_articles=num, only_complete=Requires("title", "body")) #只保留有正文和标题的文章
    for item in news:
        print(f"fundus新闻抓取成功: {item}")
        print(f"正文: {item.plaintext}")
        print(f"标题: {item.title}")
        print(f"body正文: {item.body}")
		print(f"url: {item.html.requested_url}")
        # 序列化，可以指定提取某些字段
        article_json = item.to_json("title", "plaintext", "lang")
        print(f"序列化: {article_json}")

from newspaper import Article,news_pool,Config,build
import newspaper

# 配置代理
proxies = {
    'http': 'http://172.27.176.1:10810',
    'https': 'http://172.27.176.1:10810'
}
config = Config()
config.proxies = proxies

# 定义一个从cnn获取新闻的源
def get_cnn_news() -> list:
    # 创建新闻源
    cnn_paper = newspaper.build('https://www.cnn.com/sitemap/news.xml', config=config, language='en')
    list_news = []
    # 获取文章
    for article in cnn_paper.articles[:10]:
        article.download()
        article.parse()
        list_news.append({
            "title": article.title,
            "text": article.text,
            "url": article.url,
			"image":article..top_image #规则判断的最佳图片
        })
    return list_news


# 下面是单个文章的测试
def get_single_article(url:str) -> dict:
    article = Article(url) # 创建文章对象
    article.download() # 下载文章
    # print(article.html)
    article.parse() # 解析文章
    print(article.title) # 获取标题
    print(article.text) # 获取文章的正文
    return {
        "title": article.title,
        "text": article.text,
        "url": article.url,
		"images":article.images
    }
    # print(article.authors) # 获取文章的作者
    # print(article.publish_date) # 获取文章的发布时间
    # print(article.top_image) # 获取文章的封面图片
    # article.nlp() # 对文章进行自然语言处理,需要额外安装nltk包
    # print(article.keywords) # 获取文章的关键词


# 多个新闻源多线程抓取
def fetch_news_from_sources(sources: list[str], threads_per_source: int = 2) -> list[dict]:
    """
    从多个新闻源并行抓取文章，并返回 url、标题、正文
    Args:
        sources: 新闻源 list[str]
        threads_per_source: 每个新闻源分配的线程数，默认为2
    Returns:
        一个包含若干字典的列表，每个字典包含 'url', 'title', 'text' 三个字段
    """
    # 构建每个新闻源的 Paper 对象，并使用代理
    papers = [build(src,config=config) for src in sources]  # 构建新闻源
    
    # 设置全局线程池：sources 个源 * threads_per_source 线程
    news_pool.set(papers, threads_per_source=threads_per_source)  # 为每个源分配指定线程数
    news_pool.join()  # 等待所有文章 download() 完成
    
    results = []  # 用来存放最终结果
    for paper in papers:
        for article in paper.articles:
            try:
                article.parse()  # 解析下载好的 HTML，提取标题和正文
                obj = {
                    'url':   article.url,     # 文章链接
                    'title': article.title,   # 文章标题
                    'text':  article.text     # 文章正文
                }
                results.append(obj)
                print(f"解析成功: {obj}")
            except Exception:
                # 如果解析失败，则忽略这篇文章
                print(f"解析失败: {article.url}")
                continue

    return results  # 返回包含所有文章信息的列表



# 测试
if __name__ == "__main__":
    # print(get_cnn_news())
    # get_single_article('https://www.cnn.com/2025/05/22/us/harvard-university-trump-international-students')
    sources = ['https://edition.cnn.com/world', 'https://www.bbc.com', 'https://www.bloomberg.com/']
    fetch_news_from_sources(sources)

from newspaper import Article
from playwright.sync_api import sync_playwright

# 使用 Playwright 获取完整渲染后的 HTML
with sync_playwright() as p:
    browser = p.chromium.launch(headless=True)
    page = browser.new_page()
    page.goto(url)
    html = page.content() # 得到网页html
    browser.close()

# 创建一个 Article 对象
article = Article(url)
article.set_html(html)  # 手动设置 HTML 内容
article.parse()