Playwright爬虫

patchright(Playwright的变种版本)

仅限chromium内核浏览器
https://github.com/Kaliiiiiiiiii-Vinyzu/patchright-python

pip install patchright
patchright install chromium
patchright install chrome 官方建议

python
from patchright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch_persistent_context(
    user_data_dir="...",
    channel="chrome", #也可以不使用chrome,甚至自己指定本地浏览器参考Playwright相同方法即可
    headless=False,
    no_viewport=True,
)
    page = browser.new_page()
    page.goto('http://playwright.dev')
    browser.close()

Playwright支持运行js的虚拟浏览器

pip install playwright
安装系统依赖和指定浏览器内核，如果不需要浏览器不用执行下面的操作
playwright install --with-deps chromium
或者
playwright install 初始化并安装浏览器内核
playwright install-deps安装系统额外依赖

示例

python
from playwright.sync_api import sync_playwright  # 无头浏览器

with sync_playwright() as p:
	url = "https://www.baidu.com"
	browser = p.chromium.launch(headless=True)  # 无头模式
	page = browser.new_page() #新页面
	page.goto(url, wait_until="domcontentloaded") #页面访问网址

	# 指定元素路径
	article_selector = "#app > div > div.box-kr-article-new-y"
	# 等待文章内容区域加载
	try:
		article_content = page.wait_for_selector(article_selector,timeout=15000) #此代码会等待元素加载后才会继续后面的代码
	except Exception:
		#15秒后还未出现会抛出错误，这里可以执行跳过
        pass
	# 获取HTML内容
	html_content = article_content.inner_html()
	# print(f"当前html_content:{html_content}")
	# 使用BeautifulSoup处理HTML
	soup = BeautifulSoup(html_content, "html.parser")

Readability.js 火狐浏览器阅读模式

不支持图片
https://github.com/mozilla/readability
在playwright中注入，待验证

python
page.add_script_tag(content=readability_script)
# 提取内容
result = page.evaluate("""
() => {
    const article = new Readability(document).parse();
    return article;
}
""")

过检测测试代码

python
from playwright.sync_api import sync_playwright
from time import sleep
def playwright_bot(url: str):
    with sync_playwright() as p:
        # 启动浏览器，设置不同的参数
        browser_args = []
        browser = p.firefox.launch( #使用火狐浏览器，无头模式下更好
            headless=False, # 有头模式，显示浏览器
            args=browser_args,
            timeout=99000  # 浏览器启动超时
            )
        # 创建浏览器上下文
        browser_context = browser.new_context(
            viewport={'width': 1366, 'height': 768}, # 设置浏览器窗口大小
            java_script_enabled=True, # 启用JavaScript
            ignore_https_errors=True, # 忽略HTTPS错误
            user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', # 设置用户代理
        )
        browser_context.add_init_script(path="crowdpulse/info_gathering/stealth.min.js")
        page = browser_context.new_page()

        # 设置请求头
        headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'Accept-Language': 'zh-CN,zh;q=0.9,zh-TW;q=0.8,en-US;q=0.7,ja;q=0.6,ko;q=0.5',
            'Accept-Encoding': 'gzip, deflate, br',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1'
        }
        page.set_extra_http_headers(headers)

        # 设置无限期等待
        page.goto(url,timeout=9999000)
        
        # 循环等待，防止自动关闭浏览器
        while browser.is_connected():
            sleep(3)  # 每秒检查一次浏览器状态
        # while True:
        #     user_input = input("输入exit关闭浏览器")
        #     if user_input.lower() == "exit":
        #         break


if __name__ == "__main__":
    playwright_bot("https://bot.sannysoft.com/")

playwright隔离IP和缓存

python
from playwright.sync_api import sync_playwright
import os


# 传入用户id,返回路径
def get_user_data_dir(user_id: str) -> str:
    """
    传入用户id,返回路径，如果没有则会创建
    """
    # 取本环境目录
    current_dir = os.path.dirname(__file__)
    # 拼装用户目录
    user_data_dir = os.path.join(current_dir, "browser_profiles", user_id)

    # 判断用户目录是否存在
    if os.path.exists(user_data_dir):
        return user_data_dir
    # 如果不存在则创建
    try:
        os.makedirs(user_data_dir, exist_ok=True)
    except Exception as e:
        print(f"创建用户目录失败: {e}")
        return None

    # 判断是否创建成功
    if os.path.exists(user_data_dir):
        return user_data_dir
    return None

# 用 Playwright 抓取网页内容（包括 JS 渲染）
def playwright_bot(url: str, user_id: str, proxy: str | dict = None):
    """
    使用完全隔离的浏览器环境访问网页
    
    Args:
        url: 要访问的网址
        user_id: 用户ID，用于创建隔离的浏览器配置文件
        proxy: 代理服务器地址(字符串)或代理配置对象(字典)
    """
    # 得到用户数据目录
    user_data_dir = get_user_data_dir(user_id)
    if not user_data_dir:
        print(f"用户数据目录创建失败")
        return
        
    with sync_playwright() as p:
        # 配置选项
        browser_context_options = {
            # 浏览器窗口选项
            "viewport": {'width': 1366, 'height': 768},
            # 浏览器功能选项
            "java_script_enabled": True,
            "ignore_https_errors": True,
            "headless": False,
            # 身份标识
            "user_agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
            # 超时设置
            "timeout": 99000,
        }
        
        # 如果提供了代理，添加到选项中
        if proxy:
            browser_context_options["proxy"] = proxy
        
        # 启动Firefox并直接创建持久化上下文（只会打开一个页面）
        browser = p.firefox.launch_persistent_context(
            user_data_dir=user_data_dir,
            **browser_context_options
        )
        
        # 直接使用第一个页面
        page = browser.pages[0]
        
        # 添加反机器人检测脚本
        stealth_js_path = os.path.join(os.path.dirname(__file__), "stealth.min.js")
        if os.path.exists(stealth_js_path):
            page.add_init_script(path=stealth_js_path)
            page.add_init_script("""
                //伪装内存
                Object.defineProperty(navigator, 'deviceMemory', {
                    get: () => 8
                });
                //拒绝电量检测
                Object.defineProperty(navigator, 'getBattery', {
                    configurable: true,
                    writable: true,
                    value: function() {
                return Promise.reject(new Error('Not a getBattery'));
                }
                });                   
            """)
        else:
            print(f"stealth.min.js 文件不存在")
            return
        
        # 设置请求头
        headers = {
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'Accept-Language': 'zh-CN,zh;q=0.9,zh-TW;q=0.8,en-US;q=0.7,ja;q=0.6,ko;q=0.5',
            'Accept-Encoding': 'gzip, deflate, br',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1'
        }
        page.set_extra_http_headers(headers)

        # 导航到目标网址
        page.goto(url, timeout=60000)
        
        # 循环等待，防止自动关闭浏览器
        while True:
            # 输入exit退出
            if input("输入exit退出") == "exit":
                break
        # page.pause() # 暂停
     
if __name__ == "__main__":
    playwright_bot(
        url="https://fingerprint-scan.com/",
        user_id="11112",
        proxy={"server": "http://172.27.176.1:10810",
            "username": "admin",
            "password": "admin"}
    )

防检测js注入脚本

js
//伪装内存
Object.defineProperty(navigator, 'deviceMemory', {
    get: () => 8
});
//拒绝电量检测
Object.defineProperty(navigator, 'getBattery', {
    configurable: true,
    writable: true,
    value: function() {
return Promise.reject(new Error('Not a getBattery'));
}
});
// 伪装硬件并发
Object.defineProperty(navigator, 'hardwareConcurrency', {
    get: () => 8
});
// 处理Canvas指纹
const originalToDataURL = HTMLCanvasElement.prototype.toDataURL;
HTMLCanvasElement.prototype.toDataURL = function(type) {
    if (this.width === 0 && this.height === 0) {
        return originalToDataURL.apply(this, arguments);
    }
    
    // 对某些Canvas调用添加微小噪音
    const context = this.getContext('2d');
    const imageData = context.getImageData(0, 0, this.width, this.height);
    const pixels = imageData.data;
    
    // 随机修改几个像素点，幅度很小以避免视觉差异
    for (let i = 0; i < pixels.length; i += 4) {
        // 只修改1/1000的像素
        if (Math.random() < 0.001) {
            pixels[i] = Math.max(0, Math.min(255, pixels[i] + Math.floor(Math.random() * 3) - 1));
            pixels[i+1] = Math.max(0, Math.min(255, pixels[i+1] + Math.floor(Math.random() * 3) - 1));
            pixels[i+2] = Math.max(0, Math.min(255, pixels[i+2] + Math.floor(Math.random() * 3) - 1));
        }
    }
    
    context.putImageData(imageData, 0, 0);
    return originalToDataURL.apply(this, arguments);
};

// 处理WebGL指纹
const getParameterProxyHandler = {
    apply: function(target, thisArg, args) {
        const param = args[0];
        // 修改特定的WebGL参数
        if (param === 37445) { // UNMASKED_VENDOR_WEBGL
            return 'Intel Inc.';
        }
        if (param === 37446) { // UNMASKED_RENDERER_WEBGL
            return 'Intel Iris OpenGL Engine';
        }
        return target.apply(thisArg, args);
    }
};

// 如果WebGL可用，则修改其getParameter方法
if (window.WebGLRenderingContext) {
    const prototype = WebGLRenderingContext.prototype;
    const getParameter = prototype.getParameter;
    prototype.getParameter = new Proxy(getParameter, getParameterProxyHandler);
}

about:support 火狐查看当前浏览器信息

ws连接浏览器

python
from playwright.sync_api import sync_playwright
import os
import requests

# 获取ws地址
def get_ws_url(port: int):
    return requests.get(f"http://localhost:{port}/json/version").json()["webSocketDebuggerUrl"]


def playwright_bot(url: str, port: int=9000):

    with sync_playwright() as p:
        # 持久化配置选项
        browser = p.chromium.connect_over_cdp(get_ws_url(port))
        # 2. 获取所有已存在的浏览上下文（通常只有一个持久化上下文）
        contexts = browser.contexts
        if not contexts:
            print("未找到任何上下文")
            return
        context = contexts[0]  # 取第一个上下文（通常就是你启动时默认的）

        # 3. 获取该上下文下所有已打开的页面
        pages = context.pages
        if not pages:
            print("当前上下文中没有已打开的页面")
            return
        page = pages[0]  # 取第一个已打开的页面
        # page.reload()
        page.goto(url)
        # # 循环等待，防止自动关闭浏览器
        # while True:
        #     # 输入exit退出
        #     if input("输入exit退出") == "exit":
        #         break

if __name__ == "__main__":
    playwright_bot(
        url="https://fingerprint-scan.com/",
        port=9000,
    )

    # print(get_ws_url(9000))

ubuntu启动浏览器命令
brave-browser --remote-allow-origins=* --remote-debugging-port=9000 --disable-blink-features=AutomationControlled --window-size=1344,768 --no-first-run --no-default-browser-check

启动后可通过http://localhost:{debug_port}/json/version

css
brave-browser \
  --remote-debugging-port=5880 \
  --remote-allow-origins=* \
  --user-data-dir=/path/to/user/data/dir \
  --profile-directory=Default \
  --lang=zh-CN \
  --disable-blink-features=AutomationControlled \
  --window-size=1344,768 \
  --start-maximized \
  --enable-extensions \
  --no-first-run \
  --no-default-browser-check \
  --disable-brave-update \
  --disable-logging \
  --disable-breakpad \
  --disable-machine-id

windows将brave添加到环境后：
brave --remote-debugging-port=5880 --remote-debugging-address=0.0.0.0 --remote-allow-origins=* --profile-directory=Default --lang=zh-CN --window-size=1344,768 --start-maximized --enable-extensions --no-first-run --no-default-browser-check --disable-brave-update --disable-machine-id

## browserforge 随机指纹

pip install browserforge 安装
python -m browserforge update 第一次使用前，下载模型文件

python
from browserforge.headers import HeaderGenerator
headers = HeaderGenerator()
headers.generate() # 生成标头

# 生成部分参数指定标头
headers = HeaderGenerator(
    browser='chrome',
    os='windows',
    device='desktop',
    locale=('en-US', 'en', 'de'),
    http_version=2
)

# 生成指纹
from browserforge.fingerprints import FingerprintGenerator
fingerprints = FingerprintGenerator()
fingerprints.generate()

#生成部分指定参数指纹
from browserforge.fingerprints import Screen

screen = Screen(
    min_width=100
    max_width=1280
    min_height=400
    max_height=720
)

fingerprints = FingerprintGenerator(screen=screen)

browserforge注入playwright

python
from browserforge.injectors.playwright import NewContext

def main():
    with sync_playwright() as playwright:
        browser = playwright.chromium.launch()
        # Create a new context with the injected fingerprint
        context = NewContext(browser, fingerprint=fingerprint)
        page = context.new_page()

pjstealth随机指纹(不可复用)

https://github.com/winner-hue/pjstealth

pip install pjstealth

python
from playwright.sync_api import sync_playwright
from pjstealth import stealth_sync

p = sync_playwright().start()
proxy = {
    "server": "127.0.0.1:7890"
}
browser = p.chromium.launch(headless=False)

page = browser.new_page(
    user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
    proxy=proxy)
stealth_sync(page) # 修改指纹
page.goto("https://web.uutool.cn/")

不显示中文乱码解决

bash
# 安装中文字体
sudo apt install -y fonts-noto-cjk fonts-wqy-zenhei fonts-wqy-microhei fonts-noto-color-emoji fontconfig

# 2) 刷新字体缓存
sudo fc-cache -fv