From b2f74a24f38791dbf49f549348ef91a619640d26 Mon Sep 17 00:00:00 2001 From: daijro Date: Wed, 9 Oct 2024 05:49:22 -0500 Subject: [PATCH] Update pythonlib & README for beta.11 - Removed warnings for using headless mode - Xvfb is now activated with headless='virtual' - Correctly pass window.screenX & calculate window.screenY - Update documentation on main README and pythonlib README - Bumped pythonlib to 0.2.10 --- README.md | 19 +++++++++------ pythonlib/README.md | 8 +++---- pythonlib/camoufox/async_api.py | 7 +++--- pythonlib/camoufox/exceptions.py | 8 +++++++ pythonlib/camoufox/fingerprints.py | 34 ++++++++++++++++++++++---- pythonlib/camoufox/sync_api.py | 9 +++---- pythonlib/camoufox/utils.py | 38 +++++------------------------- pythonlib/camoufox/virtdisplay.py | 17 ++++++++++++- pythonlib/camoufox/warnings.yml | 20 ++++++++-------- pythonlib/pyproject.toml | 2 +- 10 files changed, 96 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index ca5e978..67eeef9 100644 --- a/README.md +++ b/README.md @@ -440,6 +440,18 @@ Miscellaneous (battery status, etc) - Geolocation, timezone, and locale spoofing - etc. +#### Stealth patches + +- Avoids main world execution leaks. All page agent javascript is sandboxed +- Avoids frame execution context leaks +- Fixes `navigator.webdriver` detection +- Fixes Firefox headless detection via pointer type ([#26](https://github.com/daijro/camoufox/issues/26)) +- Removed potentially leaking anti-zoom/meta viewport handling patches +- Uses non-default screen & window sizes +- Re-enable fission content isolations +- Re-enable PDF.js +- Other leaking config properties changed + #### Anti font fingerprinting - Automatically uses the correct system fonts for your User Agent @@ -450,13 +462,6 @@ Miscellaneous (battery status, etc) - Custom implementation of Playwright for the latest Firefox - Various config patches to evade bot detection -- Fixes leaking Playwright patches: - - All page agent javascript is sandboxed - - Fixes frame execution context leaks - - Fixes `navigator.webdriver` detection - - Removed potentially leaking anti-zoom/meta viewport handling patches - - Re-enable fission content isolation - - Re-enable PDF.js #### Debloat/Optimizations diff --git a/pythonlib/README.md b/pythonlib/README.md index ae892f6..fffeab2 100644 --- a/pythonlib/README.md +++ b/pythonlib/README.md @@ -136,9 +136,9 @@ Parameters: ff_version (Optional[int]): Firefox version to use. Defaults to the current Camoufox version. To prevent leaks, only use this for special cases. - headless (Optional[bool]): + headless (Union[bool, Literal['virtual']]): Whether to run the browser in headless mode. Defaults to False. - WARNING: Please avoid using headless mode until issue #26 is fixed. + If you are running linux, passing 'virtual' will use Xvfb. executable_path (Optional[str]): Custom Camoufox browser executable path. firefox_user_prefs (Optional[Dict[str, Any]]): @@ -270,7 +270,7 @@ with sync_playwright() as p: ### Virtual Display -In headless mode, all browsers are prone to being detected by anti-bot services due to the drastic differences in the browser's architecture. It is generally **NOT** recommended to use Camoufox in headless mode on a non-Linux OS. +While Camoufox includes patches to prevent headless detection, running in headless mode may still be detectable in the future. It's recommended to use a virtual display buffer to run Camoufox headlessly. If you are running Linux, and would like to run Camoufox headlessly in a virtual display, install `xvfb`: @@ -293,7 +293,7 @@ $ which Xvfb /usr/bin/Xvfb ``` -Now, passing `headless=True` will spawn a new lightweight virtual display in the background for Camoufox to run in. +Now, passing `headless='virtual'` will spawn a new lightweight virtual display in the background for Camoufox to run in.
diff --git a/pythonlib/camoufox/async_api.py b/pythonlib/camoufox/async_api.py index 60be5c6..3c55be4 100644 --- a/pythonlib/camoufox/async_api.py +++ b/pythonlib/camoufox/async_api.py @@ -7,6 +7,7 @@ from playwright.async_api import ( Playwright, PlaywrightContextManager, ) +from typing_extensions import Literal from .addons import DefaultAddons from .utils import ListOrString, _clean_locals, get_launch_options @@ -51,7 +52,7 @@ async def AsyncNewBrowser( screen: Optional[Screen] = None, fingerprint: Optional[Fingerprint] = None, ff_version: Optional[int] = None, - headless: Optional[bool] = None, + headless: Optional[Union[bool, Literal['virtual']]] = None, executable_path: Optional[str] = None, firefox_user_prefs: Optional[Dict[str, Any]] = None, proxy: Optional[Dict[str, str]] = None, @@ -105,9 +106,9 @@ async def AsyncNewBrowser( ff_version (Optional[int]): Firefox version to use. Defaults to the current Camoufox version. To prevent leaks, only use this for special cases. - headless (Optional[bool]): + headless (Union[bool, Literal['virtual']]): Whether to run the browser in headless mode. Defaults to False. - WARNING: Please avoid using headless mode until issue #26 is fixed. + If you are running linux, passing 'virtual' will use Xvfb. executable_path (Optional[str]): Custom Camoufox browser executable path. firefox_user_prefs (Optional[Dict[str, Any]]): diff --git a/pythonlib/camoufox/exceptions.py b/pythonlib/camoufox/exceptions.py index c3d02b3..9a02abe 100644 --- a/pythonlib/camoufox/exceptions.py +++ b/pythonlib/camoufox/exceptions.py @@ -141,3 +141,11 @@ class CannotExecuteXvfb(VirtualDisplayError): """ ... + + +class VirtualDisplayNotSupported(VirtualDisplayError): + """ + Raised when the user tried to use a virtual display on a non-Linux OS. + """ + + ... diff --git a/pythonlib/camoufox/fingerprints.py b/pythonlib/camoufox/fingerprints.py index 19e79a5..2bae12e 100644 --- a/pythonlib/camoufox/fingerprints.py +++ b/pythonlib/camoufox/fingerprints.py @@ -1,8 +1,9 @@ import re from dataclasses import asdict +from random import randrange from typing import Any, Dict, Optional -from browserforge.fingerprints import Fingerprint, FingerprintGenerator +from browserforge.fingerprints import Fingerprint, FingerprintGenerator, Screen from camoufox.pkgman import load_yaml @@ -30,15 +31,38 @@ def _cast_to_properties( if isinstance(data, dict): _cast_to_properties(camoufox_data, type_key, data, ff_version) continue - # Fix values that are out of bounds - if type_key.startswith("screen.") and isinstance(data, int) and data < 0: - data = 0 # Replace the Firefox versions with ff_version if ff_version and isinstance(data, str): data = re.sub(r'(? None: + """ + Helper method to set window.screenY based on Browserforge's screenX value. + """ + # Default screenX to 0 if not provided + screenX = fp_screen.screenX + if not screenX: + camoufox_data['window.screenX'] = 0 + camoufox_data['window.screenY'] = 0 + return + + # If screenX is within [-50, 50], use the same value for screenY + if screenX in range(-50, 51): + camoufox_data['window.screenY'] = screenX + return + + # Browserforge thinks the browser is windowed. # Randomly generate a screenY value. + screenY = fp_screen.availHeight - fp_screen.outerHeight + if screenY == 0: + camoufox_data['window.screenY'] = 0 + elif screenY > 0: + camoufox_data['window.screenY'] = randrange(0, screenY) # nosec + else: + camoufox_data['window.screenY'] = randrange(screenY, 0) # nosec + + def from_browserforge(fingerprint: Fingerprint, ff_version: Optional[str] = None) -> Dict[str, Any]: """ Converts a Browserforge fingerprint to a Camoufox config. @@ -50,6 +74,8 @@ def from_browserforge(fingerprint: Fingerprint, ff_version: Optional[str] = None bf_dict=asdict(fingerprint), ff_version=ff_version, ) + handle_screenXY(camoufox_data, fingerprint.screen) + return camoufox_data diff --git a/pythonlib/camoufox/sync_api.py b/pythonlib/camoufox/sync_api.py index 85a6cc2..c8c7a7c 100644 --- a/pythonlib/camoufox/sync_api.py +++ b/pythonlib/camoufox/sync_api.py @@ -7,6 +7,7 @@ from playwright.sync_api import ( Playwright, PlaywrightContextManager, ) +from typing_extensions import Literal from .addons import DefaultAddons from .utils import ListOrString, _clean_locals, get_launch_options @@ -51,7 +52,7 @@ def NewBrowser( screen: Optional[Screen] = None, fingerprint: Optional[Fingerprint] = None, ff_version: Optional[int] = None, - headless: Optional[bool] = None, + headless: Optional[Union[bool, Literal['virtual']]] = None, executable_path: Optional[str] = None, firefox_user_prefs: Optional[Dict[str, Any]] = None, proxy: Optional[Dict[str, str]] = None, @@ -61,7 +62,7 @@ def NewBrowser( i_know_what_im_doing: Optional[bool] = None, debug: Optional[bool] = None, **launch_options: Dict[str, Any] -) -> Browser: +) -> Union[Browser, BrowserContext]: """ Launches a new browser instance for Camoufox. Accepts all Playwright Firefox launch options, along with the following: @@ -105,9 +106,9 @@ def NewBrowser( ff_version (Optional[int]): Firefox version to use. Defaults to the current Camoufox version. To prevent leaks, only use this for special cases. - headless (Optional[bool]): + headless (Union[bool, Literal['virtual']]): Whether to run the browser in headless mode. Defaults to False. - WARNING: Please avoid using headless mode until issue #26 is fixed. + If you are running linux, passing 'virtual' will use Xvfb. executable_path (Optional[str]): Custom Camoufox browser executable path. firefox_user_prefs (Optional[Dict[str, Any]]): diff --git a/pythonlib/camoufox/utils.py b/pythonlib/camoufox/utils.py index b720268..273a22c 100644 --- a/pythonlib/camoufox/utils.py +++ b/pythonlib/camoufox/utils.py @@ -3,7 +3,6 @@ import sys from os import environ from pprint import pprint from random import randrange -from shutil import which from typing import Any, Dict, List, Literal, Optional, Tuple, Union, cast import numpy as np @@ -227,35 +226,6 @@ def _clean_locals(data: Dict[str, Any]) -> Dict[str, Any]: return data -def handle_headless( - headless: Optional[bool], - env: Dict[str, Union[str, float, bool]], - debug: Optional[bool], - i_know_what_im_doing: Optional[bool], -) -> bool: - """ - Handles the headless mode. - """ - # If headless is not being used, return False - if not headless: - return False - - # Warn the user if headless is being used on a non-Linux OS - # https://github.com/daijro/camoufox/issues/26 - if OS_NAME != 'lin': - LeakWarning.warn('headless-non-linux', i_know_what_im_doing) - return True - - # If Xvfb is avaliable, use it instead of headless to prevent leaks - if which('Xvfb'): - env['DISPLAY'] = VIRTUAL_DISPLAY.new_or_reuse(debug=debug) - return False - - # If Linux is being used and Xvfb is not avaliable, warn the user - LeakWarning.warn('headless-linux', i_know_what_im_doing) - return True - - def merge_into(target: Dict[str, Any], source: Dict[str, Any]) -> None: """ Merges new keys/values from the source dictionary into the target dictionary. @@ -335,7 +305,7 @@ def get_launch_options( allow_webgl: Optional[bool] = None, proxy: Optional[Dict[str, str]] = None, ff_version: Optional[int] = None, - headless: Optional[bool] = None, + headless: Optional[Union[bool, Literal['virtual']]] = None, firefox_user_prefs: Optional[Dict[str, Any]] = None, launch_options: Optional[Dict[str, Any]] = None, debug: Optional[bool] = None, @@ -348,6 +318,8 @@ def get_launch_options( config = {} # Set default values for optional arguments + if headless is None: + headless = False if addons is None: addons = [] if args is None: @@ -360,7 +332,9 @@ def get_launch_options( env = cast(Dict[str, Union[str, float, bool]], environ) # Handle headless mode cases - headless = handle_headless(headless, env, debug, i_know_what_im_doing) + if headless == 'virtual': + env['DISPLAY'] = VIRTUAL_DISPLAY.new_or_reuse(debug=debug) + headless = False # Warn the user for manual config settings if not i_know_what_im_doing: diff --git a/pythonlib/camoufox/virtdisplay.py b/pythonlib/camoufox/virtdisplay.py index 1e09ffd..4dbc2e6 100644 --- a/pythonlib/camoufox/virtdisplay.py +++ b/pythonlib/camoufox/virtdisplay.py @@ -4,7 +4,12 @@ from glob import glob from shutil import which from typing import List, Optional -from camoufox.exceptions import CannotExecuteXvfb, CannotFindXvfb +from camoufox.exceptions import ( + CannotExecuteXvfb, + CannotFindXvfb, + VirtualDisplayNotSupported, +) +from camoufox.pkgman import OS_NAME class VirtualDisplay: @@ -72,6 +77,8 @@ class VirtualDisplay: """ Get the display number """ + self.assert_linux() + if self.proc is None: self.execute_xvfb_singleton(debug) elif debug: @@ -116,5 +123,13 @@ class VirtualDisplay: self._display = self._free_display() return self._display + @staticmethod + def assert_linux(): + """ + Assert that the current OS is Linux + """ + if OS_NAME != 'lin': + raise VirtualDisplayNotSupported("Virtual display is only supported on Linux.") + VIRTUAL_DISPLAY = VirtualDisplay() diff --git a/pythonlib/camoufox/warnings.yml b/pythonlib/camoufox/warnings.yml index 0aefae7..9d6bee7 100644 --- a/pythonlib/camoufox/warnings.yml +++ b/pythonlib/camoufox/warnings.yml @@ -1,12 +1,3 @@ -headless-non-linux: >- - Headless mode is only recommended on Linux at this time. - Some WAFs are able to detect headless browsers. The issue is currently being investigated. - -headless-linux: >- - Headless mode is only recommended on Linux with Xvfb installed. - Please see the install guide here: - https://github.com/daijro/camoufox/tree/main/pythonlib#virtual-display - navigator: >- Manually setting navigator properties is not recommended. Device information is automatically generated within Camoufox @@ -39,4 +30,13 @@ allow_webgl: >- ff_version: >- Spoofing the Firefox version will likely lead to detection. If rotating the Firefox version is absolutely necessary, it would be more advisable to - rotate between older versions of Camoufox instead. \ No newline at end of file + rotate between older versions of Camoufox instead. + +# headless-non-linux: >- +# Headless mode is only recommended on Linux at this time. +# Some WAFs are able to detect headless browsers. The issue is currently being investigated. + +# headless-linux: >- +# Headless mode is only recommended on Linux with Xvfb installed. +# Please see the install guide here: +# https://github.com/daijro/camoufox/tree/main/pythonlib#virtual-display \ No newline at end of file diff --git a/pythonlib/pyproject.toml b/pythonlib/pyproject.toml index 1b38ba3..8a7ba53 100644 --- a/pythonlib/pyproject.toml +++ b/pythonlib/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "camoufox" -version = "0.2.9" +version = "0.2.10" description = "Wrapper around Playwright to help launch Camoufox" authors = ["daijro "] license = "MIT"