diff --git a/pythonlib/README.md b/pythonlib/README.md index e62e9f3..b324c01 100644 --- a/pythonlib/README.md +++ b/pythonlib/README.md @@ -92,7 +92,7 @@ Parameters: Camoufox properties to use. os (Optional[ListOrString]): Operating system to use for the fingerprint generation. - Can be "windows", "macos", or "linux", or a list of these to choose from randomly. + Can be "windows", "macos", "linux", "android", "ios", or a list to randomly choose from. Default: ["windows", "macos", "linux"] block_images (Optional[bool]): Whether to block all images. @@ -103,6 +103,10 @@ Parameters: geoip (Optional[Union[str, bool]]): Calculate longitude, latitude, timezone, country, & locale based on the IP address. Pass the target IP address to use, or `True` to find the IP address automatically. + humanize (Optional[Union[bool, float]]): + Humanize the cursor movement. + Takes either `True`, or the MAX duration in seconds of the cursor movement. + The cursor typically takes up to 1.5 seconds to move across the window. locale (Optional[str]): Locale to use in Camoufox. addons (Optional[List[str]]): @@ -112,12 +116,16 @@ Parameters: Takes a list of font family names that are installed on the system. exclude_addons (Optional[List[DefaultAddons]]): Default addons to exclude. Passed as a list of camoufox.DefaultAddons enums. - fingerprint (Optional[Fingerprint]): - Use a custom BrowserForge fingerprint. Note: Not all values will be implemented. - If not provided, a random fingerprint will be generated based on the provided os & user_agent. screen (Optional[Screen]): Constrains the screen dimensions of the generated fingerprint. Takes a browserforge.fingerprints.Screen instance. + fingerprint (Optional[Fingerprint]): + Use a custom BrowserForge fingerprint. Note: Not all values will be implemented. + If not provided, a random fingerprint will be generated based on the provided + `os` & `screen` constraints. + ff_version (Optional[int]): + Firefox version to use. Defaults to the current Camoufox version. + To prevent leaks, only use this for special cases. headless (Optional[bool]): Whether to run the browser in headless mode. Defaults to True. executable_path (Optional[str]): @@ -127,13 +135,14 @@ Parameters: proxy (Optional[Dict[str, str]]): Proxy to use for the browser. Note: If geoip is True, a request will be sent through this proxy to find the target IP. - ff_version (Optional[int]): - Firefox version to use. Defaults to the current Camoufox version. - To prevent leaks, only use this for special cases. args (Optional[List[str]]): Arguments to pass to the browser. env (Optional[Dict[str, Union[str, float, bool]]]): Environment variables to set. + persistent_context (Optional[bool]): + Whether to use a persistent context. + debug (Optional[bool]): + Prints the config being sent to Camoufox. **launch_options (Dict[str, Any]): Additional Firefox launch options. ``` @@ -245,7 +254,35 @@ with sync_playwright() as p: Camoufox is compatible with [BrowserForge](https://github.com/daijro/browserforge) fingerprints. -By default, Camoufox will use a random fingerprint. You can also inject your own Firefox Browserforge fingerprint into Camoufox with the following example: +By default, Camoufox will generate an use a random BrowserForge fingerprint based on the target `os` & `screen` constraints. + +```python +from camoufox.sync_api import Camoufox +from browserforge.fingerprints import Screen + +with Camoufox( + os=('windows', 'macos', 'linux'), + screen=Screen(max_width=1920, max_height=1080), +) as browser: + page = browser.new_page() + page.goto("https://example.com/") +``` + +**Notes:** + +- If Camoufox is being ran in headful mode, the max screen size will be generated based on your monitor's dimensions (+15%). + +- To prevent UA-spoofing leaks, Camoufox only generates fingerprints with the same browser version as the current Camoufox version by default. + + - If rotating the Firefox version is absolutely necessary, it would be more advisable to rotate between older versions of Camoufox instead. + +
+Injecting custom Fingerprint objects... + +> [!WARNING] +> It is recommended to pass `os` & `screen` constraints into Camoufox instead. Camoufox will handle fingerprint generation for you. This will be deprecated in the future. + +You can also inject your own Firefox BrowserForge fingerprint into Camoufox. ```python from camoufox.sync_api import Camoufox @@ -259,8 +296,8 @@ with Camoufox(fingerprint=fg.generate()) as browser: page.goto("https://example.com/") ``` -
- **Note:** As of now, some properties from BrowserForge fingerprints will not be passed to Camoufox. This is due to the outdated fingerprint dataset from Apify's fingerprint-suite (see [here](https://github.com/apify/fingerprint-suite/discussions/308)). Properties will be re-enabled as soon as an updated dataset is available. +
+ --- diff --git a/pythonlib/camoufox/async_api.py b/pythonlib/camoufox/async_api.py index 4bc495d..843aba0 100644 --- a/pythonlib/camoufox/async_api.py +++ b/pythonlib/camoufox/async_api.py @@ -1,10 +1,15 @@ from typing import Any, Dict, List, Optional, Union from browserforge.fingerprints import Fingerprint, Screen -from playwright.async_api import Browser, Playwright, PlaywrightContextManager +from playwright.async_api import ( + Browser, + BrowserContext, + Playwright, + PlaywrightContextManager, +) from .addons import DefaultAddons -from .utils import ListOrString, get_launch_options +from .utils import ListOrString, _clean_locals, get_launch_options class AsyncCamoufox(PlaywrightContextManager): @@ -16,9 +21,9 @@ class AsyncCamoufox(PlaywrightContextManager): def __init__(self, **launch_options): super().__init__() self.launch_options = launch_options - self.browser: Optional[Browser] = None + self.browser: Optional[Union[Browser, BrowserContext]] = None - async def __aenter__(self) -> Browser: + async def __aenter__(self) -> Union[Browser, BrowserContext]: _playwright = await super().__aenter__() self.browser = await AsyncNewBrowser(_playwright, **self.launch_options) return self.browser @@ -38,21 +43,25 @@ async def AsyncNewBrowser( block_webrtc: Optional[bool] = None, allow_webgl: Optional[bool] = None, geoip: Optional[Union[str, bool]] = None, + humanize: Optional[Union[bool, float]] = None, locale: Optional[str] = None, addons: Optional[List[str]] = None, fonts: Optional[List[str]] = None, exclude_addons: Optional[List[DefaultAddons]] = None, - fingerprint: Optional[Fingerprint] = None, screen: Optional[Screen] = None, + fingerprint: Optional[Fingerprint] = None, + ff_version: Optional[int] = None, headless: Optional[bool] = None, executable_path: Optional[str] = None, firefox_user_prefs: Optional[Dict[str, Any]] = None, proxy: Optional[Dict[str, str]] = None, - ff_version: Optional[int] = None, args: Optional[List[str]] = None, env: Optional[Dict[str, Union[str, float, bool]]] = None, + persistent_context: Optional[bool] = None, + i_know_what_im_doing: Optional[bool] = None, + debug: Optional[bool] = None, **launch_options: Dict[str, Any] -) -> Browser: +) -> Union[Browser, BrowserContext]: """ Launches a new browser instance for Camoufox. Accepts all Playwright Firefox launch options, along with the following: @@ -62,7 +71,7 @@ async def AsyncNewBrowser( Camoufox properties to use. (read https://github.com/daijro/camoufox/blob/main/README.md) os (Optional[ListOrString]): Operating system to use for the fingerprint generation. - Can be "windows", "macos", or "linux", or a list of these to choose from randomly. + Can be "windows", "macos", "linux", "android", "ios", or a list to randomly choose from. Default: ["windows", "macos", "linux"] block_images (Optional[bool]): Whether to block all images. @@ -73,6 +82,10 @@ async def AsyncNewBrowser( geoip (Optional[Union[str, bool]]): Calculate longitude, latitude, timezone, country, & locale based on the IP address. Pass the target IP address to use, or `True` to find the IP address automatically. + humanize (Optional[Union[bool, float]]): + Humanize the cursor movement. + Takes either `True`, or the MAX duration in seconds of the cursor movement. + The cursor typically takes up to 1.5 seconds to move across the window. locale (Optional[str]): Locale to use in Camoufox. addons (Optional[List[str]]): @@ -82,12 +95,16 @@ async def AsyncNewBrowser( Takes a list of font family names that are installed on the system. exclude_addons (Optional[List[DefaultAddons]]): Default addons to exclude. Passed as a list of camoufox.DefaultAddons enums. - fingerprint (Optional[Fingerprint]): - Use a custom BrowserForge fingerprint. Note: Not all values will be implemented. - If not provided, a random fingerprint will be generated based on the provided os & user_agent. screen (Optional[Screen]): Constrains the screen dimensions of the generated fingerprint. Takes a browserforge.fingerprints.Screen instance. + fingerprint (Optional[Fingerprint]): + Use a custom BrowserForge fingerprint. Note: Not all values will be implemented. + If not provided, a random fingerprint will be generated based on the provided + `os` & `screen` constraints. + ff_version (Optional[int]): + Firefox version to use. Defaults to the current Camoufox version. + To prevent leaks, only use this for special cases. headless (Optional[bool]): Whether to run the browser in headless mode. Defaults to True. executable_path (Optional[str]): @@ -97,18 +114,20 @@ async def AsyncNewBrowser( proxy (Optional[Dict[str, str]]): Proxy to use for the browser. Note: If geoip is True, a request will be sent through this proxy to find the target IP. - ff_version (Optional[int]): - Firefox version to use. Defaults to the current Camoufox version. - To prevent leaks, only use this for special cases. args (Optional[List[str]]): Arguments to pass to the browser. env (Optional[Dict[str, Union[str, float, bool]]]): Environment variables to set. + persistent_context (Optional[bool]): + Whether to use a persistent context. + debug (Optional[bool]): + Prints the config being sent to Camoufox. **launch_options (Dict[str, Any]): Additional Firefox launch options. """ - data = locals() - data.pop('playwright') + opt = get_launch_options(**_clean_locals(locals())) + + if persistent_context: + return await playwright.firefox.launch_persistent_context(**opt) - opt = get_launch_options(**data) return await playwright.firefox.launch(**opt) diff --git a/pythonlib/camoufox/exceptions.py b/pythonlib/camoufox/exceptions.py index e646475..fcad52a 100644 --- a/pythonlib/camoufox/exceptions.py +++ b/pythonlib/camoufox/exceptions.py @@ -100,3 +100,11 @@ class NotInstalledGeoIPExtra(ImportError): """ ... + + +class NonFirefoxFingerprint(Exception): + """ + Raised when a passed Browserforge fingerprint is invalid. + """ + + ... diff --git a/pythonlib/camoufox/sync_api.py b/pythonlib/camoufox/sync_api.py index 7f45651..e3a5098 100644 --- a/pythonlib/camoufox/sync_api.py +++ b/pythonlib/camoufox/sync_api.py @@ -1,10 +1,15 @@ from typing import Any, Dict, List, Optional, Union from browserforge.fingerprints import Fingerprint, Screen -from playwright.sync_api import Browser, Playwright, PlaywrightContextManager +from playwright.sync_api import ( + Browser, + BrowserContext, + Playwright, + PlaywrightContextManager, +) from .addons import DefaultAddons -from .utils import ListOrString, get_launch_options +from .utils import ListOrString, _clean_locals, get_launch_options class Camoufox(PlaywrightContextManager): @@ -16,9 +21,9 @@ class Camoufox(PlaywrightContextManager): def __init__(self, **launch_options): super().__init__() self.launch_options = launch_options - self.browser: Optional[Browser] = None + self.browser: Optional[Union[Browser, BrowserContext]] = None - def __enter__(self) -> Browser: + def __enter__(self) -> Union[Browser, BrowserContext]: super().__enter__() self.browser = NewBrowser(self._playwright, **self.launch_options) return self.browser @@ -38,19 +43,23 @@ def NewBrowser( block_webrtc: Optional[bool] = None, allow_webgl: Optional[bool] = None, geoip: Optional[Union[str, bool]] = None, + humanize: Optional[Union[bool, float]] = None, locale: Optional[str] = None, addons: Optional[List[str]] = None, fonts: Optional[List[str]] = None, exclude_addons: Optional[List[DefaultAddons]] = None, - fingerprint: Optional[Fingerprint] = None, screen: Optional[Screen] = None, + fingerprint: Optional[Fingerprint] = None, + ff_version: Optional[int] = None, headless: Optional[bool] = None, executable_path: Optional[str] = None, firefox_user_prefs: Optional[Dict[str, Any]] = None, proxy: Optional[Dict[str, str]] = None, - ff_version: Optional[int] = None, args: Optional[List[str]] = None, env: Optional[Dict[str, Union[str, float, bool]]] = None, + persistent_context: Optional[bool] = None, + i_know_what_im_doing: Optional[bool] = None, + debug: Optional[bool] = None, **launch_options: Dict[str, Any] ) -> Browser: """ @@ -62,7 +71,7 @@ def NewBrowser( Camoufox properties to use. (read https://github.com/daijro/camoufox/blob/main/README.md) os (Optional[ListOrString]): Operating system to use for the fingerprint generation. - Can be "windows", "macos", or "linux", or a list of these to choose from randomly. + Can be "windows", "macos", "linux", "android", "ios", or a list to randomly choose from. Default: ["windows", "macos", "linux"] block_images (Optional[bool]): Whether to block all images. @@ -73,6 +82,10 @@ def NewBrowser( geoip (Optional[Union[str, bool]]): Calculate longitude, latitude, timezone, country, & locale based on the IP address. Pass the target IP address to use, or `True` to find the IP address automatically. + humanize (Optional[Union[bool, float]]): + Humanize the cursor movement. + Takes either `True`, or the MAX duration in seconds of the cursor movement. + The cursor typically takes up to 1.5 seconds to move across the window. locale (Optional[str]): Locale to use in Camoufox. addons (Optional[List[str]]): @@ -82,12 +95,16 @@ def NewBrowser( Takes a list of font family names that are installed on the system. exclude_addons (Optional[List[DefaultAddons]]): Default addons to exclude. Passed as a list of camoufox.DefaultAddons enums. - fingerprint (Optional[Fingerprint]): - Use a custom BrowserForge fingerprint. Note: Not all values will be implemented. - If not provided, a random fingerprint will be generated based on the provided os & user_agent. screen (Optional[Screen]): Constrains the screen dimensions of the generated fingerprint. Takes a browserforge.fingerprints.Screen instance. + fingerprint (Optional[Fingerprint]): + Use a custom BrowserForge fingerprint. Note: Not all values will be implemented. + If not provided, a random fingerprint will be generated based on the provided + `os` & `screen` constraints. + ff_version (Optional[int]): + Firefox version to use. Defaults to the current Camoufox version. + To prevent leaks, only use this for special cases. headless (Optional[bool]): Whether to run the browser in headless mode. Defaults to True. executable_path (Optional[str]): @@ -97,18 +114,20 @@ def NewBrowser( proxy (Optional[Dict[str, str]]): Proxy to use for the browser. Note: If geoip is True, a request will be sent through this proxy to find the target IP. - ff_version (Optional[int]): - Firefox version to use. Defaults to the current Camoufox version. - To prevent leaks, only use this for special cases. args (Optional[List[str]]): Arguments to pass to the browser. env (Optional[Dict[str, Union[str, float, bool]]]): Environment variables to set. + persistent_context (Optional[bool]): + Whether to use a persistent context. + debug (Optional[bool]): + Prints the config being sent to Camoufox. **launch_options (Dict[str, Any]): Additional Firefox launch options. """ - data = locals() - data.pop('playwright') + opt = get_launch_options(**_clean_locals(locals())) + + if persistent_context: + return playwright.firefox.launch_persistent_context(**opt) - opt = get_launch_options(**data) return playwright.firefox.launch(**opt) diff --git a/pythonlib/camoufox/utils.py b/pythonlib/camoufox/utils.py index b400a7d..e55bd38 100644 --- a/pythonlib/camoufox/utils.py +++ b/pythonlib/camoufox/utils.py @@ -1,6 +1,8 @@ import os import sys +import warnings from os import environ +from pprint import pprint from random import randrange from typing import Any, Dict, List, Literal, Optional, Tuple, Union, cast @@ -17,7 +19,7 @@ from .addons import ( get_debug_port, threaded_try_load_addons, ) -from .exceptions import InvalidPropertyType, UnknownProperty +from .exceptions import InvalidPropertyType, NonFirefoxFingerprint, UnknownProperty from .fingerprints import from_browserforge, generate_fingerprint from .ip import Proxy, public_ip, valid_ipv4, valid_ipv6 from .locale import geoip_allowed, get_geolocation, normalize_locale @@ -133,7 +135,7 @@ def determine_ua_os(user_agent: str) -> Literal['mac', 'win', 'lin']: parsed_ua = user_agent_parser.ParseOS(user_agent).get('family') if not parsed_ua: raise ValueError("Could not determine OS from user agent") - if parsed_ua.startswith("Mac"): + if parsed_ua.startswith("Mac") or parsed_ua.startswith("iOS"): return "mac" if parsed_ua.startswith("Windows"): return "win" @@ -155,8 +157,8 @@ def get_screen_cons(headless: Optional[bool] = None) -> Optional[Screen]: # Use the dimensions from the monitor with greatest screen real estate monitor = max(monitors, key=lambda m: m.width * m.height) - # Add 25% buffer - return Screen(max_width=int(monitor.width * 1.25), max_height=int(monitor.height * 1.25)) + # Add 15% buffer + return Screen(max_width=int(monitor.width * 1.15), max_height=int(monitor.height * 1.15)) def update_fonts(config: Dict[str, Any], target_os: str) -> None: @@ -173,6 +175,40 @@ def update_fonts(config: Dict[str, Any], target_os: str) -> None: config['fonts'] = fonts +def check_custom_fingerprint(fingerprint: Fingerprint) -> None: + """ + Asserts that the passed BrowserForge fingerprint is a valid Firefox fingerprint. + and warns the user that passing their own fingerprint is not recommended. + """ + if any(browser in fingerprint.navigator.userAgent for browser in ('Firefox', 'FxiOS')): + return + # Tell the user what browser they're using + parsed_ua = user_agent_parser.ParseUserAgent(fingerprint.navigator.userAgent).get( + 'family', 'Non-Firefox' + ) + if parsed_ua: + raise NonFirefoxFingerprint( + f'"{parsed_ua}" fingerprints are not supported in Camoufox. ' + 'Using fingerprints from a browser other than Firefox WILL lead to detection. ' + 'If this is intentional, pass `i_know_what_im_doing=True`.' + ) + + warnings.warn( + 'Passing your own fingerprint is not recommended. ' + 'BrowserForge fingerprints are automatically generated within Camoufox ' + 'based on the provided `os` and `screen` constraints. ' + ) + + +def _clean_locals(data: Dict[str, Any]) -> Dict[str, Any]: + """ + Gets the launch options from the locals of the function. + """ + del data['playwright'] + del data['persistent_context'] + return data + + def merge_into(target: Dict[str, Any], source: Dict[str, Any]) -> None: """ Merges new keys/values from the source dictionary into the target dictionary. @@ -197,6 +233,8 @@ def get_launch_options( config: Optional[Dict[str, Any]] = None, addons: Optional[List[str]] = None, fingerprint: Optional[Fingerprint] = None, + humanize: Optional[Union[bool, float]] = None, + i_know_what_im_doing: Optional[bool] = None, exclude_addons: Optional[List[DefaultAddons]] = None, screen: Optional[Screen] = None, geoip: Optional[Union[str, bool]] = None, @@ -214,6 +252,7 @@ def get_launch_options( headless: Optional[bool] = None, firefox_user_prefs: Optional[Dict[str, Any]] = None, launch_options: Optional[Dict[str, Any]] = None, + debug: Optional[bool] = None, ) -> Dict[str, Any]: """ Builds the launch options for the Camoufox browser. @@ -242,12 +281,18 @@ def get_launch_options( else: ff_version_str = installed_verstr().split('.', 1)[0] - # Inject a unique Firefox fingerprint + # Generate a fingerprint if fingerprint is None: fingerprint = generate_fingerprint( screen=screen or get_screen_cons(headless), os=os, ) + else: + # Or use the one passed by the user + if not i_know_what_im_doing: + check_custom_fingerprint(fingerprint) + + # Inject the fingerprint into the config merge_into( config, from_browserforge(fingerprint, ff_version_str), @@ -290,9 +335,20 @@ def get_launch_options( parsed_locale = normalize_locale(locale) config.update(parsed_locale.as_config()) + # Pass the humanize option + if humanize: + set_into(config, 'humanize', True) + if isinstance(humanize, (int, float)): + set_into(config, 'humanize:maxTime', humanize) + # Validate the config validate_config(config) + # Print the config if debug is enabled + if debug: + print('[DEBUG] Config:') + pprint(config) + # Set Firefox user preferences if block_images: firefox_user_prefs['permissions.default.image'] = 2 diff --git a/pythonlib/pyproject.toml b/pythonlib/pyproject.toml index 668c0fc..2a4a1dd 100644 --- a/pythonlib/pyproject.toml +++ b/pythonlib/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "camoufox" -version = "0.2.4" +version = "0.2.5" description = "Wrapper around Playwright to help launch Camoufox" authors = ["daijro "] license = "MIT"