From 0ff90fc750e67b2ca1a433ebf91e6355db6264f8 Mon Sep 17 00:00:00 2001 From: daijro Date: Sat, 19 Oct 2024 23:10:51 -0500 Subject: [PATCH] Version range control, multi locale usage, etc - Python library now constrains the supported Camoufox version, and will force an update if you are out of date. - Added support patch for multiple accepted languages #37 - Added pysocks #43 - Added README deprecation notices - Added public launch_options command - Bumped python library to 0.3.0 - Full support for beta.12 --- Makefile | 2 +- README.md | 5 +- patches/locale-spoofing.patch | 23 ++- pythonlib/README.md | 287 +----------------------------- pythonlib/camoufox/__init__.py | 10 +- pythonlib/camoufox/__main__.py | 3 +- pythonlib/camoufox/__version__.py | 19 ++ pythonlib/camoufox/async_api.py | 101 ++--------- pythonlib/camoufox/exceptions.py | 8 + pythonlib/camoufox/locale.py | 69 +++++-- pythonlib/camoufox/pkgman.py | 172 ++++++++++++++---- pythonlib/camoufox/sync_api.py | 102 ++--------- pythonlib/camoufox/utils.py | 113 +++++++++--- pythonlib/pyproject.toml | 4 +- settings/properties.json | 1 + 15 files changed, 367 insertions(+), 552 deletions(-) create mode 100644 pythonlib/camoufox/__version__.py diff --git a/Makefile b/Makefile index 2842241..8d43476 100644 --- a/Makefile +++ b/Makefile @@ -206,7 +206,7 @@ workspace: else \ echo "Patch is not applied. Proceeding with application..."; \ fi - make checkpoint || trueZ + make checkpoint || true make patch $(_ARGS) vcredist_arch := $(shell echo $(arch) | sed 's/x86_64/x64/' | sed 's/i686/x86/') diff --git a/README.md b/README.md index ae96df6..9d013fd 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,11 @@ Camoufox aims to be a minimalistic browser for robust fingerprint injection & an --- +> [!NOTE] +> All of the latest documentation is avaliable at [camoufox.com](https://camoufox.com). + > [!WARNING] -> Camoufox is in active development! Releases are avaliable [here](https://github.com/daijro/camoufox/releases), but are not recommended for production use. +> Camoufox is in active development! Releases are avaliable, but are not recommended for production use. ## Features diff --git a/patches/locale-spoofing.patch b/patches/locale-spoofing.patch index 02523e9..e6f620c 100644 --- a/patches/locale-spoofing.patch +++ b/patches/locale-spoofing.patch @@ -1,19 +1,24 @@ diff --git a/browser/base/content/browser-init.js b/browser/base/content/browser-init.js -index 16f4dd7d42..63c9c39741 100644 +index 2456c5b4c6..826404b1b7 100644 --- a/browser/base/content/browser-init.js +++ b/browser/base/content/browser-init.js -@@ -109,6 +109,17 @@ var gBrowserInit = { +@@ -109,6 +109,22 @@ var gBrowserInit = { window.TabBarVisibility.update(); TabsInTitlebar.init(); -+ let camouLocale; -+ let language = ChromeUtils.camouGetString("locale:language"); -+ let region = ChromeUtils.camouGetString("locale:region"); -+ if (language && region) { -+ camouLocale = language + "-" + region + ", " + language; -+ } else { -+ camouLocale = ChromeUtils.camouGetString("navigator.language"); ++ // If a list of languages were passed, use them. ++ let camouLocale = ChromeUtils.camouGetString("locale:all") ++ || ChromeUtils.camouGetString("navigator.language"); ++ // If locale:all was NOT passed, but locale:language and locale:region was, ++ // fall back to it instead. ++ if (!camouLocale) { ++ let language = ChromeUtils.camouGetString("locale:language"); ++ let region = ChromeUtils.camouGetString("locale:region"); ++ if (language && region) { ++ camouLocale = language + "-" + region + ", " + language; ++ } + } ++ // Set the locale if it was found. + if (camouLocale) + Services.prefs.setCharPref("intl.accept_languages", camouLocale); + diff --git a/pythonlib/README.md b/pythonlib/README.md index 85e7b97..cbcfc4c 100644 --- a/pythonlib/README.md +++ b/pythonlib/README.md @@ -6,6 +6,9 @@ +> [!NOTE] +> All the the latest documentation is avaliable [here](https://camoufox.com/python). + --- ## What is this? @@ -68,286 +71,4 @@ Commands: ## Usage -Camoufox is fully compatible with your existing Playwright code. You only have to change your browser initialization: - -#### Sync API - -```python -from camoufox.sync_api import Camoufox - -with Camoufox(headless=False) as browser: - page = browser.new_page() - page.goto("https://example.com/") -``` - -#### Async API - -```python -from camoufox.async_api import AsyncCamoufox - -async with AsyncCamoufox(headless=False) as browser: - page = await browser.new_page() - await page.goto("https://example.com") -``` - -### Parameters List - -
- -See parameters list... - -``` -Launches a new browser instance for Camoufox. -Accepts all Playwright Firefox launch options, along with the following: - -Parameters: - os (Optional[ListOrString]): - Operating system to use for the fingerprint generation. - Can be "windows", "macos", "linux", or a list to randomly choose from. - Default: ["windows", "macos", "linux"] - block_images (Optional[bool]): - Whether to block all images. - block_webrtc (Optional[bool]): - Whether to block WebRTC entirely. - allow_webgl (Optional[bool]): - Whether to allow WebGL. To prevent leaks, only use this for special cases. - geoip (Optional[Union[str, bool]]): - Calculate longitude, latitude, timezone, country, & locale based on the IP address. - Pass the target IP address to use, or `True` to find the IP address automatically. - humanize (Optional[Union[bool, float]]): - Humanize the cursor movement. - Takes either `True`, or the MAX duration in seconds of the cursor movement. - The cursor typically takes up to 1.5 seconds to move across the window. - locale (Optional[str]): - Locale to use in Camoufox. - addons (Optional[List[str]]): - List of Firefox addons to use. - fonts (Optional[List[str]]): - Fonts to load into Camoufox (in addition to the default fonts for the target `os`). - Takes a list of font family names that are installed on the system. - exclude_addons (Optional[List[DefaultAddons]]): - Default addons to exclude. Passed as a list of camoufox.DefaultAddons enums. - screen (Optional[Screen]): - Constrains the screen dimensions of the generated fingerprint. - Takes a browserforge.fingerprints.Screen instance. - fingerprint (Optional[Fingerprint]): - *WILL BE DEPRECATED SOON* - Pass a custom BrowserForge fingerprint. Note: Not all values will be implemented. - If not provided, a random fingerprint will be generated based on the provided - `os` & `screen` constraints. - ff_version (Optional[int]): - Firefox version to use. Defaults to the current Camoufox version. - To prevent leaks, only use this for special cases. - config (Optional[Dict[str, Any]]): - Camoufox properties to use. Camoufox will warn you if you are manually setting - properties that it handles internally. - headless (Union[bool, Literal['virtual']]): - Whether to run the browser in headless mode. Defaults to False. - If you are running linux, passing 'virtual' will use Xvfb. - executable_path (Optional[str]): - Custom Camoufox browser executable path. - firefox_user_prefs (Optional[Dict[str, Any]]): - Firefox user preferences to set. - proxy (Optional[Dict[str, str]]): - Proxy to use for the browser. - Note: If geoip is True, a request will be sent through this proxy to find the target IP. - args (Optional[List[str]]): - Arguments to pass to the browser. - env (Optional[Dict[str, Union[str, float, bool]]]): - Environment variables to set. - persistent_context (Optional[bool]): - Whether to use a persistent context. - debug (Optional[bool]): - Prints the config being sent to Camoufox. - **launch_options (Dict[str, Any]): - Additional Firefox launch options. -``` - -
- -Camoufox will warn you if your passed configuration might cause leaks. - ---- - -### GeoIP & Proxy Support - -By passing `geoip=True`, or passing in a target IP address, Camoufox will automatically use the target IP's longitude, latitude, timezone, country, locale, & spoof the WebRTC IP address. - -It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region. - -[See demo](https://i.imgur.com/UhSHfaV.png). - -#### Installation - -Install Camoufox with the `geoip` extra: - -```bash -pip install -U camoufox[geoip] -``` - -#### Usage - -Pass in the proxy dictionary as you would with Playwright's `proxy` parameter: - -```python -with Camoufox( - geoip=True, - proxy={ - 'server': 'http://example.com:8080', - 'username': 'username', - 'password': 'password' - } -) as browser: - page = browser.new_page() - page.goto("https://www.browserscan.net") -``` - -
- -### Remote Server (experimental) - -**Warning! This feature is experimental. It uses a hacky workaround to gain access to undocumented Playwright methods.** - -Camoufox can be ran as a remote websocket server. It can be accessed from other devices, and languages other than Python supporting the Playwright API. - -#### Launching - -To launch a remote server, run the following CLI command: - -```bash -python -m camoufox server -``` - -Or, configure the server with a launch script: - -```python -from camoufox.server import launch_server - -launch_server( - headless=True, - geoip=True, - proxy={ - 'server': 'http://example.com:8080', - 'username': 'username', - 'password': 'password' - } -) -``` - -#### Connecting - -To connect to the remote server, use Playwright's `connect` method: - -```python -from playwright.sync_api import sync_playwright - -with sync_playwright() as p: - # Example endpoint - browser = p.firefox.connect('ws://localhost:34091/8c7c6cdea3368d937ef7db2277d6647b') - page = browser.new_page() - ... -``` - -**Note:** Because servers only use **one browser instance**, fingerprints will not rotate between sessions. If you plan on using Camoufox at scale, consider rotating the server between sessions. - -
- -### Virtual Display - -While Camoufox includes patches to prevent headless detection, running in headless mode may still be detectable in the future. It's recommended to use a virtual display buffer to run Camoufox headlessly. - -If you are running Linux, and would like to run Camoufox headlessly in a virtual display, install `xvfb`: - -#### Debian-based distros - -```bash -sudo apt-get install xvfb -``` - -#### Arch-based distros - -```bash -sudo pacman -S xorg-server-xvfb -``` - -#### Confirm `Xvfb` is installed: - -```bash -$ which Xvfb -/usr/bin/Xvfb -``` - -Now, passing `headless='virtual'` will spawn a new lightweight virtual display in the background for Camoufox to run in. - -
- -### BrowserForge Integration - -Camoufox is compatible with [BrowserForge](https://github.com/daijro/browserforge) fingerprints. - -By default, Camoufox will generate an use a random BrowserForge fingerprint based on the target `os` & `screen` constraints. - -```python -from camoufox.sync_api import Camoufox -from browserforge.fingerprints import Screen - -with Camoufox( - os=('windows', 'macos', 'linux'), - screen=Screen(max_width=1920, max_height=1080), -) as browser: - page = browser.new_page() - page.goto("https://example.com/") -``` - -If Camoufox is being ran in headful mode, the max screen size will be generated based on your monitor's dimensions unless otherwise specified. - -**Note:** To prevent UA mismatch detection, Camoufox only generates fingerprints with the same browser version as the current Camoufox version by default. If rotating the Firefox version is absolutely necessary, it would be more advisable to rotate between older versions of Camoufox instead. - -
-Injecting custom Fingerprint objects... - -> [!WARNING] -> It is recommended to pass `os` & `screen` constraints into Camoufox instead. Camoufox will handle fingerprint generation for you. This will be deprecated in the future. - -You can also inject your own Firefox BrowserForge fingerprint into Camoufox. - -```python -from camoufox.sync_api import Camoufox -from browserforge.fingerprints import FingerprintGenerator - -fg = FingerprintGenerator(browser='firefox') - -# Launch Camoufox with a random Firefox fingerprint -with Camoufox(fingerprint=fg.generate()) as browser: - page = browser.new_page() - page.goto("https://example.com/") -``` - -**Note:** As of now, some properties from BrowserForge fingerprints will not be passed to Camoufox. This is due to the outdated fingerprint dataset from Apify's fingerprint-suite (see [here](https://github.com/apify/fingerprint-suite/discussions/308)). Properties will be re-enabled as soon as an updated dataset is available. - -
- -
- -### Config - -If needed, Camoufox [config data](https://github.com/daijro/camoufox?tab=readme-ov-file#fingerprint-injection) can be overridden/passed as a dictionary to the `config` parameter. This can be used to enable features that have not yet been implemented into the Python library. - -Although, this isn't recommended, as Camoufox will populate this data for you automatically. See the parameters list above for more proper usage. - -```python -from camoufox.sync_api import Camoufox - -with Camoufox( - config={ - 'webrtc:ipv4': '123.45.67.89', - 'webrtc:ipv6': 'e791:d37a:88f6:48d1:2cad:2667:4582:1d6d', - } -) as browser: - page = browser.new_page() - page.goto("https://www.browserscan.net/webrtc") -``` - -Camoufox will warn you if you are manually setting properties that the Python library handles internally. - ---- +All of the latest documentation is avaliable at [camoufox.com/python](https://camoufox.com/python). diff --git a/pythonlib/camoufox/__init__.py b/pythonlib/camoufox/__init__.py index 331e7ff..09e1cf9 100644 --- a/pythonlib/camoufox/__init__.py +++ b/pythonlib/camoufox/__init__.py @@ -1,5 +1,13 @@ from .addons import DefaultAddons from .async_api import AsyncCamoufox, AsyncNewBrowser from .sync_api import Camoufox, NewBrowser +from .utils import launch_options -__all__ = ["Camoufox", "NewBrowser", "AsyncCamoufox", "AsyncNewBrowser", "DefaultAddons"] +__all__ = [ + "Camoufox", + "NewBrowser", + "AsyncCamoufox", + "AsyncNewBrowser", + "DefaultAddons", + "launch_options", +] diff --git a/pythonlib/camoufox/__main__.py b/pythonlib/camoufox/__main__.py index 637c3f5..fb9b771 100644 --- a/pythonlib/camoufox/__main__.py +++ b/pythonlib/camoufox/__main__.py @@ -32,6 +32,7 @@ class CamoufoxUpdate(CamoufoxFetcher): self.current_verstr = None def is_updated_needed(self) -> bool: + # Camoufox is not installed if self.current_verstr is None: return True # If the installed version is not the latest version @@ -145,7 +146,7 @@ def version() -> None: # Check for Camoufox updates if updater.is_updated_needed(): - rprint(f"(Latest: v{updater.verstr})", fg="red") + rprint(f"(Latest supported: v{updater.verstr})", fg="red") else: rprint("(Up to date!)", fg="yellow") diff --git a/pythonlib/camoufox/__version__.py b/pythonlib/camoufox/__version__.py new file mode 100644 index 0000000..71689e1 --- /dev/null +++ b/pythonlib/camoufox/__version__.py @@ -0,0 +1,19 @@ +""" +Camoufox version constants. +""" + + +class CONSTRAINTS: + """ + The minimum and maximum supported versions of the Camoufox browser. + """ + + MIN_VERSION = 'beta.12' + MAX_VERSION = '1' + + @staticmethod + def as_range() -> str: + """ + Returns the version range as a string. + """ + return f">={CONSTRAINTS.MIN_VERSION}, <{CONSTRAINTS.MAX_VERSION}" diff --git a/pythonlib/camoufox/async_api.py b/pythonlib/camoufox/async_api.py index 3c55be4..2149a36 100644 --- a/pythonlib/camoufox/async_api.py +++ b/pythonlib/camoufox/async_api.py @@ -1,16 +1,13 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, Optional, Union -from browserforge.fingerprints import Fingerprint, Screen from playwright.async_api import ( Browser, BrowserContext, Playwright, PlaywrightContextManager, ) -from typing_extensions import Literal -from .addons import DefaultAddons -from .utils import ListOrString, _clean_locals, get_launch_options +from .utils import launch_options class AsyncCamoufox(PlaywrightContextManager): @@ -38,96 +35,22 @@ class AsyncCamoufox(PlaywrightContextManager): async def AsyncNewBrowser( playwright: Playwright, *, - config: Optional[Dict[str, Any]] = None, - os: Optional[ListOrString] = None, - block_images: Optional[bool] = None, - block_webrtc: Optional[bool] = None, - allow_webgl: Optional[bool] = None, - geoip: Optional[Union[str, bool]] = None, - humanize: Optional[Union[bool, float]] = None, - locale: Optional[str] = None, - addons: Optional[List[str]] = None, - fonts: Optional[List[str]] = None, - exclude_addons: Optional[List[DefaultAddons]] = None, - screen: Optional[Screen] = None, - fingerprint: Optional[Fingerprint] = None, - ff_version: Optional[int] = None, - headless: Optional[Union[bool, Literal['virtual']]] = None, - executable_path: Optional[str] = None, - firefox_user_prefs: Optional[Dict[str, Any]] = None, - proxy: Optional[Dict[str, str]] = None, - args: Optional[List[str]] = None, - env: Optional[Dict[str, Union[str, float, bool]]] = None, - persistent_context: Optional[bool] = None, - i_know_what_im_doing: Optional[bool] = None, - debug: Optional[bool] = None, - **launch_options: Dict[str, Any] + from_options: Optional[Dict[str, Any]] = None, + persistent_context: bool = False, + **kwargs, ) -> Union[Browser, BrowserContext]: """ - Launches a new browser instance for Camoufox. - Accepts all Playwright Firefox launch options, along with the following: + Launches a new browser instance for Camoufox given a set of launch options. Parameters: - config (Optional[Dict[str, Any]]): - Camoufox properties to use. (read https://github.com/daijro/camoufox/blob/main/README.md) - os (Optional[ListOrString]): - Operating system to use for the fingerprint generation. - Can be "windows", "macos", "linux", or a list to randomly choose from. - Default: ["windows", "macos", "linux"] - block_images (Optional[bool]): - Whether to block all images. - block_webrtc (Optional[bool]): - Whether to block WebRTC entirely. - allow_webgl (Optional[bool]): - Whether to allow WebGL. To prevent leaks, only use this for special cases. - geoip (Optional[Union[str, bool]]): - Calculate longitude, latitude, timezone, country, & locale based on the IP address. - Pass the target IP address to use, or `True` to find the IP address automatically. - humanize (Optional[Union[bool, float]]): - Humanize the cursor movement. - Takes either `True`, or the MAX duration in seconds of the cursor movement. - The cursor typically takes up to 1.5 seconds to move across the window. - locale (Optional[str]): - Locale to use in Camoufox. - addons (Optional[List[str]]): - List of Firefox addons to use. - fonts (Optional[List[str]]): - Fonts to load into Camoufox (in addition to the default fonts for the target `os`). - Takes a list of font family names that are installed on the system. - exclude_addons (Optional[List[DefaultAddons]]): - Default addons to exclude. Passed as a list of camoufox.DefaultAddons enums. - screen (Optional[Screen]): - Constrains the screen dimensions of the generated fingerprint. - Takes a browserforge.fingerprints.Screen instance. - fingerprint (Optional[Fingerprint]): - Use a custom BrowserForge fingerprint. Note: Not all values will be implemented. - If not provided, a random fingerprint will be generated based on the provided - `os` & `screen` constraints. - ff_version (Optional[int]): - Firefox version to use. Defaults to the current Camoufox version. - To prevent leaks, only use this for special cases. - headless (Union[bool, Literal['virtual']]): - Whether to run the browser in headless mode. Defaults to False. - If you are running linux, passing 'virtual' will use Xvfb. - executable_path (Optional[str]): - Custom Camoufox browser executable path. - firefox_user_prefs (Optional[Dict[str, Any]]): - Firefox user preferences to set. - proxy (Optional[Dict[str, str]]): - Proxy to use for the browser. - Note: If geoip is True, a request will be sent through this proxy to find the target IP. - args (Optional[List[str]]): - Arguments to pass to the browser. - env (Optional[Dict[str, Union[str, float, bool]]]): - Environment variables to set. - persistent_context (Optional[bool]): + from_options (Dict[str, Any]): + A set of launch options generated by `launch_options()` to use + persistent_context (bool): Whether to use a persistent context. - debug (Optional[bool]): - Prints the config being sent to Camoufox. - **launch_options (Dict[str, Any]): - Additional Firefox launch options. + **kwargs: + All other keyword arugments passed to `launch_options()`. """ - opt = get_launch_options(**_clean_locals(locals())) + opt = launch_options(**kwargs) if persistent_context: return await playwright.firefox.launch_persistent_context(**opt) diff --git a/pythonlib/camoufox/exceptions.py b/pythonlib/camoufox/exceptions.py index 21db403..cc05f7d 100644 --- a/pythonlib/camoufox/exceptions.py +++ b/pythonlib/camoufox/exceptions.py @@ -1,3 +1,11 @@ +class UnsupportedVersion(Exception): + """ + Raised when the Camoufox executable is outdated. + """ + + ... + + class UnsupportedArchitecture(Exception): """ Raised when the architecture is not supported. diff --git a/pythonlib/camoufox/locale.py b/pythonlib/camoufox/locale.py index cfb10ed..511d9f9 100644 --- a/pythonlib/camoufox/locale.py +++ b/pythonlib/camoufox/locale.py @@ -1,7 +1,6 @@ import xml.etree.ElementTree as ET # nosec from dataclasses import dataclass -from random import choice as randchoice -from typing import Any, Dict, List, Optional, Tuple, cast +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union, cast import numpy as np from language_tags import tags @@ -38,17 +37,20 @@ class Locale: """ language: str - region: str + region: Optional[str] = None script: Optional[str] = None @property def as_string(self) -> str: - return f"{self.language}-{self.region}" + if self.region: + return f"{self.language}-{self.region}" + return self.language def as_config(self) -> Dict[str, str]: """ - Converts the locale to a config dictionary. + Converts the locale to a intl config dictionary. """ + assert self.region data = { 'locale:region': self.region, 'locale:language': self.language, @@ -90,24 +92,21 @@ Helpers to validate and normalize locales """ -def verify_locales(locales: List[str]) -> None: +def verify_locale(loc: str) -> None: """ - Verifies that all locales are valid. + Verifies that a locale is valid. + Takes either language-region or language. """ - for loc in locales: - if tags.check(loc): - continue - raise InvalidLocale.invalid_input(loc) + if tags.check(loc): + return + raise InvalidLocale.invalid_input(loc) def normalize_locale(locale: str) -> Locale: """ Normalizes and validates a locale code. """ - locales = locale.split(',') - verify_locales(locales) - if len(locales) > 1: - locale = randchoice(locales) # nosec + verify_locale(locale) # Parse the locale parser = tags.tag(locale) @@ -124,18 +123,26 @@ def normalize_locale(locale: str) -> Locale: ) -def handle_locale(locale: str) -> Locale: +def handle_locale(locale: str, ignore_region: bool = False) -> Locale: """ Handles a locale input, normalizing it if necessary. """ + # If the user passed in `language-region` or `language-script-region`, normalize it. if len(locale) > 3: return normalize_locale(locale) + # Case: user passed in `region` and needs a full locale try: return SELECTOR.from_region(locale) except UnknownTerritory: pass + # Case: user passed in `language`, and doesn't care about the region + if ignore_region: + verify_locale(locale) + return Locale(language=locale) + + # Case: user passed in `language` and wants a region try: language = SELECTOR.from_language(locale) except UnknownLanguage: @@ -144,9 +151,39 @@ def handle_locale(locale: str) -> Locale: LeakWarning.warn('no_region') return language + # Locale is not in a valid format. raise InvalidLocale.invalid_input(locale) +def handle_locales(locales: Union[str, List[str]], config: Dict[str, Any]) -> None: + """ + Handles a list of locales. + """ + if isinstance(locales, str): + locales = [loc.strip() for loc in locales.split(',')] + + # First, handle the first locale. This will be used for the intl api. + intl_locale = handle_locale(locales[0]) + config.update(intl_locale.as_config()) + + if len(locales) < 2: + return + + # If additional locales were passed, validate them. + # Note: in this case, we do not need the region. + config['locale:all'] = _join_unique( + handle_locale(locale, ignore_region=True).as_string for locale in locales + ) + + +def _join_unique(seq: Iterable[str]) -> str: + """ + Joins a sequence of strings without duplicates + """ + seen: Set[str] = set() + return ', '.join(x for x in seq if not (x in seen or seen.add(x))) + + """ Helpers to fetch geolocation, timezone, and locale data given an IP. """ diff --git a/pythonlib/camoufox/pkgman.py b/pythonlib/camoufox/pkgman.py index e4152c6..02c2b90 100644 --- a/pythonlib/camoufox/pkgman.py +++ b/pythonlib/camoufox/pkgman.py @@ -5,9 +5,11 @@ import shlex import shutil import sys import tempfile +from dataclasses import dataclass +from functools import total_ordering from io import BufferedWriter, BytesIO from pathlib import Path -from typing import List, Literal, Optional, Union +from typing import List, Literal, Optional, Tuple, Union from zipfile import ZipFile import click @@ -18,7 +20,8 @@ from tqdm import tqdm from typing_extensions import TypeAlias from yaml import CLoader, load -from .exceptions import UnsupportedArchitecture, UnsupportedOS +from .__version__ import CONSTRAINTS +from .exceptions import UnsupportedArchitecture, UnsupportedOS, UnsupportedVersion DownloadBuffer: TypeAlias = Union[BytesIO, tempfile._TemporaryFileWrapper, BufferedWriter] @@ -57,12 +60,81 @@ def rprint(*a, **k): click.secho(*a, **k, bold=True) +@total_ordering +@dataclass +class Version: + """ + A version string that can be compared to other version strings. + Stores versions up to 5 parts. + """ + + release: str + version: Optional[str] = None + + def __post_init__(self) -> None: + # Build an internal sortable structure + self.sorted_rel = tuple( + [ + *(int(x) if x.isdigit() else ord(x[0]) - 1024 for x in self.release.split('.')), + *(0 for _ in range(5 - self.release.count('.'))), + ] + ) + + @property + def full_string(self) -> str: + return f"{self.version}-{self.release}" + + def __eq__(self, other) -> bool: + return self.sorted_rel == other.sorted_rel + + def __lt__(self, other) -> bool: + return self.sorted_rel < other.sorted_rel + + def is_supported(self) -> bool: + return VERSION_MIN <= self < VERSION_MAX + + @staticmethod + def from_path(path: Optional[Path] = None) -> 'Version': + """ + Get the version from the given path. + """ + version_path = (path or INSTALL_DIR) / 'version.json' + if not os.path.exists(version_path): + raise FileNotFoundError( + f"Version information not found at {version_path}. " + "You are likely using an unsupported version of Camoufox." + ) + with open(version_path, 'rb') as f: + version_data = orjson.loads(f.read()) + return Version(**version_data) + + @staticmethod + def is_supported_path(path: Path) -> bool: + """ + Check if the version at the given path is supported. + """ + return Version.from_path(path) >= VERSION_MIN + + @staticmethod + def build_minmax() -> Tuple['Version', 'Version']: + return Version(release=CONSTRAINTS.MIN_VERSION), Version(release=CONSTRAINTS.MAX_VERSION) + + +# The minimum and maximum supported versions +VERSION_MIN, VERSION_MAX = Version.build_minmax() + + class CamoufoxFetcher: + """ + Handles fetching and installing the latest version of Camoufox. + """ + def __init__(self) -> None: self.arch = self.get_platform_arch() - self._version: Optional[str] = None - self._release: Optional[str] = None - self.pattern: re.Pattern = re.compile(rf'camoufox-(.+)-(.+)-{OS_NAME}\.{self.arch}\.zip') + self._version_obj: Optional[Version] = None + self.pattern: re.Pattern = re.compile( + rf'camoufox-(?P.+)-(?P.+)-{OS_NAME}\.{self.arch}\.zip' + ) self.fetch_latest() @@ -91,6 +163,30 @@ class CamoufoxFetcher: return arch + def find_release(self, releases: List[dict]) -> Optional[Tuple[Version, str]]: + """ + Finds the latest release from a GitHub releases API response that + supports the Camoufox version constraints, the OS, and architecture. + + Returns: + Optional[Tuple[Version, str]]: The version and URL of a release + """ + # Search through releases for the first supported version + for release in releases: + for asset in release['assets']: + match = self.pattern.match(asset['name']) + if not match: + continue + + # Check if the version is supported + version = Version(release=match['release'], version=match['version']) + if not version.is_supported(): + continue + + # Asset was found. Return data + return version, asset['browser_download_url'] + return None + def fetch_latest(self) -> None: """ Fetch the URL of the latest camoufox release for the current platform. @@ -100,23 +196,23 @@ class CamoufoxFetcher: requests.RequestException: If there's an error fetching release data ValueError: If no matching release is found for the current platform """ - api_url = "https://api.github.com/repos/daijro/camoufox/releases/latest" - response = requests.get(api_url, timeout=20) - response.raise_for_status() + api_url = "https://api.github.com/repos/daijro/camoufox/releases" + resp = requests.get(api_url, timeout=20) + resp.raise_for_status() - release_data = response.json() - assets = release_data['assets'] + # Find a release that fits the constraints + releases = resp.json() + release_data = self.find_release(releases) - for asset in assets: - if match := self.pattern.match(asset['name']): - # Set the version and release - self._version = match.group(1) - self._release = match.group(2) - # Return the download URL - self._url = asset['browser_download_url'] - return + if release_data is None: + raise UnsupportedVersion( + f"No matching release found for {OS_NAME} {self.arch} in the " + f"supported range: ({CONSTRAINTS.as_range()}). " + "Please update the Python library." + ) - raise ValueError(f"No matching release found for {OS_NAME}-{self.arch}") + # Set the version and URL + self._version_obj, self._url = release_data @staticmethod def download_file(file: DownloadBuffer, url: str) -> DownloadBuffer: @@ -215,9 +311,10 @@ class CamoufoxFetcher: Raises: ValueError: If the version is not available (fetch_latest not ran) """ - if self._version is None: + if self._version_obj is None or not self._version_obj.version: raise ValueError("Version is not available. Make sure to run the fetch_latest first.") - return self._version + + return self._version_obj.version @property def release(self) -> str: @@ -230,11 +327,12 @@ class CamoufoxFetcher: Raises: ValueError: If the release information is not available (fetch_latest not ran) """ - if self._release is None: + if self._version_obj is None: raise ValueError( "Release information is not available. Make sure to run the installation first." ) - return self._release + + return self._version_obj.release @property def verstr(self) -> str: @@ -244,36 +342,38 @@ class CamoufoxFetcher: Returns: str: The version of the installed camoufox """ - return f"{self.version}-{self.release}" + if self._version_obj is None: + raise ValueError("Version is not available. Make sure to run the installation first.") + return self._version_obj.full_string def installed_verstr() -> str: """ Get the full version string of the installed camoufox. """ - version_path = INSTALL_DIR / 'version.json' - if not os.path.exists(version_path): - raise FileNotFoundError(f"Version information not found at {version_path}") - - with open(version_path, 'rb') as f: - version_data = orjson.loads(f.read()) - return f"{version_data['version']}-{version_data['release']}" + return Version.from_path().full_string def camoufox_path(download_if_missing: bool = True) -> Path: """ Full path to the camoufox folder. """ + if os.path.exists(INSTALL_DIR) and Version.from_path().is_supported(): + return INSTALL_DIR + + # Ensure the directory exists if not os.path.exists(INSTALL_DIR): if not download_if_missing: raise FileNotFoundError(f"Camoufox executable not found at {INSTALL_DIR}") - installer = CamoufoxFetcher() - installer.install() - # Rerun and ensure it's installed - return camoufox_path() + # Ensure the version is supported + else: + if not download_if_missing: + raise UnsupportedVersion("Camoufox executable is outdated.") - return INSTALL_DIR + # Install and recheck + CamoufoxFetcher().install() + return camoufox_path() def get_path(file: str) -> str: diff --git a/pythonlib/camoufox/sync_api.py b/pythonlib/camoufox/sync_api.py index c8c7a7c..9c9dd38 100644 --- a/pythonlib/camoufox/sync_api.py +++ b/pythonlib/camoufox/sync_api.py @@ -1,16 +1,13 @@ -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, Optional, Union -from browserforge.fingerprints import Fingerprint, Screen from playwright.sync_api import ( Browser, BrowserContext, Playwright, PlaywrightContextManager, ) -from typing_extensions import Literal -from .addons import DefaultAddons -from .utils import ListOrString, _clean_locals, get_launch_options +from .utils import launch_options class Camoufox(PlaywrightContextManager): @@ -38,97 +35,22 @@ class Camoufox(PlaywrightContextManager): def NewBrowser( playwright: Playwright, *, - config: Optional[Dict[str, Any]] = None, - os: Optional[ListOrString] = None, - block_images: Optional[bool] = None, - block_webrtc: Optional[bool] = None, - allow_webgl: Optional[bool] = None, - geoip: Optional[Union[str, bool]] = None, - humanize: Optional[Union[bool, float]] = None, - locale: Optional[str] = None, - addons: Optional[List[str]] = None, - fonts: Optional[List[str]] = None, - exclude_addons: Optional[List[DefaultAddons]] = None, - screen: Optional[Screen] = None, - fingerprint: Optional[Fingerprint] = None, - ff_version: Optional[int] = None, - headless: Optional[Union[bool, Literal['virtual']]] = None, - executable_path: Optional[str] = None, - firefox_user_prefs: Optional[Dict[str, Any]] = None, - proxy: Optional[Dict[str, str]] = None, - args: Optional[List[str]] = None, - env: Optional[Dict[str, Union[str, float, bool]]] = None, - persistent_context: Optional[bool] = None, - i_know_what_im_doing: Optional[bool] = None, - debug: Optional[bool] = None, - **launch_options: Dict[str, Any] + from_options: Optional[Dict[str, Any]] = None, + persistent_context: bool = False, + **kwargs, ) -> Union[Browser, BrowserContext]: """ - Launches a new browser instance for Camoufox. - Accepts all Playwright Firefox launch options, along with the following: + Launches a new browser instance for Camoufox given a set of launch options. Parameters: - config (Optional[Dict[str, Any]]): - Camoufox properties to use. (read https://github.com/daijro/camoufox/blob/main/README.md) - os (Optional[ListOrString]): - Operating system to use for the fingerprint generation. - Can be "windows", "macos", "linux", or a list to randomly choose from. - Default: ["windows", "macos", "linux"] - block_images (Optional[bool]): - Whether to block all images. - block_webrtc (Optional[bool]): - Whether to block WebRTC entirely. - allow_webgl (Optional[bool]): - Whether to allow WebGL. To prevent leaks, only use this for special cases. - geoip (Optional[Union[str, bool]]): - Calculate longitude, latitude, timezone, country, & locale based on the IP address. - Pass the target IP address to use, or `True` to find the IP address automatically. - humanize (Optional[Union[bool, float]]): - Humanize the cursor movement. - Takes either `True`, or the MAX duration in seconds of the cursor movement. - The cursor typically takes up to 1.5 seconds to move across the window. - locale (Optional[str]): - Locale to use in Camoufox. - addons (Optional[List[str]]): - List of Firefox addons to use. - fonts (Optional[List[str]]): - Fonts to load into Camoufox (in addition to the default fonts for the target `os`). - Takes a list of font family names that are installed on the system. - exclude_addons (Optional[List[DefaultAddons]]): - Default addons to exclude. Passed as a list of camoufox.DefaultAddons enums. - screen (Optional[Screen]): - Constrains the screen dimensions of the generated fingerprint. - Takes a browserforge.fingerprints.Screen instance. - fingerprint (Optional[Fingerprint]): - Use a custom BrowserForge fingerprint. Note: Not all values will be implemented. - If not provided, a random fingerprint will be generated based on the provided - `os` & `screen` constraints. - ff_version (Optional[int]): - Firefox version to use. Defaults to the current Camoufox version. - To prevent leaks, only use this for special cases. - headless (Union[bool, Literal['virtual']]): - Whether to run the browser in headless mode. Defaults to False. - If you are running linux, passing 'virtual' will use Xvfb. - executable_path (Optional[str]): - Custom Camoufox browser executable path. - firefox_user_prefs (Optional[Dict[str, Any]]): - Firefox user preferences to set. - proxy (Optional[Dict[str, str]]): - Proxy to use for the browser. - Note: If geoip is True, a request will be sent through this proxy to find the target IP. - args (Optional[List[str]]): - Arguments to pass to the browser. - env (Optional[Dict[str, Union[str, float, bool]]]): - Environment variables to set. - persistent_context (Optional[bool]): + from_options (Dict[str, Any]): + A set of launch options generated by `launch_options()` to use + persistent_context (bool): Whether to use a persistent context. - debug (Optional[bool]): - Prints the config being sent to Camoufox. - **launch_options (Dict[str, Any]): - Additional Firefox launch options. + **kwargs: + All other keyword arugments passed to `launch_options()`. """ - opt = get_launch_options(**_clean_locals(locals())) - + opt = from_options or launch_options(**kwargs) if persistent_context: return playwright.firefox.launch_persistent_context(**opt) diff --git a/pythonlib/camoufox/utils.py b/pythonlib/camoufox/utils.py index e671c42..950c204 100644 --- a/pythonlib/camoufox/utils.py +++ b/pythonlib/camoufox/utils.py @@ -1,6 +1,8 @@ import os import sys from os import environ +from os.path import abspath +from pathlib import Path from pprint import pprint from random import randrange from typing import Any, Dict, List, Literal, Optional, Tuple, Union, cast @@ -26,7 +28,7 @@ from .exceptions import ( ) from .fingerprints import from_browserforge, generate_fingerprint from .ip import Proxy, public_ip, valid_ipv4, valid_ipv6 -from .locale import geoip_allowed, get_geolocation, handle_locale +from .locale import geoip_allowed, get_geolocation, handle_locales from .pkgman import OS_NAME, get_path, installed_verstr from .warnings import LeakWarning from .xpi_dl import add_default_addons @@ -76,22 +78,25 @@ def get_env_vars( return env_vars -def _load_properties() -> Dict[str, str]: +def _load_properties(path: Optional[Path] = None) -> Dict[str, str]: """ Loads the properties.json file. """ - prop_file = get_path("properties.json") + if path: + prop_file = str(path.parent / "properties.json") + else: + prop_file = get_path("properties.json") with open(prop_file, "rb") as f: prop_dict = orjson.loads(f.read()) return {prop['property']: prop['type'] for prop in prop_dict} -def validate_config(config_map: Dict[str, str]) -> None: +def validate_config(config_map: Dict[str, str], path: Optional[Path] = None) -> None: """ Validates the config map. """ - property_types = _load_properties() + property_types = _load_properties(path=path) for key, value in config_map.items(): expected_type = property_types.get(key) @@ -289,34 +294,93 @@ def warn_manual_config(config: Dict[str, Any]) -> None: LeakWarning.warn('viewport', False) -def get_launch_options( +def launch_options( *, config: Optional[Dict[str, Any]] = None, - addons: Optional[List[str]] = None, - fingerprint: Optional[Fingerprint] = None, - humanize: Optional[Union[bool, float]] = None, - i_know_what_im_doing: Optional[bool] = None, - exclude_addons: Optional[List[DefaultAddons]] = None, - screen: Optional[Screen] = None, - geoip: Optional[Union[str, bool]] = None, - locale: Optional[str] = None, os: Optional[ListOrString] = None, - fonts: Optional[List[str]] = None, - args: Optional[List[str]] = None, - executable_path: Optional[str] = None, - env: Optional[Dict[str, Union[str, float, bool]]] = None, block_images: Optional[bool] = None, block_webrtc: Optional[bool] = None, allow_webgl: Optional[bool] = None, - proxy: Optional[Dict[str, str]] = None, + geoip: Optional[Union[str, bool]] = None, + humanize: Optional[Union[bool, float]] = None, + locale: Optional[Union[str, List[str]]] = None, + addons: Optional[List[str]] = None, + fonts: Optional[List[str]] = None, + exclude_addons: Optional[List[DefaultAddons]] = None, + screen: Optional[Screen] = None, + fingerprint: Optional[Fingerprint] = None, ff_version: Optional[int] = None, headless: Optional[Union[bool, Literal['virtual']]] = None, + executable_path: Optional[str] = None, firefox_user_prefs: Optional[Dict[str, Any]] = None, - launch_options: Optional[Dict[str, Any]] = None, + proxy: Optional[Dict[str, str]] = None, + args: Optional[List[str]] = None, + env: Optional[Dict[str, Union[str, float, bool]]] = None, + i_know_what_im_doing: Optional[bool] = None, debug: Optional[bool] = None, + **launch_options: Dict[str, Any], ) -> Dict[str, Any]: """ - Builds the launch options for the Camoufox browser. + Launches a new browser instance for Camoufox. + Accepts all Playwright Firefox launch options, along with the following: + + Parameters: + config (Optional[Dict[str, Any]]): + Camoufox properties to use. (read https://github.com/daijro/camoufox/blob/main/README.md) + os (Optional[ListOrString]): + Operating system to use for the fingerprint generation. + Can be "windows", "macos", "linux", or a list to randomly choose from. + Default: ["windows", "macos", "linux"] + block_images (Optional[bool]): + Whether to block all images. + block_webrtc (Optional[bool]): + Whether to block WebRTC entirely. + allow_webgl (Optional[bool]): + Whether to allow WebGL. To prevent leaks, only use this for special cases. + geoip (Optional[Union[str, bool]]): + Calculate longitude, latitude, timezone, country, & locale based on the IP address. + Pass the target IP address to use, or `True` to find the IP address automatically. + humanize (Optional[Union[bool, float]]): + Humanize the cursor movement. + Takes either `True`, or the MAX duration in seconds of the cursor movement. + The cursor typically takes up to 1.5 seconds to move across the window. + locale (Optional[Union[str, List[str]]]): + Locale(s) to use in Camoufox. The first listed locale will be used for the Intl API. + addons (Optional[List[str]]): + List of Firefox addons to use. + fonts (Optional[List[str]]): + Fonts to load into Camoufox (in addition to the default fonts for the target `os`). + Takes a list of font family names that are installed on the system. + exclude_addons (Optional[List[DefaultAddons]]): + Default addons to exclude. Passed as a list of camoufox.DefaultAddons enums. + screen (Optional[Screen]): + Constrains the screen dimensions of the generated fingerprint. + Takes a browserforge.fingerprints.Screen instance. + fingerprint (Optional[Fingerprint]): + Use a custom BrowserForge fingerprint. Note: Not all values will be implemented. + If not provided, a random fingerprint will be generated based on the provided + `os` & `screen` constraints. + ff_version (Optional[int]): + Firefox version to use. Defaults to the current Camoufox version. + To prevent leaks, only use this for special cases. + headless (Union[bool, Literal['virtual']]): + Whether to run the browser in headless mode. Defaults to False. + If you are running linux, passing 'virtual' will use Xvfb. + executable_path (Optional[str]): + Custom Camoufox browser executable path. + firefox_user_prefs (Optional[Dict[str, Any]]): + Firefox user preferences to set. + proxy (Optional[Dict[str, str]]): + Proxy to use for the browser. + Note: If geoip is True, a request will be sent through this proxy to find the target IP. + args (Optional[List[str]]): + Arguments to pass to the browser. + env (Optional[Dict[str, Union[str, float, bool]]]): + Environment variables to set. + debug (Optional[bool]): + Prints the config being sent to Camoufox. + **launch_options (Dict[str, Any]): + Additional Firefox launch options. """ # Build the config if config is None: @@ -335,6 +399,8 @@ def get_launch_options( i_know_what_im_doing = False if env is None: env = cast(Dict[str, Union[str, float, bool]], environ) + if isinstance(executable_path, str): + executable_path = Path(abspath(executable_path)) # Handle headless mode cases if headless == 'virtual': @@ -423,8 +489,7 @@ def get_launch_options( # Set locale if locale: - parsed_locale = handle_locale(locale) - config.update(parsed_locale.as_config()) + handle_locales(locale, config) # Pass the humanize option if humanize: @@ -433,7 +498,7 @@ def get_launch_options( set_into(config, 'humanize:maxTime', humanize) # Validate the config - validate_config(config) + validate_config(config, path=executable_path) # Print the config if debug is enabled if debug: diff --git a/pythonlib/pyproject.toml b/pythonlib/pyproject.toml index 6c085ea..1709734 100644 --- a/pythonlib/pyproject.toml +++ b/pythonlib/pyproject.toml @@ -4,11 +4,12 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "camoufox" -version = "0.2.15" +version = "0.3.0" description = "Wrapper around Playwright to help launch Camoufox" authors = ["daijro "] license = "MIT" repository = "https://github.com/daijro/camoufox" +homepage = "https://camoufox.com/python" readme = "README.md" keywords = [ "client", @@ -41,6 +42,7 @@ typing_extensions = "*" screeninfo = "*" lxml = "*" language-tags = "*" +pysocks = "*" geoip2 = {version = "*", optional = true} [tool.poetry.extras] diff --git a/settings/properties.json b/settings/properties.json index e36e76e..cd54143 100644 --- a/settings/properties.json +++ b/settings/properties.json @@ -60,6 +60,7 @@ { "property": "locale:language", "type": "str" }, { "property": "locale:region", "type": "str" }, { "property": "locale:script", "type": "str" }, + { "property": "locale:all", "type": "str" }, { "property": "humanize", "type": "bool" }, { "property": "humanize:maxTime", "type": "double" }, { "property": "humanize:minTime", "type": "double" },