From ad556225504e6dfb40fde31af76d1a24941c2f73 Mon Sep 17 00:00:00 2001 From: daijro Date: Mon, 30 Sep 2024 04:06:16 -0500 Subject: [PATCH] pythonlib: Implement viewport hijacking 0.2.2 - Allow Browerforge fingerprints to override screen & viewport data. - Set Camoufox's window dimensions to a fixed width/height generated from Browserforge. - If the browser is headful, do not exceed 125% of the screen size. Other changes: - Allow public IP finder to work without verification. - Add script to publish to pypi. - More descriptive progress bars when downloading default addons. - Bump to 0.2.2 --- pythonlib/README.md | 4 ++- pythonlib/camoufox/addons.py | 2 +- pythonlib/camoufox/async_api.py | 5 +++- pythonlib/camoufox/browserforge.yml | 22 +++++++------- pythonlib/camoufox/fingerprints.py | 19 +++++++----- pythonlib/camoufox/ip.py | 23 +++++++++++---- pythonlib/camoufox/sync_api.py | 5 +++- pythonlib/camoufox/utils.py | 46 ++++++++++++++++++++--------- pythonlib/camoufox/xpi_dl.py | 8 ++--- pythonlib/publish.sh | 11 +++++++ pythonlib/pyproject.toml | 3 +- 11 files changed, 100 insertions(+), 48 deletions(-) create mode 100644 pythonlib/publish.sh diff --git a/pythonlib/README.md b/pythonlib/README.md index 2d98ef2..d2fd5e9 100644 --- a/pythonlib/README.md +++ b/pythonlib/README.md @@ -114,8 +114,10 @@ Parameters: Use a custom BrowserForge fingerprint. Note: Not all values will be implemented. If not provided, a random fingerprint will be generated based on the provided os & user_agent. screen (Optional[Screen]): - NOT YET IMPLEMENTED: Constrains the screen dimensions of the generated fingerprint. + Constrains the screen dimensions of the generated fingerprint. Takes a browserforge.fingerprints.Screen instance. + headless (Optional[bool]): + Whether to run the browser in headless mode. Defaults to True. executable_path (Optional[str]): Custom Camoufox browser executable path. firefox_user_prefs (Optional[Dict[str, Any]]): diff --git a/pythonlib/camoufox/addons.py b/pythonlib/camoufox/addons.py index a725b4c..974e3f1 100644 --- a/pythonlib/camoufox/addons.py +++ b/pythonlib/camoufox/addons.py @@ -16,7 +16,7 @@ class DefaultAddons(Enum): Default addons to be downloaded """ - uBO = "https://addons.mozilla.org/firefox/downloads/latest/ublock-origin/latest.xpi" + UBO = "https://addons.mozilla.org/firefox/downloads/latest/ublock-origin/latest.xpi" BPC = "https://gitflic.ru/project/magnolia1234/bpc_uploads/blob/raw?file=bypass_paywalls_clean-latest.xpi" diff --git a/pythonlib/camoufox/async_api.py b/pythonlib/camoufox/async_api.py index 3876288..4bc495d 100644 --- a/pythonlib/camoufox/async_api.py +++ b/pythonlib/camoufox/async_api.py @@ -44,6 +44,7 @@ async def AsyncNewBrowser( exclude_addons: Optional[List[DefaultAddons]] = None, fingerprint: Optional[Fingerprint] = None, screen: Optional[Screen] = None, + headless: Optional[bool] = None, executable_path: Optional[str] = None, firefox_user_prefs: Optional[Dict[str, Any]] = None, proxy: Optional[Dict[str, str]] = None, @@ -85,8 +86,10 @@ async def AsyncNewBrowser( Use a custom BrowserForge fingerprint. Note: Not all values will be implemented. If not provided, a random fingerprint will be generated based on the provided os & user_agent. screen (Optional[Screen]): - NOT YET IMPLEMENTED: Constrains the screen dimensions of the generated fingerprint. + Constrains the screen dimensions of the generated fingerprint. Takes a browserforge.fingerprints.Screen instance. + headless (Optional[bool]): + Whether to run the browser in headless mode. Defaults to True. executable_path (Optional[str]): Custom Camoufox browser executable path. firefox_user_prefs (Optional[Dict[str, Any]]): diff --git a/pythonlib/camoufox/browserforge.yml b/pythonlib/camoufox/browserforge.yml index a62b040..cf5962e 100644 --- a/pythonlib/camoufox/browserforge.yml +++ b/pythonlib/camoufox/browserforge.yml @@ -28,23 +28,21 @@ navigator: screen: # hasHDR is not implemented in Camoufox - # Screen size values seem to be inconsistent, and will not be implemented for the time being. - # availHeight: screen.availHeight - # availWidth: screen.availWidth - # availTop: screen.availTop - # availLeft: screen.availLeft - # height: screen.height - # width: screen.width + availLeft: screen.availLeft + availTop: screen.availTop + availWidth: screen.availWidth + availHeight: screen.availHeight + height: screen.height + width: screen.width colorDepth: screen.colorDepth pixelDepth: screen.pixelDepth # devicePixelRatio is not recommended. Any value other than 1.0 is suspicious. pageXOffset: screen.pageXOffset pageYOffset: screen.pageYOffset - # Disable viewport hijacking temporarily. - # outerHeight: window.outerHeight - # outerWidth: window.outerWidth - # innerHeight: window.innerHeight - # innerWidth: window.innerWidth + outerHeight: window.outerHeight + outerWidth: window.outerWidth + innerHeight: window.innerHeight + innerWidth: window.innerWidth screenX: window.screenX # Tends to generate out of bounds (network inconsistencies): # clientWidth: document.body.clientWidth diff --git a/pythonlib/camoufox/fingerprints.py b/pythonlib/camoufox/fingerprints.py index f870f49..a5ffd16 100644 --- a/pythonlib/camoufox/fingerprints.py +++ b/pythonlib/camoufox/fingerprints.py @@ -1,7 +1,7 @@ import os.path import re from dataclasses import asdict -from typing import Optional +from typing import Any, Dict, Optional from browserforge.fingerprints import Fingerprint, FingerprintGenerator from yaml import CLoader, load @@ -40,8 +40,11 @@ def _cast_to_properties( camoufox_data[type_key] = data -def from_browserforge(fingerprint: Fingerprint, ff_version: Optional[str] = None) -> dict: - camoufox_data = {} +def from_browserforge(fingerprint: Fingerprint, ff_version: Optional[str] = None) -> Dict[str, Any]: + """ + Converts a Browserforge fingerprint to a Camoufox config. + """ + camoufox_data: Dict[str, Any] = {} _cast_to_properties( camoufox_data, cast_enum=BROWSERFORGE_DATA, @@ -51,15 +54,15 @@ def from_browserforge(fingerprint: Fingerprint, ff_version: Optional[str] = None return camoufox_data -def generate(ff_version: Optional[str] = None, **config) -> dict: +def generate_fingerprint(**config) -> Fingerprint: """ - Generates a Firefox fingerprint. + Generates a Firefox fingerprint with Browserforge. """ - data = FP_GENERATOR.generate(**config) - return from_browserforge(data, ff_version=ff_version) + return FP_GENERATOR.generate(**config) if __name__ == "__main__": from pprint import pprint - pprint(generate()) + fp = generate_fingerprint() + pprint(from_browserforge(fp)) diff --git a/pythonlib/camoufox/ip.py b/pythonlib/camoufox/ip.py index b018612..50220c4 100644 --- a/pythonlib/camoufox/ip.py +++ b/pythonlib/camoufox/ip.py @@ -1,9 +1,13 @@ import re +import warnings +from contextlib import contextmanager from dataclasses import dataclass from functools import lru_cache from typing import Dict, Optional, Tuple import requests +from urllib3 import disable_warnings +from urllib3.exceptions import InsecureRequestWarning from .exceptions import InvalidIP, InvalidProxy @@ -69,6 +73,13 @@ def validate_ip(ip: str) -> None: raise InvalidIP(f"Invalid IP address: {ip}") +@contextmanager +def _suppress_insecure_warning(): + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=InsecureRequestWarning) + yield + + @lru_cache(maxsize=None) def public_ip(proxy: Optional[str] = None) -> str: """ @@ -86,11 +97,13 @@ def public_ip(proxy: Optional[str] = None) -> str: ] for url in URLS: try: - resp = requests.get( - url, - proxies=Proxy.as_requests_proxy(proxy) if proxy else None, - timeout=5, - ) + with _suppress_insecure_warning(): + resp = requests.get( # nosec + url, + proxies=Proxy.as_requests_proxy(proxy) if proxy else None, + timeout=5, + verify=False, + ) resp.raise_for_status() ip = resp.text.strip() validate_ip(ip) diff --git a/pythonlib/camoufox/sync_api.py b/pythonlib/camoufox/sync_api.py index 8174f79..7f45651 100644 --- a/pythonlib/camoufox/sync_api.py +++ b/pythonlib/camoufox/sync_api.py @@ -44,6 +44,7 @@ def NewBrowser( exclude_addons: Optional[List[DefaultAddons]] = None, fingerprint: Optional[Fingerprint] = None, screen: Optional[Screen] = None, + headless: Optional[bool] = None, executable_path: Optional[str] = None, firefox_user_prefs: Optional[Dict[str, Any]] = None, proxy: Optional[Dict[str, str]] = None, @@ -85,8 +86,10 @@ def NewBrowser( Use a custom BrowserForge fingerprint. Note: Not all values will be implemented. If not provided, a random fingerprint will be generated based on the provided os & user_agent. screen (Optional[Screen]): - NOT YET IMPLEMENTED: Constrains the screen dimensions of the generated fingerprint. + Constrains the screen dimensions of the generated fingerprint. Takes a browserforge.fingerprints.Screen instance. + headless (Optional[bool]): + Whether to run the browser in headless mode. Defaults to True. executable_path (Optional[str]): Custom Camoufox browser executable path. firefox_user_prefs (Optional[Dict[str, Any]]): diff --git a/pythonlib/camoufox/utils.py b/pythonlib/camoufox/utils.py index d612201..b400a7d 100644 --- a/pythonlib/camoufox/utils.py +++ b/pythonlib/camoufox/utils.py @@ -7,6 +7,7 @@ from typing import Any, Dict, List, Literal, Optional, Tuple, Union, cast import numpy as np import orjson from browserforge.fingerprints import Fingerprint, Screen +from screeninfo import get_monitors from typing_extensions import TypeAlias from ua_parser import user_agent_parser @@ -17,7 +18,7 @@ from .addons import ( threaded_try_load_addons, ) from .exceptions import InvalidPropertyType, UnknownProperty -from .fingerprints import from_browserforge, generate +from .fingerprints import from_browserforge, generate_fingerprint from .ip import Proxy, public_ip, valid_ipv4, valid_ipv6 from .locale import geoip_allowed, get_geolocation, normalize_locale from .pkgman import OS_NAME, get_path, installed_verstr @@ -139,6 +140,25 @@ def determine_ua_os(user_agent: str) -> Literal['mac', 'win', 'lin']: return "lin" +def get_screen_cons(headless: Optional[bool] = None) -> Optional[Screen]: + """ + Determines a sane viewport size for Camoufox if being ran in headful mode. + """ + if headless is False: + return None # Skip if headless + try: + monitors = get_monitors() + except Exception: + return None # Skip if there's an error getting the monitors + if not monitors: + return None # Skip if there are no monitors + + # Use the dimensions from the monitor with greatest screen real estate + monitor = max(monitors, key=lambda m: m.width * m.height) + # Add 25% buffer + return Screen(max_width=int(monitor.width * 1.25), max_height=int(monitor.height * 1.25)) + + def update_fonts(config: Dict[str, Any], target_os: str) -> None: """ Updates the fonts for the target OS. @@ -191,6 +211,7 @@ def get_launch_options( allow_webgl: Optional[bool] = None, proxy: Optional[Dict[str, str]] = None, ff_version: Optional[int] = None, + headless: Optional[bool] = None, firefox_user_prefs: Optional[Dict[str, Any]] = None, launch_options: Optional[Dict[str, Any]] = None, ) -> Dict[str, Any]: @@ -221,21 +242,17 @@ def get_launch_options( else: ff_version_str = installed_verstr().split('.', 1)[0] - # Generate new fingerprint + # Inject a unique Firefox fingerprint if fingerprint is None: - merge_into( - config, - generate( - ff_version=ff_version_str, - screen=screen, - os=os, - ), - ) - else: - merge_into( - config, - from_browserforge(fingerprint, ff_version_str), + fingerprint = generate_fingerprint( + screen=screen or get_screen_cons(headless), + os=os, ) + merge_into( + config, + from_browserforge(fingerprint, ff_version_str), + ) + target_os = get_target_os(config) # Set a random window.history.length @@ -296,5 +313,6 @@ def get_launch_options( "env": env_vars, "firefox_user_prefs": firefox_user_prefs, "proxy": proxy, + "headless": headless, **(launch_options if launch_options is not None else {}), } diff --git a/pythonlib/camoufox/xpi_dl.py b/pythonlib/camoufox/xpi_dl.py index d00d432..ff34893 100644 --- a/pythonlib/camoufox/xpi_dl.py +++ b/pythonlib/camoufox/xpi_dl.py @@ -20,13 +20,13 @@ def add_default_addons( maybe_download_addons(addons, addons_list) -def download_and_extract(url: str, extract_path: str) -> None: +def download_and_extract(url: str, extract_path: str, name: str) -> None: """ Downloads and extracts an addon from a given URL to a specified path """ # Create a temporary file to store the downloaded zip - buffer = webdl(url, desc="Downloading addon") - unzip(buffer, extract_path) + buffer = webdl(url, desc=f"Downloading addon ({name})") + unzip(buffer, extract_path, f"Extracting addon ({name})") def get_addon_path(addon_name: str) -> str: @@ -54,7 +54,7 @@ def maybe_download_addons(addons: List[DefaultAddons], addons_list: List[str]) - # Addon doesn't exist, create directory and download try: os.makedirs(addon_path, exist_ok=True) - download_and_extract(addon.value, addon_path) + download_and_extract(addon.value, addon_path, addon.name) # Add the new addon directory path to addons_list addons_list.append(addon_path) except Exception as e: diff --git a/pythonlib/publish.sh b/pythonlib/publish.sh new file mode 100644 index 0000000..cd4860d --- /dev/null +++ b/pythonlib/publish.sh @@ -0,0 +1,11 @@ +rm -rf ./dist +rm -rf ./camoufox/*.mmdb + +python -m build +twine check dist/* + +read -p "Confirm publish? (y/n) " -n 1 -r +echo +if [[ $REPLY =~ ^[Yy]$ ]]; then + twine upload dist/* +fi diff --git a/pythonlib/pyproject.toml b/pythonlib/pyproject.toml index ace2fa4..b64b5d5 100644 --- a/pythonlib/pyproject.toml +++ b/pythonlib/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "camoufox" -version = "0.2.1" +version = "0.2.2" description = "Wrapper around Playwright to help launch Camoufox" authors = ["daijro "] license = "MIT" @@ -38,6 +38,7 @@ tqdm = "*" numpy = "*" ua_parser = "*" typing_extensions = "*" +screeninfo = "*" lxml = "*" language-tags = "*" geoip2 = {version = "*", optional = true}