From 4c52518dd2482245c58c9c47cfc852ab56b8da87 Mon Sep 17 00:00:00 2001 From: daijro Date: Wed, 4 Dec 2024 19:35:54 -0600 Subject: [PATCH] pythonlib: Cleanup & bump to 0.4.6 - `camoufox test` will no longer highlight the cursor by default - Fixed launch_options blocking async - WebGL database cleanup & added ability to query all possible vendor/renderer pairs --- pythonlib/camoufox/__main__.py | 2 +- pythonlib/camoufox/async_api.py | 12 ++++-- pythonlib/camoufox/pkgman.py | 4 +- pythonlib/camoufox/sync_api.py | 7 ++-- pythonlib/camoufox/webgl/sample.py | 50 ++++++++++++++++++------- pythonlib/camoufox/webgl/webgl_data.db | Bin 274432 -> 266240 bytes pythonlib/pyproject.toml | 2 +- 7 files changed, 54 insertions(+), 23 deletions(-) diff --git a/pythonlib/camoufox/__main__.py b/pythonlib/camoufox/__main__.py index d819a04..8514c62 100644 --- a/pythonlib/camoufox/__main__.py +++ b/pythonlib/camoufox/__main__.py @@ -114,7 +114,7 @@ def test(url: Optional[str] = None) -> None: """ from .sync_api import Camoufox - with Camoufox(headless=False, env=environ) as browser: + with Camoufox(headless=False, env=environ, config={'showcursor': False}) as browser: page = browser.new_page() if url: page.goto(url) diff --git a/pythonlib/camoufox/async_api.py b/pythonlib/camoufox/async_api.py index f30d424..1b61b68 100644 --- a/pythonlib/camoufox/async_api.py +++ b/pythonlib/camoufox/async_api.py @@ -1,3 +1,5 @@ +import asyncio +from functools import partial from typing import Any, Dict, Optional, Union, overload from playwright.async_api import ( @@ -82,13 +84,17 @@ async def AsyncNewBrowser( else: virtual_display = None - opt = from_options or launch_options(headless=headless, debug=debug, **kwargs) + if not from_options: + from_options = await asyncio.get_event_loop().run_in_executor( + None, + partial(launch_options, headless=headless, debug=debug, **kwargs), + ) # Persistent context if persistent_context: - context = await playwright.firefox.launch_persistent_context(**opt) + context = await playwright.firefox.launch_persistent_context(**from_options) return await async_attach_vd(context, virtual_display) # Browser - browser = await playwright.firefox.launch(**opt) + browser = await playwright.firefox.launch(**from_options) return await async_attach_vd(browser, virtual_display) diff --git a/pythonlib/camoufox/pkgman.py b/pythonlib/camoufox/pkgman.py index 9210ebd..7a8ca34 100644 --- a/pythonlib/camoufox/pkgman.py +++ b/pythonlib/camoufox/pkgman.py @@ -55,16 +55,16 @@ LOCAL_DATA: Path = Path(os.path.abspath(__file__)).parent # The supported architectures for each OS OS_ARCH_MATRIX: Dict[str, List[str]] = { - 'mac': ['x86_64', 'arm64'], 'win': ['x86_64', 'i686'], + 'mac': ['x86_64', 'arm64'], 'lin': ['x86_64', 'arm64', 'i686'], } # The relative path to the camoufox executable LAUNCH_FILE = { 'win': 'camoufox.exe', - 'lin': 'camoufox-bin', 'mac': '../MacOS/camoufox', + 'lin': 'camoufox-bin', } diff --git a/pythonlib/camoufox/sync_api.py b/pythonlib/camoufox/sync_api.py index da191f7..1a06176 100644 --- a/pythonlib/camoufox/sync_api.py +++ b/pythonlib/camoufox/sync_api.py @@ -82,13 +82,14 @@ def NewBrowser( else: virtual_display = None - opt = from_options or launch_options(headless=headless, debug=debug, **kwargs) + if not from_options: + from_options = launch_options(headless=headless, debug=debug, **kwargs) # Persistent context if persistent_context: - context = playwright.firefox.launch_persistent_context(**opt) + context = playwright.firefox.launch_persistent_context(**from_options) return sync_attach_vd(context, virtual_display) # Browser - browser = playwright.firefox.launch(**opt) + browser = playwright.firefox.launch(**from_options) return sync_attach_vd(browser, virtual_display) diff --git a/pythonlib/camoufox/webgl/sample.py b/pythonlib/camoufox/webgl/sample.py index ddb6d18..f963d05 100644 --- a/pythonlib/camoufox/webgl/sample.py +++ b/pythonlib/camoufox/webgl/sample.py @@ -1,10 +1,15 @@ import sqlite3 from pathlib import Path -from typing import Dict, Optional +from typing import Dict, List, Optional, Tuple import numpy as np import orjson +from camoufox.pkgman import OS_ARCH_MATRIX + +# Get database path relative to this file +DB_PATH = Path(__file__).parent / 'webgl_data.db' + def sample_webgl( os: str, vendor: Optional[str] = None, renderer: Optional[str] = None @@ -24,23 +29,19 @@ def sample_webgl( Raises: ValueError: If invalid OS provided or no data found for OS/vendor/renderer """ - # Map OS to probability column - os_map = {'win': 'windows', 'mac': 'macos', 'lin': 'linux'} - if os not in os_map: - raise ValueError(f'Invalid OS: {os}. Must be one of: {", ".join(os_map)}') - os = os_map[os] - - # Get database path relative to this file - db_path = Path(__file__).parent / 'webgl_data.db' + # Check that the OS is valid (avoid SQL injection) + if os not in OS_ARCH_MATRIX: + raise ValueError(f'Invalid OS: {os}. Must be one of: win, mac, lin') # Connect to database - conn = sqlite3.connect(db_path) + conn = sqlite3.connect(DB_PATH) cursor = conn.cursor() if vendor and renderer: # Get specific vendor/renderer pair and verify it exists for this OS cursor.execute( - f'SELECT vendor, renderer, data, {os} FROM webgl_fingerprints WHERE vendor = ? AND renderer = ?', + f'SELECT vendor, renderer, data, {os} FROM webgl_fingerprints ' # nosec + 'WHERE vendor = ? AND renderer = ?', (vendor, renderer), ) result = cursor.fetchone() @@ -51,7 +52,7 @@ def sample_webgl( if result[3] <= 0: # Check OS-specific probability # Get a list of possible (vendor, renderer) pairs for this OS cursor.execute( - f'SELECT DISTINCT vendor, renderer FROM webgl_fingerprints WHERE {os} > 0' + f'SELECT DISTINCT vendor, renderer FROM webgl_fingerprints WHERE {os} > 0' # nosec ) possible_pairs = cursor.fetchall() raise ValueError( @@ -63,7 +64,9 @@ def sample_webgl( return orjson.loads(result[2]) # Get all vendor/renderer pairs and their probabilities for this OS - cursor.execute(f'SELECT vendor, renderer, data, {os} FROM webgl_fingerprints WHERE {os} > 0') + cursor.execute( + f'SELECT vendor, renderer, data, {os} FROM webgl_fingerprints WHERE {os} > 0' # nosec + ) results = cursor.fetchall() conn.close() @@ -82,3 +85,24 @@ def sample_webgl( # Parse the JSON data string return orjson.loads(data_strs[idx]) + + +def get_possible_pairs() -> Dict[str, List[Tuple[str, str]]]: + """ + Get all possible (vendor, renderer) pairs for all OS, where the probability is greater than 0. + """ + # Connect to database + conn = sqlite3.connect(DB_PATH) + cursor = conn.cursor() + + # Get all vendor/renderer pairs for each OS where probability > 0 + result: Dict[str, List[Tuple[str, str]]] = {} + for os_type in OS_ARCH_MATRIX: + cursor.execute( + 'SELECT DISTINCT vendor, renderer FROM webgl_fingerprints ' + f'WHERE {os_type} > 0 ORDER BY {os_type} DESC', # nosec + ) + result[os_type] = cursor.fetchall() + + conn.close() + return result diff --git a/pythonlib/camoufox/webgl/webgl_data.db b/pythonlib/camoufox/webgl/webgl_data.db index 5caf7f155eb7ea71fd873c879e3e9810945e7bce..f8448aab92ea813fe576d34c404c3c75b0546740 100644 GIT binary patch delta 186 zcmZp8AkeTtV1hI+GXn#IBM^hZL=AI(W(K{g0$w1Okw1umKWKBHz!N^MMr}rRaYIAK zCdSDj{PiLVL9UKIItsao$xv2KX5QrY{4$Kpn|TDn`5Du?3m!zyu^%7`HuO z{?E^7zexc|vMy6#5!k}QBGAl|(9V*;2*ga=SrV9k`!m{XJHY&3p3!cb08n~6%Le8@ W{M#-t-(}i-H%N(>(Q(-aW&r>V11|Ic delta 460 zcmXxgOG|=56vpux-#05W=gsbEX?H6#&B`unTC|Cv%ce~WBR4^15mB_!T(nk%)~#$? z3idVn04@9mwF!cP`U|!A%{(&C4ASX&77oK7i#_K z?A2zu3CjA;KtN_&3VSN;Or|c+-8*aX4X^un{~-0Bj}A{Rt_p9;DVNJuV^_&|A?eCl zqZ-A`CKgSW{69*rmbVSN%4a^(l1u&1GJ6U=d4* zAc_^#K5G^XCdMp2ZIyS8xx})G+l0JLm?{bFXH!OsRn54oYWlPOsC~KQO3BTmaPdVm PN?W?|pcBe^>|6Z;0G2{@ diff --git a/pythonlib/pyproject.toml b/pythonlib/pyproject.toml index ca1be07..e7ad9c7 100644 --- a/pythonlib/pyproject.toml +++ b/pythonlib/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "camoufox" -version = "0.4.6-beta" +version = "0.4.6" description = "Wrapper around Playwright to help launch Camoufox" authors = ["daijro "] license = "MIT"