From ca7c3e3aa8e0ada41a10de8be521f26bdc0834e9 Mon Sep 17 00:00:00 2001 From: daijro Date: Sun, 3 Nov 2024 04:01:17 -0600 Subject: [PATCH] No progress bar on addon download #65 - Removed progress bars on addon download (shows as percent instead) - Default addons are downloaded on `camoufox fetch` --- pythonlib/camoufox/__main__.py | 20 ++++++++++++++++---- pythonlib/camoufox/pkgman.py | 15 +++++++++++++-- pythonlib/camoufox/xpi_dl.py | 18 ++++++++++++------ 3 files changed, 41 insertions(+), 12 deletions(-) diff --git a/pythonlib/camoufox/__main__.py b/pythonlib/camoufox/__main__.py index 034de3b..dac8416 100644 --- a/pythonlib/camoufox/__main__.py +++ b/pythonlib/camoufox/__main__.py @@ -10,10 +10,17 @@ from os import environ from typing import Optional import click -from browserforge.download import download as update_browserforge +from .addons import DefaultAddons from .locale import ALLOW_GEOIP, download_mmdb, remove_mmdb from .pkgman import INSTALL_DIR, CamoufoxFetcher, installed_verstr, rprint +from .xpi_dl import maybe_download_addons + +try: + from browserforge.download import download as update_browserforge +except ImportError: + # Account for other Browserforge versions + from browserforge.download import Download as update_browserforge class CamoufoxUpdate(CamoufoxFetcher): @@ -70,16 +77,21 @@ def cli() -> None: @cli.command(name='fetch') -@click.option('--browserforge', is_flag=True, help='Update browserforge\'s header and fingerprint definitions') +@click.option( + '--browserforge', is_flag=True, help='Update browserforge\'s header and fingerprint definitions' +) def fetch(browserforge=False) -> None: """ - Fetch the latest version of Camoufox and optionally update the browserforge's database + Fetch the latest version of Camoufox and optionally update Browserforge's database """ CamoufoxUpdate().update() # Fetch the GeoIP database if ALLOW_GEOIP: download_mmdb() - + + # Download default addons + maybe_download_addons(list(DefaultAddons)) + if browserforge: update_browserforge(headers=True, fingerprints=True) diff --git a/pythonlib/camoufox/pkgman.py b/pythonlib/camoufox/pkgman.py index ed866f8..46ba77a 100644 --- a/pythonlib/camoufox/pkgman.py +++ b/pythonlib/camoufox/pkgman.py @@ -389,6 +389,7 @@ def webdl( url: str, desc: Optional[str] = None, buffer: Optional[DownloadBuffer] = None, + bar: bool = True, ) -> DownloadBuffer: """ Download a file from the given URL and return it as BytesIO. @@ -396,6 +397,7 @@ def webdl( Args: url (str): The URL to download the file from buffer (Optional[BytesIO]): A BytesIO object to store the downloaded file + bar (bool): Whether to show the progress bar Returns: DownloadBuffer: The downloaded file content as a BytesIO object @@ -411,7 +413,13 @@ def webdl( if buffer is None: buffer = BytesIO() - with tqdm(total=total_size, unit='iB', unit_scale=True, desc=desc) as progress_bar: + with tqdm( + total=total_size, + unit='iB', + bar_format=None if bar else '{desc}: {percentage:3.0f}%', + unit_scale=True, + desc=desc, + ) as progress_bar: for data in response.iter_content(block_size): size = buffer.write(data) progress_bar.update(size) @@ -424,6 +432,7 @@ def unzip( zip_file: DownloadBuffer, extract_path: str, desc: Optional[str] = None, + bar: bool = True, ) -> None: """ Extract the contents of a zip file to the installation directory. @@ -436,7 +445,9 @@ def unzip( OSError: If there's an error creating directories or writing files """ with ZipFile(zip_file) as zf: - for member in tqdm(zf.infolist(), desc=desc): + for member in tqdm( + zf.infolist(), desc=desc, bar_format=None if bar else '{desc}: {percentage:3.0f}%' + ): zf.extract(member, extract_path) diff --git a/pythonlib/camoufox/xpi_dl.py b/pythonlib/camoufox/xpi_dl.py index ff34893..b05822b 100644 --- a/pythonlib/camoufox/xpi_dl.py +++ b/pythonlib/camoufox/xpi_dl.py @@ -1,4 +1,5 @@ import os +from multiprocessing import Lock from typing import List, Optional from .addons import DefaultAddons @@ -17,7 +18,8 @@ def add_default_addons( addons = [addon for addon in DefaultAddons if addon not in exclude_list] - maybe_download_addons(addons, addons_list) + with Lock(): + maybe_download_addons(addons, addons_list) def download_and_extract(url: str, extract_path: str, name: str) -> None: @@ -25,8 +27,8 @@ def download_and_extract(url: str, extract_path: str, name: str) -> None: Downloads and extracts an addon from a given URL to a specified path """ # Create a temporary file to store the downloaded zip - buffer = webdl(url, desc=f"Downloading addon ({name})") - unzip(buffer, extract_path, f"Extracting addon ({name})") + buffer = webdl(url, desc=f"Downloading addon ({name})", bar=False) + unzip(buffer, extract_path, f"Extracting addon ({name})", bar=False) def get_addon_path(addon_name: str) -> str: @@ -36,7 +38,9 @@ def get_addon_path(addon_name: str) -> str: return get_path(os.path.join("addons", addon_name)) -def maybe_download_addons(addons: List[DefaultAddons], addons_list: List[str]) -> None: +def maybe_download_addons( + addons: List[DefaultAddons], addons_list: Optional[List[str]] = None +) -> None: """ Downloads and extracts addons from a given dictionary to a specified list Skips downloading if the addon is already downloaded @@ -48,7 +52,8 @@ def maybe_download_addons(addons: List[DefaultAddons], addons_list: List[str]) - # Check if the addon is already extracted if os.path.exists(addon_path): # Add the existing addon path to addons_list - addons_list.append(addon_path) + if addons_list is not None: + addons_list.append(addon_path) continue # Addon doesn't exist, create directory and download @@ -56,6 +61,7 @@ def maybe_download_addons(addons: List[DefaultAddons], addons_list: List[str]) - os.makedirs(addon_path, exist_ok=True) download_and_extract(addon.value, addon_path, addon.name) # Add the new addon directory path to addons_list - addons_list.append(addon_path) + if addons_list is not None: + addons_list.append(addon_path) except Exception as e: print(f"Failed to download and extract {addon.name}: {e}")