Source code for pyrogram.methods.advanced.save_file

#  Pyrogram - Telegram MTProto API Client Library for Python
#  Copyright (C) 2017-present Dan <https://github.com/delivrance>
#
#  This file is part of Pyrogram.
#
#  Pyrogram is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Lesser General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  Pyrogram is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public License
#  along with Pyrogram.  If not, see <http://www.gnu.org/licenses/>.

from __future__ import annotations

import asyncio
import contextlib
import functools
import inspect
import io
import logging
import math
import os
import time
from hashlib import md5
from typing import BinaryIO, TYPE_CHECKING

import pyrogram
from pyrogram import StopTransmission, raw, utils
from pyrogram.errors import Flood, RPCError
from pyrogram.methods.rate_limiter import AdaptiveBucket

TokenBucket = AdaptiveBucket
from pyrogram.session import Session

if TYPE_CHECKING:
    from collections.abc import Callable

log = logging.getLogger(__name__)

PART_SIZE = 512 * 1024
POOL_SIZE = 20
MAX_RETRIES = 16
STALL_TIMEOUT = 900
READ_BUFFER = 4 * 1024 * 1024
MAX_BATCH = 4 * 1024 * 1024
PACER_BURST = 16


async def _stop_workers(queue: asyncio.Queue, workers: list) -> list:
    """Retire the upload workers and collect what they ended with.

    The sentinels are what the workers exit on, so the happy path must deliver
    one to each. But ``queue`` is bounded by the worker count, so once the
    workers are gone there is nobody to make room and an unconditional ``put``
    waits for a consumer that will never run - with ``save_file_semaphore``
    still held, which wedges every other upload on the client.
    """
    delivered = 0

    for _ in workers:
        if all(t.done() for t in workers):
            break

        try:
            await asyncio.wait_for(queue.put(None), Session.MEDIA_WAIT_TIMEOUT)
        except asyncio.TimeoutError:
            break

        delivered += 1

    if delivered < len(workers):
        for t in workers:
            if not t.done():
                t.cancel()

    return await asyncio.gather(*workers, return_exceptions=True)


class SaveFile:
    async def save_file(
        self: pyrogram.Client,
        path: str | os.PathLike | BinaryIO,
        file_id: int | None = None,
        file_part: int = 0,
        progress: Callable | None = None,
        progress_args: tuple = (),
    ):
        """Upload a file onto Telegram servers, without sending the message to anyone.

        Useful whenever an InputFile type is required for raw API functions.

        .. include:: /_includes/usable-by/users-bots.rst

        Parameters:
            path (``str`` | ``BinaryIO``):
                File path or binary file-like object.

            file_id (``int``, *optional*):
                File ID to resume an interrupted upload.

            file_part (``int``, *optional*):
                Part number to resume from.

            progress (``Callable``, *optional*):
                Callback — takes *(current, total)* as positional arguments.

            progress_args (``tuple``, *optional*):
                Extra arguments for the progress callback.

        Returns:
            ``InputFile | InputFileBig``: On success.

        Raises:
            RPCError: In case of a Telegram RPC error.
        """
        from pyrogram.client import ReadAhead

        async with self.save_file_semaphore:
            if path is None:
                return None

            async def worker(session):
                while True:
                    data = await queue.get()

                    if data is None:
                        return

                    try:
                        await _send_part(session, data)
                        _acked[0] += 1
                        _pacer.on_success()
                    finally:
                        data = None
                        budget.release()

            async def _send_part(session, data):
                for attempt in range(MAX_RETRIES):
                    try:
                        if not await session.invoke(
                            data, timeout=Session.MEDIA_WAIT_TIMEOUT, sleep_threshold=0
                        ):
                            raise OSError("part not accepted by the server")
                        break
                    except StopTransmission:
                        raise
                    except (OSError, TimeoutError, RPCError, asyncio.TimeoutError) as e:
                        if (
                            isinstance(e, RPCError)
                            and not isinstance(e, Flood)
                            and not 500 <= (e.CODE or 0) < 600
                        ):
                            raise
                        if attempt == MAX_RETRIES - 1:
                            log.exception(
                                "Upload part failed after %d attempts",
                                MAX_RETRIES,
                            )
                            raise
                        delay = min(2**attempt, 30)
                        err_str = str(e)
                        if isinstance(e, Flood):
                            if isinstance(e.value, int):
                                delay = min(e.value, 300)
                            _pacer.on_flood()
                        log.warning(
                            "Retrying upload part (attempt %d/%d): %s",
                            attempt + 1,
                            MAX_RETRIES,
                            err_str[:120],
                        )
                        await asyncio.sleep(delay)

            async def read_batch():
                batch_size = min(PART_SIZE * n_workers, MAX_BATCH)
                return await self.loop.run_in_executor(self.executor, fp.read, batch_size)

            part_size = PART_SIZE

            if isinstance(path, (str, os.PathLike)):
                fp = open(path, "rb", buffering=READ_BUFFER)
            elif isinstance(path, io.IOBase) and not isinstance(path, io.TextIOBase):
                fp = path
            else:
                raise ValueError(
                    "Invalid file. Expected a file path as string "
                    "or a binary (not text) file pointer"
                )

            file_name = utils.get_file_name(fp, fallback="file.jpg")

            fp.seek(0, os.SEEK_END)
            file_size = fp.tell()
            fp.seek(0)

            if file_size == 0:
                raise ValueError("File size equals to 0 B")

            is_bot = self.me.is_bot if hasattr(self.me, "is_bot") else False
            is_premium = self.me.is_premium if hasattr(self.me, "is_premium") else False

            file_size_limit_mib = 4000 if is_premium else 2000

            if file_size > file_size_limit_mib * 1024 * 1024:
                raise ValueError(f"Can't upload files bigger than {file_size_limit_mib} MiB")

            file_total_parts = int(math.ceil(file_size / part_size))
            is_big = file_size > 10 * 1024 * 1024
            pool_cap = max(1, math.ceil((file_total_parts - file_part) / 2))
            if is_bot:
                rate_limit = int(os.environ.get("TOBIGRAM_UPLOAD_RATE_BOT", 32))
                pool_size = min(
                    int(os.environ.get("TOBIGRAM_UPLOAD_POOL_BOT", 8)),
                    POOL_SIZE,
                    pool_cap,
                )
            elif is_premium:
                rate_limit = int(os.environ.get("TOBIGRAM_UPLOAD_RATE_PREMIUM", 300))
                pool_size = min(
                    int(os.environ.get("TOBIGRAM_UPLOAD_POOL_PREMIUM", 14)),
                    POOL_SIZE,
                    pool_cap,
                )
            else:
                rate_limit = int(os.environ.get("TOBIGRAM_UPLOAD_RATE_USER", 32))
                pool_size = min(
                    int(os.environ.get("TOBIGRAM_UPLOAD_POOL_USER", 12)),
                    POOL_SIZE,
                    pool_cap,
                )

            is_missing_part = file_id is not None
            file_id = file_id or self.rnd_id()
            md5_sum = md5() if not is_big and not is_missing_part else None

            dc_id = await self.storage.dc_id()
            pool_lease = contextlib.AsyncExitStack()

            try:
                pool_task = await pool_lease.enter_async_context(self._media_pool(dc_id, pool_size))
                pool = await pool_task

                if not pool:
                    raise OSError(f"No media session available for DC {dc_id}")

                _acked = [0]

                n_workers = len(pool) * 2
                queue = asyncio.Queue(n_workers)
                budget = ReadAhead(self.read_ahead_slots)
                workers = [
                    self.loop.create_task(worker(pool[i % len(pool)])) for i in range(n_workers)
                ]
            except BaseException:
                await pool_lease.aclose()
                raise
            next_batch_task = None
            _pacer = AdaptiveBucket(
                rate=rate_limit, ceiling=max(rate_limit, 300), burst=PACER_BURST, step=1.0
            )
            _stalled_since = 0.0
            _last_reported = -1

            async def _report(parts: int) -> None:
                nonlocal _last_reported

                if not progress or parts == _last_reported:
                    return

                _last_reported = parts

                func = functools.partial(
                    progress, min(parts * part_size, file_size), file_size, *progress_args
                )

                try:
                    if inspect.iscoroutinefunction(progress):
                        await func()
                    else:
                        await self.loop.run_in_executor(self.executor, func)
                except StopTransmission:
                    raise
                except Exception as e:
                    log.warning(f"Upload progress callback error: {e}")

            try:
                fp.seek(part_size * file_part)
                next_batch_task = self.loop.create_task(read_batch())

                while True:
                    batch = await next_batch_task
                    next_batch_task = self.loop.create_task(read_batch())

                    if not batch:
                        next_batch_task.cancel()
                        if not is_big and not is_missing_part:
                            md5_sum = md5_sum.hexdigest()
                        break

                    async def _check_workers():
                        for t in workers:
                            # asking a cancelled task for its exception re-raises
                            if t.done() and not t.cancelled():
                                exc = t.exception()
                                if exc is not None:
                                    raise exc

                    await _check_workers()

                    for start in range(0, len(batch), part_size):
                        chunk = batch[start : start + part_size]

                        if is_big:
                            rpc = raw.functions.upload.SaveBigFilePart(
                                file_id=file_id,
                                file_part=file_part,
                                file_total_parts=file_total_parts,
                                bytes=chunk,
                            )
                        else:
                            rpc = raw.functions.upload.SaveFilePart(
                                file_id=file_id, file_part=file_part, bytes=chunk
                            )

                        await _pacer.acquire()

                        await budget.acquire()

                        while True:
                            try:
                                await asyncio.wait_for(queue.put(rpc), timeout=30)
                                _stalled_since = 0.0
                                break
                            except asyncio.TimeoutError:
                                await _check_workers()
                                _now = time.monotonic()
                                if _stalled_since == 0.0:
                                    _stalled_since = _now
                                    log.warning(
                                        "Upload queue full: workers throttled (flood/connection churn), "
                                        "waiting up to %ss",
                                        STALL_TIMEOUT,
                                    )
                                elif _now - _stalled_since > STALL_TIMEOUT:
                                    raise TimeoutError(
                                        "Upload stalled: no part completed for "
                                        f"{STALL_TIMEOUT}s while workers are alive "
                                        "(flood or network throttling)"
                                    )
                                await asyncio.sleep(1)

                        if is_missing_part:
                            next_batch_task.cancel()
                            results = await _stop_workers(queue, workers)
                            for r in results:
                                if isinstance(r, BaseException) and not isinstance(
                                    r, asyncio.CancelledError
                                ):
                                    raise r
                            return None

                        if not is_big and not is_missing_part:
                            md5_sum.update(chunk)

                        rpc = None
                        chunk = None
                        file_part += 1

                        await _report(_acked[0])

                    batch = None

            except StopTransmission:
                raise
            except Exception as e:
                log.exception(e)
                raise
            else:
                results = await _stop_workers(queue, workers)

                for r in results:
                    if isinstance(r, BaseException) and not isinstance(r, asyncio.CancelledError):
                        raise r

                await _report(file_total_parts)

                if is_big:
                    return raw.types.InputFileBig(
                        id=file_id,
                        parts=file_total_parts,
                        name=file_name,
                    )
                else:
                    return raw.types.InputFile(
                        id=file_id,
                        parts=file_total_parts,
                        name=file_name,
                        md5_checksum=md5_sum,
                    )
            finally:
                if next_batch_task is not None and not next_batch_task.done():
                    next_batch_task.cancel()

                await _stop_workers(queue, workers)
                budget.release_all()
                await pool_lease.aclose()

                if isinstance(path, (str, os.PathLike)):
                    fp.close()