How to download a file via https using QNetworkAccessManager

Question

I'm trying to write a class using QtNetwork to download a file without freezing my GUI. This seems to work with http URLs (tested with "http://webcode.me"), but not with the https URL from my example.

import os
from typing import Optional
import urllib.parse

from PyQt5.QtCore import pyqtSignal, QByteArray, QFile, QObject, QUrl
from PyQt5.QtNetwork import QNetworkAccessManager, QNetworkReply, QNetworkRequest


class AsyncDownloader(QObject):
    def __init__(self, url: str, filename: str, parent=None):
        super().__init__(parent)
        self.net_mgr = QNetworkAccessManager()
        self.req = QNetworkRequest(QUrl(url))
        self.fetch_task: Optional[QNetworkReply] = None
        self.data: Optional[QByteArray] = None
        self.file = QFile(filename)

        self.net_mgr.sslErrors.connect(self._ignore_ssl_errors)

    def start_fetch(self):
        self.fetch_task = self.net_mgr.get(self.req)
        self.fetch_task.downloadProgress.connect(self.on_progress)
        self.fetch_task.finished.connect(self.on_finished)
    
    def _ignore_ssl_errors(self, reply: QNetworkReply, errors: List[QSslError]):
        print(f"errors {errors}")
        reply.ignoreSslErrors(errors)
    
    def on_progress(self, bytes_received: int, bytes_total: int):
        print(f"bytes received {bytes_received} (total {bytes_total})")

    def on_finished(self):
        print("finished")
        self.data = self.fetch_task.readAll()
        if not self.file.open(QFile.WriteOnly):
            raise IOError(f"Unable to write to {self.file.fileName}")
        self.file.write(self.data)
        self.file.close()
        print(f"file written to {self.file.fileName()}")


if __name__ == '__main__':
    from pathlib import Path
    from PyQt5.QtWidgets import QApplication

    dl_path = os.path.join(str(Path.home()), "test_dl")
    os.makedirs(dl_path, exist_ok=True)
    app = QApplication([])
    downloader = AsyncDownloader(
        "https://github.com/PiRK/Electrum-ABC-Build-Tools/releases/download/v1.0/tor-linux",
        os.path.join(dl_path, "tor")
    )
    downloader.start_fetch()

    app.exec_()

The errors (or warnings?) I'm getting are:

qt.network.ssl: QSslSocket: cannot resolve EVP_PKEY_base_id
qt.network.ssl: QSslSocket: cannot resolve SSL_get_peer_certificate
qt.network.ssl: QSslSocket: cannot call unresolved function SSL_get_peer_certificate
errors [<PyQt5.QtNetwork.QSslError object at 0x7fad867112a0>]
qt.network.ssl: QSslSocket: cannot call unresolved function EVP_PKEY_base_id

bytes received 0 (total 0)
finished
file written to /home/myname/test_dl/tor

The file that is written is empty.

I tried adding the following lines just after self.net_mgr = ....:

    parsed_url = urllib.parse.urlparse(url)
    if parsed_url.scheme == "https":
        self.net_mgr.connectToHostEncrypted(parsed_url.hostname)

This does not help.

The download work fine with wget:

$ wget "https://github.com/PiRK/Electrum-ABC-Build-Tools/releases/download/v1.0/tor-linux"
...
tor-linux                                    100%[=============================================================================================>]  15,34M   985KB/s    in 16s

2023-02-16 16:36:51 (969 KB/s) - ‘tor-linux’ saved [16090880/16090880]

Answer 1

After failing to get my QNetworkAccessManager to work for HTTPS, I used an alternative solution based on Python's multiprocessing standard library and the requests library (not stdlib, but recommended by the official python documentation for urllib.request).

The only drawback is that I'm not getting any download progress information.

import multiprocessing
import requests


class Downloader:
    """URL downloader designed to be run as a separate process and to communicate
    with the main process via a Queue.

    The queue can be monitored for the following messages (as str objects):

      - "@started@"
      - "@HTTP status@ {status code} {reason}"
        (e.g "@HTTP status@ 200 OK")
      - "@content size@ {size in bytes}"
      - "@finished@"
    """

    def __init__(self, url: str, filename: str):
        self.url = url
        self.filename = filename

        self.queue = multiprocessing.Queue()

    def run_download(self):
        self.queue.put("@started@")
        r = requests.get(url)
        self.queue.put(f"@HTTP status@ {r.status_code} {r.reason}")
        self.queue.put(f"@content size@ {len(r.content)}")
        with open(self.filename, "wb") as f:
            f.write(r.content)
        self.queue.put("@finished@")


if __name__ == '__main__':
    from pathlib import Path
    from PyQt5.QtWidgets import QApplication
    from PyQt5.QtCore import QTimer
    import os
    import sys

    url = sys.argv[1]
    fname = sys.argv[2]

    dl_path = os.path.join(str(Path.home()), "test_dl")
    os.makedirs(dl_path, exist_ok=True)
    app = QApplication([])

    downloader = Downloader(
        url,
        os.path.join(dl_path, fname)
    )
    process = multiprocessing.Process(target=downloader.run_download)

    def read_queue():
        while not downloader.queue.empty():
            msg = downloader.queue.get()
            print(msg)

    timer = QTimer()
    timer.timeout.connect(read_queue)
    timer.timeout.connect(lambda: print("."))

    process.start()
    timer.start(500)

    app.exec_()

Here is an example of output for a 156 MB file:

$ python downloader.py "https://stsci-opo.org/STScI-01GGF8H15VZ09MET9HFBRQX4S3.png" large_img_https.png
@started@
.
.
.
.
.
.
.
.
.
@HTTP status@ 200 OK
@content size@ 159725397
@finished@
.
.
.
^Z
[3]+  Stopped                 python downloader.py "https://stsci-opo.org/STScI-01GGF8H15VZ09MET9HFBRQX4S3.png" large_img_https.png
$ kill %3