How to find out how long a search for files will take on python?

Question

So I have a little app that searches for all xml files on my pc, copying the files that have 44 digits as the filename to the "output" folder.

The problem is that the final user needs an indication of the progress and remaining time of the task.

This is the module to copy files:

xml_search.py

import os
import re
from threading import Thread
from datetime import datetime
import time
import shutil
import winsound

os.system('cls')

def get_drives():
    response = os.popen("wmic logicaldisk get caption")
    list1 = []
    t1 = datetime.now()
    for line in response.readlines():
        line = line.strip("\n")
        line = line.strip("\r")
        line = line.strip(" ")
        if (line == "Caption" or line == ""):
            continue
        list1.append(line + '\\')
    return list1


def search1(drive):
    for root, dir, files in os.walk(drive):
        for file in files:
            if re.match("\d{44}.xml", file):
                filename = os.path.join(root, file)
                try:
                    shutil.copy(filename, os.path.join('output', file))
                except Exception as e:
                    pass

def exec_(callback):
    t1 = datetime.now()
    list2 = []   # empty list is created
    list1 = get_drives()
    for each in list1:
        process1 = Thread(target=search1, args=(each,))
        process1.start()
        list2.append(process1)

    for t in list2:
        t.join()  # Terminate the threads

    t2 = datetime.now()
    total = str(t2-t1)
    print(total, file=open('times.txt', 'a'), end="\n")
    for x in range(3):
        winsound.Beep(2000,100)
        time.sleep(.1)
    callback()


if __name__ == "__main__":
    exec_()

Answer 1

The below code uses progressbar library and it shows

indication of the progress and remaining time of the task

import progressbar
from time import sleep

bar = progressbar.ProgressBar(maxval=1120, \
    widgets=[progressbar.Bar('=', '[', ']'), ' ', progressbar.ETA()])
bar.start()
for i in range(1120):
    bar.update(i+1)
    sleep(0.1)
bar.finish()

You would need to add the above modified code to your code. So in your case, you would need to count the number of files and provide it as input to ProgressBar constructor's maxval argument and remove sleep call.

The suggested solution with progress bar should work with one thread. You would need to figure out how to initiate the progress bar and where to put the updates if you insist to work with multiple threads.

Answer 2

Try to implement a timer decorator like the following:

import time


def mytimer(func):
    def wrapper():
        t1 = time.time()
        result = func()
        t2 = time.time()
        print(f"The function {func.__name__} was run {t2 - t1} seconds")
        return result

    return wrapper

@mytimer
def TimeConsumingFunction():
    time.sleep(3)
    print("Hello timers")

TimeConsumingFunction()

Output:

/usr/bin/python3.7 /home/user/Documents/python-workspace/timers/example.py
Hello timers
The function TimeConsumingFunction was run 3.002610206604004 seconds

Process finished with exit code 0