multiple task management in python3

Question

Here I need to run many sub-tasks in 2 steps:

1. run multiple programs at the same time. These programs are some external executable programs.

2. Wait for all programs in step 1 to end before running the next function

I known the multipleprocessing will help. But how to implement these two steps?

How to hang up the main thread? How to check the sub-programs status in real time?

Thanks.

Answer 1

On phase 2 there is only process_3 with different parameters. Using the args argument of the threading.Thread object you can pass arguments to you job's function.

import threading
from time import sleep
import os

def process_1():
    print('process_1 running')
    # calling your executable
    os.system("./process_1.sh")
    # simultating the executable time
    sleep(5)
    
def process_2():
    print('process_2 running')
    os.system("./process_2.sh")
    sleep(5)

def process_3(parameter:str):
    command_list = ("./process_3.sh",  parameter)
    command = " ".join(command_list)
    print('executing : ', command)
    os.system(command)
    sleep(5)




if __name__ == '__main__':
    
    # storing the processes functions into phase 1 and 2
    phase_1_processes = [process_1, process_2]
    phase_2_processes = [process_3]
    process_3_parameters = ['param_1', 'param_2']

    

    print('starting phase 1 ...')
    # phase_1 ...
    print('phase_1 is over ... ')
    
    
    # phase 2, with only process_3
    print('starting phase 2 ...')
    jobs=[]
    for param in process_3_parameters:
        print('param inside run : ', param)
        jobs.append(threading.Thread(target=process_3,
                                     args=(param,))) # note the comma

    # starting the jobs
    for job in jobs:
        job.start()

    # Wait for all the thread to finish
    for job in jobs:
        job.join()
    print('phase_2 is over ... ')
    

Answer 2

You can use the threading package to do thread your actions, you can use the os package to run a executable file, here is an example :

import threading
from time import sleep
import os

def process_1():
    print('process_1 running')
    # calling your executable
    os.system("./path_to_process_1/process_1.sh")
    # simultating the executable time
    sleep(5)
    
def process_2():
    print('process_2 running')
    os.system("./path_to_process_2/process_2.sh")
    sleep(5)

def process_3():
    print('process_3 running')
    os.system("./path_to_process_3/process_3.sh")
    sleep(5)

def process_4():
    print('process_4 running')
    os.system('./path_to_process_4/process_4.sh')
    sleep(5)



if __name__ == '__main__':
    
    # storing the processes functions into phase 1 and 2
    phase_1_processes = [process_1, process_2]
    phase_2_processes = [process_3, process_4]
    

    print('starting phase 1 ...')
    # creating a job list in which we'll store the processes to be run
    jobs=[]
    for process in phase_1_processes:
        jobs.append(threading.Thread(target=process))

    # starting the jobs
    for job in jobs:
        job.start()

    # Wait for all the thread to finish
    for job in jobs:
        job.join()
    print('phase_1 is over ... ')
    
    
    # phase 2
    print('starting phase 2 ...')
    jobs=[]
    for process in phase_2_processes:
        jobs.append(threading.Thread(target=process))

    # starting the jobs
    for job in jobs:
        job.start()

    # Wait for all the thread to finish
    for job in jobs:
        job.join()
    print('phase_2 is over ... ')

This is multithreading and not multiprocessing, but I hope it will do the job for you, if you have a doubt on the difference between them you can look at this link.