Multiprocessing concurrent.futures not update global dict and list

164 Views Asked by At

I need your help with multiprocessing to update global dict data. I use two functions, the first function is to do multiprocessing in the for loop. the second function is to update global dict data. after multiprocessing, all the data was not updated.

Here is my sample code:

import concurrent.futures
from multiprocessing import Process
from multiprocessing import Manager


list_key = ['A','B','C','D','E','F','G','H','I','K']

# first function. increase the value in Dict_value
def first(x,y):
    
    print('Call Increase Value Function')

    print(x)
    print(y[x])
    y[x] += 1
    print(y[x])

    print('Finished Increase Value')

# second function. square the value in the Dict_value
def second(x,y): 
    
    
    print('Call sqrt_dict_value function')

    print(y)
    for i in x:
        y[i] =  y[i]* y[i]
        print(y[i])
    print('Finished sqrt_dict_value')


def main():

    # declare the list and dict
    with Manager() as manager :
        dict_value = manager.dict()
        list_key = manager.list()
        list_key = ['A','B','C','D','E','F','G','H','I','K']
        dict_value = {'A':0,'B':1,'C':2,'D':3,'E':4,'F':5,'G':6,'H':7,'I':8,'K':9}

        # multiprocessing for first function.
        with concurrent.futures.ProcessPoolExecutor(max_workers=2) as executor:
            process = executor.submit(first,list_key,dict_value)

    # square the value in the dict after increase value in the first function.       
    second(list_key,dict_value)

if __name__ == '__main__':
    
    main()

My expected result is: 1 4 9 16 25 36 49 64 81 100

1

There are 1 best solutions below

0
vũ kiên On

Finally, I find the answer to my post. But I don't like this way. Because we need more return values in my sub-function and add For loop for re-collected data. I still looking for another way such as multiprocessing.Manager() to save all the data during the program perform multiprocess. Here is my new code for main():

def main():
time_start = time.time()
# declare the list and dict
with Manager() as manager :
    # dict_value = manager.dict()
    # list_key = manager.list()
    # list_key = ['A','B','C','D','E','F','G','H','I','K']
    # dict_value = {'A':0,'B':1,'C':2,'D':3,'E':4,'F':5,'G':6,'H':7,'I':8,'K':9}
    list_key = []
    dict_value = {}
    for i in range(1000):
        list_key.append(i)
        dict_value[i]=i
    # multiprocessing for first function.
    with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor:
        for i in list_key:
            process = executor.submit(first,i,dict_value)
            process_output.append(process)
        
    for result in process_output:
        if result._result[0] in dict_value:
            print(result._result[0])
            print(result._result[1])
            dict_value[result._result[0]]= result._result[1]
        else:
            dict_value.append(result._result)
    print(dict_value)
    print('Finished Increase Value')

# square the value in the dict after increase value in the first function.       
second(list_key,dict_value)
time_end = time.time()
print(time_end-time_start)
print('Finished')