pass object of selenium from a subprocess to another subprocess

136 Views Asked by At

I try to use manager and queue to share the object of selenium in two subprocess, but it shows me the error "AttributeError: Can't pickle local object '_createenviron..encodekey'"

from selenium.webdriver import Chrome
from multiprocessing import Queue, Manager, Process
import time

def find(q):
    driver = Chrome()
    driver.get('https://querycourse.ntust.edu.tw/querycourse/api/courses')
    q.put(driver)

def refresh(q):
    driver = q.get()
    while True:
        time.sleep(9)
        driver.refresh()

if __name__=='__main__':
    with Manager() as manager:
        q = manager.Queue()
        p1 = Process(target=find, args=(q,))
        p2 = Process(target=refresh, args=(q,))
        
        p1.start()
        time.sleep(3)
        p2.start()

        p1.join()
        p2.join()
Process Process-2:
Traceback (most recent call last):
  File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\process.py", line 314, in _bootstrap
    self.run()
  File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\process.py", line 108, in run
    self._target(*self._args, **self._kwargs)
  File "D:\python_training\web crawler\kk_manager_queue.py", line 8, in find
    q.put(driver)
  File "<string>", line 2, in put
  File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\managers.py", line 817, in _callmethod
    conn.send((self._id, methodname, args, kwds))
  File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\connection.py", line 211, in send
    self._send_bytes(_ForkingPickler.dumps(obj))
  File "C:\Users\USER\AppData\Local\Programs\Python\Python310\lib\multiprocessing\reduction.py", line 51, in dumps
    cls(buf, protocol).dump(obj)
AttributeError: Can't pickle local object '_createenviron.<locals>.encodekey'

Does any one know what's wrong with my code? Any feetbeck are appreciated.

I simplified the code, I originally expected to use the code grabbing the course I want. One subprocess is to refresh the course selecting system to prevent Connect Timeouts, the other is to detect if someone drops the course and then click the “select” button, so the two subprocesses need to share the object “driver”.

This is my original code:

import urllib.request as req
import json
from selenium import webdriver
from selenium.webdriver import Chrome
from selenium.webdriver.common.by import By
import time
from selenium.webdriver.support.wait import WebDriverWait
import time
from multiprocessing import Manager, Queue, Process

def find(url, requestData, chooseStudent, q):
    while True:
        request=req.Request(url, headers={
            "content-type":"application/json; charset=utf-8"
        }, data=json.dumps(requestData).encode("utf-8"))

        with req.urlopen(request) as response:
            result=response.read().decode("utf-8")

        result=json.loads(result)

        if int(result[0]["ChooseStudent"])<chooseStudent:
            driver = q.get()
            ADDgo = driver.find_element(By.ID, "SingleAdd")
            ADDgo.click()
            break

def refresh(q):
    account='XXXXXXXX'
    password='XXXXXXXX'
    classid='GE3710302'

    driver = Chrome()
    driver.get("https://stuinfosys.ntust.edu.tw/NTUSTSSOServ/SSO/Login/CourseSelection")
    UserName = WebDriverWait(driver, timeout=10).until(lambda d: d.find_element(By.NAME,"UserName"))
    UserName.send_keys(account)
    Password = driver.find_element(By.NAME, "Password")
    Password.send_keys(password)
    btnLogIn = driver.find_element(By.NAME, "btnLogIn")
    btnLogIn.click()
    q.put(driver)
    while True:
        time.sleep(10)
        driver.refresh()

if __name__=='__main__':
    url="https://querycourse.ntust.edu.tw/querycourse/api/courses"
    requestData={"Semester":"1112","CourseNo":"GE3710302","CourseName":"","CourseTeacher":"","Dimension":"","CourseNotes":"","ForeignLanguage":0,"OnlyGeneral":0,"OnleyNTUST":0,"OnlyMaster":0,"OnlyUnderGraduate":0,"OnlyNode":0,"Language":"zh"}
    chooseStudent=51
    with Manager() as manager:
        q = manager.Queue()
        p1 = Process(target=find, args=(url, requestData, chooseStudent, q,))
        p2 = Process(target=refresh, args=(q,))

        p1.start()
        p2.start()

        p1.join()
        p2.join()
0

There are 0 best solutions below