Redirecting stdout with subprocess in python is very slow

Question

When I use subprocess in Python to redirect stdout, I get a very slow throughput. Am I doing it wrong?

Basically, I pipe the standard output of an external program to put it in a queue. Then in another function, I print it in the console.

Here is a sample code with hexdump to generate random output:

from subprocess import Popen, PIPE
from queue import Queue
import sys
from threading import Thread, Event
import threading

class Buffer(Queue):

    def __init__(self, *args, **kwargs):
        Queue.__init__(self, *args, **kwargs)

    def write(self, line):
        self.put_nowait(line)
        self.join()

    def read(self):
        element = self.get_nowait()
        self.task_done()
        return element

def write_output(buffer, stopped):

    hexdump = Popen(['hexdump', '-C', '/dev/urandom'], stdout=PIPE)
    while hexdump.returncode is None:
        for line in hexdump.stdout.readlines(8192):
            buffer.write(line)
            if stopped.is_set():
                hexdump.terminate()
                hexdump.wait()
                print('process terminated.')
                break

def read_output(buffer, stopped):
    while not stopped.is_set():
        while not buffer.empty():
            output = buffer.read()
            print('********* output: {}'.format(output))
            sys.stdout.flush()
    print('stopped')
    sys.stdout.flush()


buffer = Buffer()
stopped = Event()


generate_random_output = Thread(target=write_output, args=(buffer, stopped))
generate_random_output.name = 'generate_random_output'
generate_random_output.start()

process_output = Thread(target=read_output, args=(buffer, stopped))
process_output.name = 'process_output'
process_output.start()

try:
    while True:
        continue
except KeyboardInterrupt:
    stopped.set()
    generate_random_output.join()
    process_output.join()
    print('finished generating')
    print('finished processing')

I would appreciate any help.

Answer 1

Instead of redirecting your output to Queue - process it directly:

def write_output(buffer, stopped):

    hexdump = Popen(['hexdump', '-C', '/dev/urandom'], stdout=PIPE)
    while hexdump.poll() is None:
        while not stopped.is_set():
            for line in iter(hexdump.stdout.readline, b''):
                print('********* output: %s' % line.decode(), end='')
                sys.stdout.flush()

        hexdump.terminate()
        hexdump.wait()
        print('process terminated.')
        break