How to get unbuffered output from popen & fgets

Question

I'm using popen to execute a command and read the output. I'm setting the file descriptor to non-blocking mode so that I can put in my own timeout, as follows:

    auto stream = popen(cmd.c_str(), "r");

    int fd = fileno(stream);
    int flags = fcntl(fd, F_GETFL, 0);
    flags |= O_NONBLOCK;
    fcntl(fd, F_SETFL, flags);

    while(!feof(stream)) {
        if(fgets(buffer, MAX_BUF, stream) != NULL) {
            // do something with buffer...
        }
        sleep(10);
    }
    pclose(stream);

This works just fine, except that fgets keeps returning NULL, until the program has finished executing, at which time it returns all the output as expected.

In other words, even if the program immediately outputs some text and a newline to the stdout, my loop doesn't read it immediately; it only sees it later.

In the documentation for popen I see:

Note that output popen() streams are block buffered by default.

I've tried a few things to turn off buffering (ex. setvbuf(stream, NULL, _IONBF, 0)) , but so far no luck.

How do I turn off buffering so that I can read the output in real-time?

Thank you!

Answer 1

A solution based on something like select() would be more accurate and flexible. Try this :

    #include <stdio.h>
    #include <unistd.h>
    #include <errno.h>
    #include <sys/select.h>


    void read_cmd(const char *cmd)
    {
    FILE *stream;
    int fd;
    int flags;
    char buffer[1024];
    fd_set fdset;
    struct timeval timeout;
    int rc;
    int eof;

      stream = popen(cmd, "r");

      fd = fileno(stream);

      eof = 0;
      while(!eof) {

        timeout.tv_sec = 10; // 10 seconds
        timeout.tv_usec = 0;

        FD_ZERO(&fdset);
        FD_SET(fd, &fdset);

        rc = select(fd + 1, &fdset, 0, 0, &timeout);

        switch(rc) {

          case -1: {
            // Error
            if (errno != EINTR) {
              fprintf(stderr, "select(): error '%m' (%d)\n", errno);
            }
            return;
          }
          break;

          case 0: {
            // Timeout
            printf("Timeout\n");
          }
          break;

          case 1: {
            // Something to read
            rc = read(fd, buffer, sizeof(buffer) - 1);
            if (rc > 0) {
              buffer[rc] = '\0';
              printf("%s", buffer);
              fflush(stdout);
            }

            if (rc < 0) {
              fprintf(stderr, "read(): error '%m' (%d)\n", errno);
              eof = 1;
            }

            if (0 == rc) {
              // End of file
              eof = 1;
            }

          }
          break;

        } // End switch

      } // End while

      pclose(stream);

    }

    int main(int ac, char *av[])
    {

      read_cmd(av[1]);

      return 0;

    } // main