Difference in kqueue handling of fifos between Mac OS and FreeBSD?

Question

I'm working on an application that uses fifos for IPC and uses an event-notification API (such as epoll or kqueue) to monitor the fifos for data to be read.

The application expects that if the writer for a fifo terminates that the reader will receive an event via the event notification API, allowing the reader to notice that the writer terminated.

I'm currently porting this application to macos and I'm running into some odd behavior with kqueue. I've been able to create a reproducer of this behavior:

#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <fcntl.h>
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/event.h>
#include <sys/time.h>
#include <sys/errno.h>

static int child() {
    char child_fifo_path[64];
    char parent_fifo_path[64];

    printf("Child %d\n", getpid());

    sprintf(child_fifo_path, "/tmp/child-%d", getpid());
    sprintf(parent_fifo_path, "/tmp/parent-%d", getpid());

    mkfifo(child_fifo_path, 0644);
    mkfifo(parent_fifo_path, 0644);

    int parent_fd = open(parent_fifo_path, O_RDONLY);
    if (parent_fd == -1) {
        perror("open");
        return EXIT_FAILURE;
    }

    unsigned char parent_val;
    read(parent_fd, &parent_val, 1);

    printf("Received %hhx from parent\n", parent_val);

    int child_fd = open(child_fifo_path, O_WRONLY);
    if (child_fd == -1) {
        perror("open");
        return EXIT_FAILURE;
    }

    write(child_fd, &parent_val, 1);

    printf("Wrote %hhx to parent\n", parent_val);

    close(parent_fd);
    close(child_fd);
    return EXIT_SUCCESS;
}

static int parent(pid_t child_pid) {
    char child_fifo_path[64];
    char parent_fifo_path[64];

    printf("Parent %d\n", getpid());

    sprintf(child_fifo_path, "/tmp/child-%d", child_pid);
    sprintf(parent_fifo_path, "/tmp/parent-%d", child_pid);

    int result = -1;
    while (result == -1) {
        struct stat buf;
        result = stat(child_fifo_path, &buf);
        if (result == -1) {
            if (errno != ENOENT) {
                perror("open");
                return EXIT_FAILURE;
            }
        }
    }

    unsigned char val = 20;

    int parent_fd = open(parent_fifo_path, O_WRONLY);
    if (parent_fd == -1) {
        perror("open");
        return EXIT_FAILURE;
    }

    write(parent_fd, &val, 1);

    printf("Wrote %hhx to child\n", val);

    int child_fd = open(child_fifo_path, O_RDONLY);
    if (child_fd == -1) {
        perror("open");
        close(parent_fd);
        return EXIT_FAILURE;
    }

    int kq = kqueue();

    struct kevent event;
    EV_SET(&event, child_fd, EVFILT_READ, EV_ADD, 0, 0, 0);

    result = kevent(kq, &event, 1, NULL, 0, NULL);
    if (result == -1) {
        perror("kevent");
        close(child_fd);
        close(parent_fd);
        return EXIT_FAILURE;
    }

    int done = 0;
    while (!done) {
        memset(&event, 0, sizeof(event));

        printf("Waiting for events\n");

        result = kevent(kq, NULL, 0, &event, 1, NULL);
        if (result == -1) {
            perror("kevent");
            close(child_fd);
            close(parent_fd);
            return EXIT_FAILURE;
        }

        if (event.ident == child_fd) {
            if (event.flags & EV_EOF) {
                printf("Child exited\n");
                done = 1;
            }else if ( event.data > 0 ) {
                unsigned char child_val;

                result = read(child_fd, &child_val, 1);
                if (result == -1) {
                    perror("read");
                    return EXIT_FAILURE;
                }

                printf("Received %hhx from child\n", child_val);
            }
        }
    }

    return EXIT_SUCCESS;
}

int main(int argc, char *argv[]) {

    pid_t child_pid = fork();
    if (child_pid == -1) {
        perror("fork");
        return EXIT_FAILURE;
    }

    if (child_pid) {
        return parent(child_pid);
    } else {
        return child();
    }
}

This reproducer forks a child process, which creates 2 fifos: /tmp/parent-$CHILD_PID and /tmp/child-$CHILD_PID. The parent waits until /tmp/parent-$CHILD_PID is created and then writes a byte to it. The child opens /tmp/parent-$CHILD_PID and blocks to read the byte written by the parent. Once complete, the child goes to write that same byte to the parent via /tmp/child-$CHILD_PID. The parent uses kqueue to observe the write to /tmp/child-$CHILD_PID.

This sequence of events works fine.

The issue occurs when the child closes its file referring to /tmp/child-$CHILD_PID. I'm seeing that this event is not reported to the parent via kqueue.

The most interesting part: this code works as I would expect on FreeBSD.

Version info:

Mac OS X: 10.11.6

FreeBSD 10.4-RELEASE-p3

Is there a difference between kqueue on macos and FreeBSD in this context? If so, is there some documentation that documents this difference?

Answer 1

This is not the best answer to your question but I hope can help you find other differences that may influence your code behavior when using kqueue between macOS and FreeBSD

In my case I use kqueue EVFILT_VNODE to check for changes, but based on the operating system I need to define different flags openModeDir when using the syscall.Open

For macOS (openmode_darwin.go) I use this:

openModeDir  = syscall.O_EVTONLY | syscall.O_DIRECTORY
openModeFile = syscall.O_EVTONLY

And for FreeBSD (openmode.go) I use:

openModeDir  = syscall.O_NONBLOCK | syscall.O_RDONLY | syscall.O_DIRECTORY
openModeFile = syscall.O_NONBLOCK | syscall.O_RDONLY

From macOS docs open(2), this is the flag description:

O_EVTONLY       descriptor requested for event notifications only

And from FreeBSD open(2), there is no O_EVTONLY.

Putting all together this is how I call kqueue:

...
watchfd, err := syscall.Open(dir, openModeDir, 0700)
if err != nil {
    return err
}

kq, err := syscall.Kqueue()
if err != nil {
    syscall.Close(watchfd)
    return err
}

ev1 := syscall.Kevent_t{
    Ident:  uint64(watchfd),
    Filter: syscall.EVFILT_VNODE,
    Flags:  syscall.EV_ADD | syscall.EV_ENABLE | syscall.EV_CLEAR,
    Fflags: syscall.NOTE_WRITE | syscall.NOTE_ATTRIB,
    Data:   0,
}
...

I am using go, but as mentioned before hope can give you an idea while dealing with Kqueue, In my case this simple change of flags made a difference.