Sticking with dplyr and tidyr (appreciate it's not as neat as other answers!):
df %>%
mutate(rowId = 1:n()) %>%
group_by(rowId) %>%
spread(left, right) %>% #gives you wide df with NAs
ungroup() %>%
select(-rowId) %>%
mutate(temp = rep(1:3, each = 4)) %>%
group_by(temp) %>%
summarise_each(funs(mean(., na.rm = TRUE))) %>%
ungroup() %>%
select(-temp)