How to convert sync and async recursive function to iteration in JavaScript

Question

Looking for a specific implementation for both sync and async recursive functions that could be used as a starting point to turn future recursive functions into flat iteration.<

Answer 1

There's a general way to translate recursive function to use an explicit stack instead: Simulate the way a compiler might handle recursive calls. Save all local state on a stack, change the argument values to what would have been passed to the recursive call, and jump to the top of the function. Then where the function returns, rather than merely returning, check the stack. If non-empty, pop state and jump to the place where the recursive call would have returned. Since javascript doesn't allow jumps (goto), it's then necessary to do algebra on the code to transform them to loops.

Start with recursive code that will be a bit easier to deal with than your original. This is just a classical recursive DFS with "visited" marks for objects (graph nodes) we've already searched and "current" marks for objects on the path from topmost (root) object to the current one. We'll discover all cycles if for each object reference (graph edge), we check whether the destination is marked "current."

The current marks are deleted along the way. The visited marks remain after the graph has been searched.

function get_back_refs(obj, back_refs) {
  if (obj && typeof obj == 'object' && !('__visited__' in obj)) {
    mark(obj, '__visited__')
    mark(obj, '__current__')
    var iter = getKeyIterator(obj)
    while (iter.hasNext()) {
      var key = iter.next()
      if ('__current__' in obj[key]) {
        back_refs.push([obj, obj[key]])
      } else {
        get_back_refs(obj[key], back_refs)
      }
    }
    unmark(obj, '__current__')
  }
}

var object = {
  a: {
    b: {
      c: {
        d: {
          e: 10,
          f: 11,
          g: 12
        }
      }
    }
  }
}
object.a.b.c.d.x = object
object.a.b.c.d.y = object.a.b

var id = 0

function mark(obj, name) {
  Object.defineProperty(obj, name, { value: ++id, configurable: true })
}

function unmark(obj, name) {
  delete obj[name]
}

function getKeyIterator(obj) {
  return {
    obj: obj,
    keys: Object.keys(obj).filter(k => obj[k] && typeof obj[k] == 'object'),
    i: 0,
    hasNext: function() { return this.i < this.keys.length },
    next: function() { return this.keys[this.i++] }
  }
}

var back_refs = []
get_back_refs(object, back_refs)
for (var i = 0; i < back_refs.length; ++i) {
  var pair = back_refs[i]
  console.log(pair[0].__visited__ + ', ' + pair[1].__visited__)
}

Note that I think this fixes bugs in your code. Since the hierarchy is a general directed graph, you want to avoid searching objects twice. Skipping this can easily lead to run time exponential in the graph size. Yet, shared structure in the graph doesn't necessarily mean there's a cycle. The graph could be a DAG - directed and acyclic.

In this case, the local state is contained nicely in the iterator, so that's all we need on the stack:

function get_back_refs2(obj, back_refs) {
  var stk = []
  var iter = null
 start:
  if (obj && typeof obj == 'object' && !('__visited__' in obj)) {
    mark(obj, '__visited__')
    mark(obj, '__current__')
    iter = getKeyIterator(obj)
    while (iter.hasNext()) {
      var key = iter.next()
      if ('__current__' in obj[key]) {
        back_refs.push([obj, obj[key]])
      } else {
        stk.push(iter) // Save state on stack.
        obj = obj[key] // Update parameter value.
        goto start     // Eliminated recursive call.          
       rtn:            // Where call would have returned.
      }
    }
    unmark(obj, '__current__')
  }
  if (stk.length == 0) return
  iter = stk.pop()  // Restore iterator from stack.
  obj = iter.obj    // Restore parameter value.
  goto rtn
}

Now to eliminate the gotos. This article describes a very similar transformation for searching a tree rather than a general graph, so I won't belabor it here. We end up with this intermediate result:

function get_back_refs2(obj, back_refs) {
  var stk = []
  var iter = null
  for (;;) {
    if (obj && typeof obj == 'object' && !('__visited__' in obj)) {
      mark(obj, '__visited__')
      mark(obj, '__current__')
      iter = getKeyIterator(obj)
      var key = null
      while (iter.hasNext()) {
        key = iter.next()
        if ('__current__' in obj[key]) back_refs.push([obj, obj[key]])
        else break
      }
      if (key) {
        stk.push(iter)
        obj = obj[key]
        continue           
      }
      unmark(obj, '__current__')
    }
    for(;;) {
      if (stk.length == 0) return
      iter = stk.pop()
      obj = iter.obj
      var key = null
      while (iter.hasNext()) {
        key = iter.next()
        if ('__current__' in obj[key]) back_refs.push([obj, obj[key]])
        else break
      }
      if (key) {
        stk.push(iter)
        obj = obj[key]
        break
      }           
      unmark(obj, '__current__')
    }
  }
}

Replacing gotos with the code they cause to execute results in repetition. But we can DRY that up with a shared local function:

function get_back_refs2(obj, back_refs) {
  var stk = []
  var iter = null
  var descend_to_next_child = function() {
    var key = null
    while (iter.hasNext()) {
      key = iter.next()
      if ('__current__' in obj[key]) back_refs.push([obj, obj[key]])
      else break
    }
    if (key) {
      stk.push(iter)
      obj = obj[key]
      return true           
    }
    unmark(obj, '__current__')
    return false
  }
  for (;;) {
    while (obj && typeof obj == 'object' && !('__visited__' in obj)) {
      mark(obj, '__visited__')
      mark(obj, '__current__')
      iter = getKeyIterator(obj)
      if (!descend_to_next_child()) break
    }
    for(;;) {
      if (stk.length == 0) return
      iter = stk.pop()
      obj = iter.obj
      if (descend_to_next_child()) break
    }
  }
}

Unless I made an algebra mistake, which is certainly possible, this is a drop-in replacement for the original recursive version.

Though the method doesn't involve reasoning beyond the code algebra, now that we're done it's pretty clear that the first loop descends into the graph, always to the first child it finds that isn't a back-reference, pushing iterators onto the stack as it goes. The second loop pops the stack looking for an iterator with work left to do: at least one child to search. When it finds one, it returns control to the first loop. This is exactly what the recursive version does, expressed in a different manner.

It would be fun to build a tool to do these transformations automatically.