[removed] Determine unknown array length and map dynamically

Question

Going to do my best at explaining what I am trying to do.

I have two models, mine and an api response I am receiving. When the items api response comes in, I need to map

Answer 1

Well, an interesting problem. Programmatically constructing nested objects from a property accessor string (or the reverse) isn't much of a problem, even doing so with multiple descriptors in parallel. Where it does get complicated are arrays, which require iteration; and that isn't as funny any more when it gets to different levels on setter and getter sides and multiple descriptor strings in parallel.

So first we need to distinguish the array levels of each accessor description in the script, and parse the text:

function parse(script) {
    return script.split(/\s*[;\r\n]+\s*/g).map(function(line) {
        var assignment = line.split(/\s*=\s*/);
        return assignment.length == 2 ? assignment : null; // console.warn ???
    }).filter(Boolean).map(function(as) {
        as = as.map(function(accessor) {
            var parts = accessor.split("[]").map(function(part) {
                return part.split(".");
            });
            for (var i=1; i 1) // getter contains array but setter does not
            as[0].unshift(["output"]); // implicitly return array (but better throw an error)
        return {setter:as[0], getter:as[1]};
    });
}

With that, the textual input can be made into a usable data structure, and now looks like this:

[{"setter":[["outputModel","items"],["item","name"]],
  "getter":[["items"],["item","name"]]},
 {"setter":[["outputModel","items"],["item","sku"]],
  "getter":[["items"],["item","skus"],["sku","num"]]}]

The getters already transform nicely into nested loops like

for (item of items)
    for (sku of item.skus)
        … sku.num …;

and that's exactly where we are going to. Each of those rules is relatively easy to process, copying properties on objects and iterating array for array, but here comes our most crucial issue: We have multiple rules. The basic solution when we deal with iterating multiple arrays is to create their cartesian product and this is indeed what we will need. However, we want to restrict this a lot - instead of creating every combination of all names and all nums in the input, we want to group them by the item that they come from.

To do so, we'll build some kind of prefix tree for our output structure that'll contain generators of objects, each of those recursivley being a tree for the respective output substructure again.

function multiGroupBy(arr, by) {
    return arr.reduce(function(res, x) {
        var p = by(x);
        (res[p] || (res[p] = [])).push(x);
        return res;
    }, {});
}
function group(rules) {
    var paths = multiGroupBy(rules, function(rule) {
        return rule.setter[0].slice(1).join(".");
    });
    var res = [];
    for (var path in paths) {
        var pathrules = paths[path],
            array = [];
        for (var i=0; i 1) // its an array
                array.push({
                    generator: rule.getter.slice(0, comb),
                    next: {
                        setter: rule.setter.slice(1),
                        getter: rule.getter.slice(comb)
                    }
                })
            else if (rule.getter.length == 1 && i==0)
                res.push({
                    set: rule.setter[0],
                    get: rule.getter[0]
                });
            else
                console.error("invalid:", rule);
        }
        if (array.length)
            res.push({
                set: pathrules[0].setter[0],
                cross: product(array)
            });
    }
    return res;
}
function product(pathsetters) {
    var groups = multiGroupBy(pathsetters, function(pathsetter) {
        return pathsetter.generator[0].slice(1).join(".");
    });
    var res = [];
    for (var genstart in groups) {
        var creators = groups[genstart],
            nexts = [],
            nests = [];
        for (var i=0; i

Now, our ruleset group(parse(script)) looks like this:

[{
    "set": ["outputModel","items"],
    "cross": [{
        "get": ["items"],
        "cross": [{
            "set": ["item","name"],
            "get": ["item","name"]
        }, {
            "get": ["item","skus"],
            "cross": [{
                "set": ["item","sku"],
                "get": ["sku","num"]
            }]
        }]
    }]
}]

and that is a structure we can actually work with, as it now clearly conveys the intention on how to match together all those nested arrays and the objects within them. Let's dynamically interpret this, building an output for a given input:

function transform(structure, input, output) {
    for (var i=0; i

And this does indeed work with

var result = transform(group(parse(script)), items); // your expected result

But we can do better, and much more performant:

function compile(structure) {
    function make(descriptor) {
        if (descriptor.get)
            return {inputName: descriptor.get[0], output: descriptor.get.join(".") };

        var outputName = descriptor.set[descriptor.set.length-1];
        var loops = descriptor.cross.reduce(function horror(next, descriptor) {
            if (descriptor.set)
                return function(it, cb) {
                    return next(it, function(res){
                        res.push(descriptor)
                        return cb(res);
                    });
                };
            else // its a crosser
                return function(it, cb) {
                    var arrName = descriptor.get[descriptor.get.length-1],
                        itName = String.fromCharCode(it);
                    var inner = descriptor.cross.reduce(horror, next)(it+1, cb);
                    return {
                        inputName: descriptor.get[0],
                        statement:  (descriptor.get.length>1 ? "var "+arrName+" = "+descriptor.get.join(".")+";\n" : "")+
                                    "for (var "+itName+" = 0; "+itName+" < "+arrName+".length; "+itName+"++) {\n"+
                                    "var "+inner.inputName+" = "+arrName+"["+itName+"];\n"+
                                    inner.statement+
                                    "}\n"
                    };
                };
        }, function(_, cb) {
            return cb([]);
        })(105, function(res) {
            var item = joinSetters(res);
            return {
                inputName: item.inputName,
                statement: (item.statement||"")+outputName+".push("+item.output+");\n"
            };
        });
        return {
            statement: "var "+outputName+" = [];\n"+loops.statement,
            output: outputName,
            inputName: loops.inputName
        };
    }
    function joinSetters(descriptors) {
        if (descriptors.length == 1 && descriptors[0].set.length == 1)
            return make(descriptors[0]);
        var paths = multiGroupBy(descriptors, function(d){ return d.set[1] || console.error("multiple assignments on "+d.set[0], d); });
        var statements = [],
            inputName;
        var props = Object.keys(paths).map(function(p) {
            var d = joinSetters(paths[p].map(function(d) {
                var names = d.set.slice(1);
                names[0] = d.set[0]+"_"+names[0];
                return {set:names, get:d.get, cross:d.cross};
            }));
            inputName = d.inputName;
            if (d.statement)
                statements.push(d.statement)
            return JSON.stringify(p) + ": " + d.output;
        });
        return {
            inputName: inputName,
            statement: statements.join(""),
            output: "{"+props.join(",")+"}"
        };
    }
    var code = joinSetters(structure);
    return new Function(code.inputName, code.statement+"return "+code.output+";");
}

So here is what you will get in the end:

> var example = compile(group(parse("outputModel.items[].name = items[].name;outputModel.items[].sku = items[].skus[].num;")))
function(items) {
    var outputModel_items = []; 
    for (var i = 0; i < items.length; i++) {
        var item = items[i];
        var skus = item.skus;
        for (var j = 0; j < skus.length; j++) {
            var sku = skus[j];
            outputModel_items.push({"name": item.name,"sku": sku.num});
        }
    }
    return {"items": outputModel_items};
}
> var flatten = compile(group(parse("as[]=bss[][]")))
function(bss) {
    var as = []; 
    for (var i = 0; i < bss.length; i++) {
        var bs = bss[i];
        for (var j = 0; j < bs.length; j++) {
            var b = bs[j];
            as.push(b);
        }
    }
    return as;
}
> var parallelRecords = compile(group(parse("x.as[]=y[].a; x.bs[]=y[].b")))
function(y) {
    var x_as = []; 
    for (var i = 0; i < y.length; i++) {
        var y = y[i];
        x_as.push(y.a);
    }
    var x_bs = []; 
    for (var i = 0; i < y.length; i++) {
        var y = y[i];
        x_bs.push(y.b);
    }
    return {"as": x_as,"bs": x_bs};
}

And now you can easily pass your input data to that dynamically created function and it will be transformed quite fast :-)