Recursivly traverse and flatten JSON object in DataWeave

问题

I want to traverse and flatten a big JSON file that has following structure showing a product hierarchie (think of it as navigation in an online shop):

productGroups: [
    {
        "key": "child 1"
        ...
        "childrenProductGroups": [
            {
                "key": "child 1.1",
                ...,
                "childrenProductGroups": []
            },
            {
                "key": "child 1.2"
                ...
                "childrenProductGroups": [
                    {
                        "key": "child 1.2.1",
                        ...,
                        "childrenProductGroups": [
                            {
                                "key": "child 1.2.1.1",
                                ...,
                                childrenProductGroups": [
                                    ...
                                ]
                            }
                        ]
                    },
                    {
                        "key": "child 1.2.2",
                        ...,
                        "childrenProductGroups": []
                    }
                ]
                
            },
            {
                "key": "child 1.3",
                ...,
                "childrenProductGroups": [
                    ...
                ]
            }
        ]
    }, 
    {
        "key": "child 2",
        ...,
        "childrenProductGroups": [
            ...
        ]
    },
    {
        "key": "child 3",
        ...,
        "childrenProductGroups": [
            ...
        ]
    }
]

And I want to flatten them in a format like this:

{
    "hierarchieSet": [
        {
            "Nodeid": "00000001", # Number in this json
            "Nodename": "child 1",
            "Tlevel": "01", # First child of product group
            "Parentid": "00000000", # Parent is null
            "Childid": "00000002", # Child node number
            "Nextid": "00000008" # Node number on the same level (child 2)
        }, 
        {
            "Nodeid": "00000002",
            "Nodename": "child 1.1",
            "Tlevel": "02",
            "Parentid": "00000001",
            "Childid": "00000003",
            "Nextid": "00000003"
        }, 
        {
            "Nodeid": "00000003",
            "Nodename": "child 1.2",
            "Tlevel": "02",
            "Parentid": "00000002",
            "Childid": "00000005",
            "Nextid": "00000007"
        }, 
        {
            "Nodeid": "00000004",
            "Nodename": "child 1.2.1",
            "Tlevel": "03",
            "Parentid": "00000003",
            "Childid": "0000005",
            "Nextid": "00000006"
        }
        , 
        {
            "Nodeid": "00000005",
            "Nodename": "child 1.2.1.1",
            "Tlevel": "04",
            "Parentid": "00000004",
            "Childid": "0000000", #No more children
            "Nextid": "00000000"
        }, 
        {
            "Nodeid": "00000006",
            "Nodename": "child 1.2.2",
            "Tlevel": "03",
            "Parentid": "00000003",
            "Childid": "0000000",
            "Nextid": "00000000"
        }, 
        {
            "Nodeid": "00000007",
            "Nodename": "child 1.3",
            "Tlevel": "02",
            "Parentid": "00000001",
            "Childid": "0000000",
            "Nextid": "00000000"
        }, 
        {
            "Nodeid": "00000008",
            "Nodename": "child 2",
            "Tlevel": "01",
            "Parentid": "00000000",
            "Childid": "0000009", # 00000009 not shown
            "Nextid": "00000014" # 
        }, 
        ...
        {
            "Nodeid": "000000014",
            "Nodename": "child 3",
            "Tlevel": "01",
            "Parentid": "00000000",
            "Childid": "00000015",
            "Nextid": "00000000" # 00000010 does not exist
        }
    ]
}

Thus I have identified some main concerns:

Recursion of the tree structure
Transforming the elements
Flattening the structure
Keeping track of parents, siblings and children
Keeping track of recursion level
Formatting numbers

I tried to solve this issue by 2 different approaches:

Use DataWeave to transform all elements
Use Java to traverse the structure

As I'm fairly new to functional programming I put more focus on the Java implementation but ran into a number of issues.

Java approach

Read json > Init Tree var and assign the Java instance > for each element in top-level array invoke traverse(data, level) in Tree.java.
Tree.java:

import org.json.JSONObject;

public class Tree {
    private int id = 0;
    private List<Node> nodes = new ArrayList<Node>();
    
    public Tree() {
        nodes.add(new Node("01", "00000001", "HOME", "01", "00000000", "00000002", "00000000"));
    }
    
    public void traverse(String data, int level) {
        System.out.println(data);
        // TODO parse json
    }
    
    private void visit(JSONObject parent, JSONObject node, int level) {
        id++;
        nodes.add(new Node("01", String.valueOf(id), node.getString("key"), String.valueOf(level), "", "", ""));
    }
    
    public List<Node> getNodes() {
        return nodes;
    }

    private static class Node {
        private String zshop, nodename, parentid, childid, nextid, nodeid, tlevel;
        
        public Node(String zshop, String nodeid, String nodename, String tlevel, String parentid, String childid, String nextid) {
            this.zshop = zshop;
            this.nodeid = nodeid;
            this.nodename = nodename;
            this.tlevel = tlevel;
            this.parentid = parentid;
            this.childid = childid;
            this.nextid = nextid;
        }
    }
}

When calling the invoke action I use this payload:

%dw 2.0
output application/java
---
{
    data: vars.rootMessage.payload as String,
    level: 1
}

But this yields following error:

"Cannot coerce Object { encoding: UTF-8, mediaType: text/json; charset=UTF-8, mimeType: text/json, raw: org.mule.weave.v2.el.SeekableCursorStream@50ecee52 } (org.mule.weave.v2.el.MuleTypedValue@511ba9cc) to String

5| data: vars.rootMessage.payload as String, ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Trace: at main (line: 5, column: 7)" evaluating expression: "%dw 2.0 output application/java --- { data: vars.rootMessage.payload as String, level: 1 }".

I tried a number of things:

Cast it to a ProductGroup object I wrote in Java
Try to cast the object retrieved to org.json.JSONObject
Try to buffer and read vars.rootMessage.payload (Binary)

But I wasn't able to solve it with any of these issues.

DataWeave approach My .dw Script

%dw 2.0
fun append
(item: Object, acc: Object = {
}) = acc ++ item

fun mapper(item: Object) = 
{
    Zshop: "01",
    Nodeid: "00000000",
    Nodename: item.key as Number as String {format: ""},
    Tlevel: "02",
    Parentid: "00000000",
    Childid: "00000000",
    Nextid: "00000000"
}
    
fun traverse(a: Array, level: Number) = 
    a map $ flatMap(value, index) -> value
    
output application/json
---
{
    test: payload.productGroups reduce (item, acc) -> append(mapper(item), acc)
}

Where I tried to solve some of the problems. mapper(item) should create json objects that I can append to the final output with appender(item, acc). Recursion has been sketched, but is not my main concern yet.

This yields this result:

(original payload),
"Zshop": "01",
"Nodeid": "00000000",
"Nodename": "800",
"Tlevel": "02",
"Parentid": "00000000",
"Childid": "00000000",
"Nextid": "00000000",
"Zshop": "01",
"Nodeid": "00000000",
"Nodename": "110",
"Tlevel": "02",
"Parentid": "00000000",
"Childid": "00000000",
"Nextid": "00000000",
"Zshop": "01",
"Nodeid": "00000000",
"Nodename": "720",
"Tlevel": "02",
"Parentid": "00000000",
"Childid": "00000000",
"Nextid": "00000000",
"Zshop": "01",
"Nodeid": "00000000",
"Nodename": "710",
"Tlevel": "02",
"Parentid": "00000000",
"Childid": "00000000",
"Nextid": "00000000",
...

Where I wonder why I'm getting a flat result without any object structure.

My questions:

Java: Why can't I cast the String or how is it done properly
DataWeave: Is there an easy solution I don't see?
Why is it a flat result and not an object?
Are the usages of the reduce and flatMap functions correct for this purpose?

Any help and / or feedback is welcome.

回答1:

Java: Why can't I cast the String or how is it done properly

JSON is not String. Use write(payload,'application/json') to have String.

DataWeave: Is there an easy solution I don't see?

Just pass the object, It is Map in Java. Since it is tree - each branch is another Map inside this Map.

Why is it a flat result and not an object?

It is ALWAYS Object. There are no other things in Java world.

Are the usages of the reduce and flatMap functions correct for this purpose?

No. mapObject and recursion should be good approach.

回答2:

Basen on the help I received I could craft this solution:

public class Tree implements Serializable {
    private int id = 0;
    private List<Node> nodes = new ArrayList<Node>();
    
    public Tree() {
        nodes.add(new Node("01", "00000001", "HOME", "0", "00000000", "00000002", "00000000"));
    }
    
    public void enter(String jsonString, Integer level) {
        JSONObject json = new JSONObject(jsonString);
        traverse(json, level);
    }
    
    public void traverse(JSONObject json, int level) {
        visit(json, level);
        
        JSONArray arr = json.getJSONArray("childProductGroups");
        if(arr != null) {
            for(int i = 0; i < arr.length(); i++) {
                traverse(arr.getJSONObject(i), level + 1);
            }
        }       
    }
    
    private void visit(JSONObject object, int level) {
        id++;
        nodes.add(new Node("01", String.valueOf(id), object.getString("name_de"), String.valueOf(level), "", "", ""));
    }
    
    public Node[] getNodes() {
        assignParentIds();
        assignNextIds();
        assignChildIds();
            
        Node[] nodeArr = new Node[nodes.size()];
        for(int i = 0; i < nodes.size(); i++) {
            nodeArr[i] = nodes.get(i);
        }
        return nodeArr;
    }
    
    private void assignParentIds() {
        Map<String, Node> lastNodesWithHigherLevel = new HashMap<String, Node>();
        for(Node node : nodes) {
            Node higher = lastNodesWithHigherLevel.get(String.valueOf(Integer.valueOf(node.tlevel) - 1));
            if(higher != null) {
                node.parentid = higher.nodeid;
            }   
            lastNodesWithHigherLevel.put(node.tlevel, node);
        }
    }
    
    private void assignNextIds() {
        Map<String, Node> lastNodeOnSameLevel = new HashMap<String, Node>();
        for(Node node : nodes) {
            Node last = lastNodeOnSameLevel.get(node.tlevel);
            if(last != null) {
                if(last.parentid.equals(node.parentid)) {
                    // If the last and this node have the same parent
                    last.nextid = node.nodeid;
                }
            }   
            lastNodeOnSameLevel.put(node.tlevel, node);
        }
    }

    private void assignChildIds() {
        Iterator<Node> parentIterator = nodes.iterator();
        Iterator<Node> childIterator = nodes.iterator();
        
        // Init child iterator one further
        if(childIterator.hasNext()) {
            childIterator.next();
        }
        
        do {
            Node parent = parentIterator.next();
            Node child = childIterator.next();
            // If level of parent is higher (<) set the value
            if(Integer.valueOf(parent.tlevel) < Integer.valueOf(child.tlevel)) {
                parent.childid = child.nodeid;
            }
        } while (childIterator.hasNext());
        // Because childIterator will first be done, but parentIterator might need one more reference, we need do while
    }
    
    private static class Node implements Serializable {
        public String zshop, nodename, parentid, childid, nextid, nodeid, tlevel;
        
        public Node(String zshop, String nodeid, String nodename, String tlevel, String parentid, String childid, String nextid) {
            this.zshop = zshop;
            this.nodeid = nodeid;
            this.nodename = nodename;
            this.tlevel = tlevel;
            this.parentid = parentid;
            this.childid = childid;
            this.nextid = nextid;
        }
        
        @Override
        public String toString() {
            return "{zshop:" + zshop + ", nodename:" + nodename + ", parentid:" + parentid + ", childid:" + childid
                    + ", nextid:" + nextid + ", nodeid:" + nodeid + ", tlevel:" + tlevel + "}";
        }
    }
}

And the flow config:

<?xml version="1.0" encoding="UTF-8"?>

<mule xmlns:java="http://www.mulesoft.org/schema/mule/java" xmlns:ee="http://www.mulesoft.org/schema/mule/ee/core"
    xmlns:file="http://www.mulesoft.org/schema/mule/file"
    xmlns:http="http://www.mulesoft.org/schema/mule/http" xmlns="http://www.mulesoft.org/schema/mule/core" xmlns:doc="http://www.mulesoft.org/schema/mule/documentation" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mulesoft.org/schema/mule/core http://www.mulesoft.org/schema/mule/core/current/mule.xsd
http://www.mulesoft.org/schema/mule/http http://www.mulesoft.org/schema/mule/http/current/mule-http.xsd
http://www.mulesoft.org/schema/mule/file http://www.mulesoft.org/schema/mule/file/current/mule-file.xsd
http://www.mulesoft.org/schema/mule/ee/core http://www.mulesoft.org/schema/mule/ee/core/current/mule-ee.xsd
http://www.mulesoft.org/schema/mule/java http://www.mulesoft.org/schema/mule/java/current/mule-java.xsd">
    <http:listener-config name="HTTP_Listener_config" doc:name="HTTP Listener config" doc:id="e31dc867-3f9f-457b-8013-cd50a74c0af1" >
        <http:listener-connection host="0.0.0.0" port="8081" />
    </http:listener-config>
    <file:config name="File_Config" doc:name="File Config" doc:id="3279487e-61ca-4845-89a2-5332e7c02638" />
    <flow name="java-flow" doc:id="3c036218-560d-44f0-aef5-0c7f0ef4d776" >
        <http:listener doc:name="Listener" doc:id="265a1691-4f9b-40e2-a280-592daf719002" config-ref="HTTP_Listener_config" path="java">
            <http:response >
                <http:body ><![CDATA[#[output application/json ---
payload]]]></http:body>
            </http:response>
        </http:listener>
        <file:read doc:name="Copy_of_Read" doc:id="9f8b3c6f-f674-45f7-80fe-756f64602b30" config-ref="File_Config" path="response.json" />
        <set-variable doc:name="Set Variable" doc:id="ecff267b-3b73-45d1-924e-227a01a99e4e" variableName="Tree" value="#[null]" />
        <java:new doc:name="New Tree" doc:id="764d60bf-da41-4f17-8e13-c8bf31f141dc" class="valion.Tree" constructor="Tree()" target="Tree" />
        <foreach doc:name="For Each" doc:id="e612fd8b-636e-41c2-b603-e6861514306b" collection="#[payload.productGroups]">
            <java:invoke doc:name="Traverse" doc:id="96da4743-00d3-4970-a9e5-712877bcf2a9" class="valion.Tree" instance="#[vars.Tree]" method="enter(java.lang.String,java.lang.Integer)">
                <java:args><![CDATA[#[%dw 2.0
output application/java
---
{
    jsonString: write( payload,'application/json'),
    level: 1
}]]]></java:args>
            </java:invoke>
        </foreach>
        <java:invoke doc:name="Get Nodes" doc:id="999d5f4c-80b4-4793-9188-e17725ad030b" instance="#[vars.Tree]" class="valion.Tree" method="getNodes()">
        </java:invoke>
        <ee:transform doc:name="Transform Message" doc:id="b078bef3-45aa-4821-b7d1-720c2c7d0580">
            <ee:message>
                <ee:set-payload><![CDATA[%dw 2.0
output application/json
---
{
    hierarchieSet: payload map ( item , index ) -> {
        zshop: item.zshop,
        nodeid: item.nodeid,
        nodename: item.nodename,
        tlevel: item.tlevel,
        parentid: item.parentid,
        childid: item.childid,
        nextid: item.nextid
    }
}]]></ee:set-payload>
            </ee:message>
        </ee:transform>
    </flow>
</mule>

来源：https://stackoverflow.com/questions/64908969/recursivly-traverse-and-flatten-json-object-in-dataweave

标签

java

json

transformation

dataweave

mulesoft