How to call the Scan variadic function using reflection

Question

I\'m looking to call the Rows.Scan() function using reflection. However it takes a variable number of pointers, but there are not a lot of source examples. I need to use r

Answer 1

Here's the solution that I've arrived at. It doesn't get the Types before traversing the data, and so doesn't know before hand the type of each value before pulling the values out through Scan(), but the point really is to not have to know the types before hand.

The trick was to create 2 slices, one for the values, and one that holds pointers in parallel to the values slice. Then once the pointers are used to fill data, the values array is actually filled with the data, which can then be used to populate other data structures.

package main

import (
    "fmt"
    _ "github.com/lib/pq"
    "database/sql"
)

func main() {
    db, _ := sql.Open(
        "postgres",
        "user=postgres dbname=go_testing password=pass sslmode=disable")

    rows, _ := db.Query("SELECT * FROM _user;")

    columns, _ := rows.Columns()
    count := len(columns)
    values := make([]interface{}, count)
    valuePtrs := make([]interface{}, count)

    for rows.Next() {
        for i := range columns {
            valuePtrs[i] = &values[i]
        }

        rows.Scan(valuePtrs...)

        for i, col := range columns {
            val := values[i]

            b, ok := val.([]byte)
            var v interface{}
            if (ok) {
                v = string(b)
            } else {
                v = val
            }

            fmt.Println(col, v)
        }
    }
}

Answer 2

To lucidquiet: you can also assign a interface instead of making a slice

The following code works good:

var sql = "select * from table"
rows, err := db.Query(sql)
columns, err = rows.Columns()
colNum := len(columns)

var values = make([]interface{}, colNum)
for i, _ := range values {
    var ii interface{}
    values[i] = &ii
}

for rows.Next() {
    err := rows.Scan(values...)
    for i, colName := range columns {
        var raw_value = *(values[i].(*interface{}))
        var raw_type = reflect.TypeOf(raw_value)

        fmt.Println(colName,raw_type,raw_value)
    }
}

Answer 3

I don't think you need reflection for this - you can use a slice and the ... operator to pass multiple values to a variadic function.

col := rows.Columns()
vals := make([]interface{}, col)
rows.Scan(vals...)

I may be misunderstanding what you want to do though!

Answer 4

The following solution allows you to refer to the field by field name instead of index. It's more like PHP style:

Table definition:

CREATE TABLE `salesOrder` (
  `idOrder` int(10) unsigned NOT NULL AUTO_INCREMENT,
  `uid` int(10) unsigned NOT NULL,
  `changed` datetime NOT NULL,
  PRIMARY KEY (`idOrder`)
) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8;

main.go:

package main

import (
        "database/sql"
        "encoding/json"
        "fmt"
        _ "github.com/go-sql-driver/mysql"
        "log"
        "reflect"
        "strings"
)

var (
        db *sql.DB
)

func initDB() {
        var err error

        // The database/sql package manages the connection pooling automatically for you.
        // sql.Open(..) returns a handle which represents a connection pool, not a single connection.
        // The database/sql package automatically opens a new connection if all connections in the pool are busy.
        // Reference: http://stackoverflow.com/questions/17376207/how-to-share-mysql-connection-between-http-goroutines
        db, err = sql.Open("mysql", "MyUser:MyPassword@tcp(localhost:3306)/MyDB")
        //db, err = sql.Open("mysql", "MyUser:MyPassword@tcp(localhost:3306)/MyDB?tx_isolation='READ-COMMITTED'") // optional

        if err != nil {
                log.Fatalf("Error on initializing database connection: %v", err.Error())
        }

        // Open doesn't open a connection. Validate DSN data:
        err = db.Ping()

        if err != nil {
                log.Fatalf("Error on opening database connection: %v", err.Error())
        }
}

func StrutToSliceOfFieldAddress(s interface{}) []interface{} {
        fieldArr := reflect.ValueOf(s).Elem()

        fieldAddrArr := make([]interface{}, fieldArr.NumField())

        for i := 0; i < fieldArr.NumField(); i++ {
                f := fieldArr.Field(i)
                fieldAddrArr[i] = f.Addr().Interface()
        }

        return fieldAddrArr
}

func testSelectMultipleRowsV3(optArr map[string]interface{}) {
        // queries
        query := []string{}
        param := []interface{}{}

        if val, ok := optArr["idOrder"]; ok {
                query = append(query, "salesOrder.idOrder >= ?")
                param = append(param, val)
        }

        // The first character of the field name must be in upper case. Otherwise, you would get:
        // panic: reflect.Value.Interface: cannot return value obtained from unexported field or method
        var sqlField = struct {
                IdOrder int
                Uid     int
                Changed string
        }{}

        var rowArr []interface{}

        sqlFieldArrPtr := StrutToSliceOfFieldAddress(&sqlField)

        sql := "SELECT "
        sql += "  salesOrder.idOrder "
        sql += ", salesOrder.uid "
        sql += ", salesOrder.changed "
        sql += "FROM salesOrder "
        sql += "WHERE " + strings.Join(query, " AND ") + " "
        sql += "ORDER BY salesOrder.idOrder "

        stmt, err := db.Prepare(sql)
        if err != nil {
                log.Printf("Error: %v", err)
        }
        defer stmt.Close()

        rows, err := stmt.Query(param...)

        if err != nil {
                log.Printf("Error: %v", err)
        }

        defer rows.Close()

        if err != nil {
                log.Printf("Error: %v", err)
        }

        //sqlFields, err := rows.Columns()

        for rows.Next() {
                err := rows.Scan(sqlFieldArrPtr...)

                if err != nil {
                        log.Printf("Error: %v", err)
                }

                // Show the type of each struct field
                f1 := reflect.TypeOf(sqlField.IdOrder)
                f2 := reflect.TypeOf(sqlField.Uid)
                f3 := reflect.TypeOf(sqlField.Changed)
                fmt.Printf("Type: %v\t%v\t%v\n", f1, f2, f3)

                // Show the value of each field
                fmt.Printf("Row: %v\t%v\t%v\n\n", sqlField.IdOrder, sqlField.Uid, sqlField.Changed)

                rowArr = append(rowArr, sqlField)
        }

        if err := rows.Err(); err != nil {
                log.Printf("Error: %v", err)
        }

        // produces neatly indented output
        if data, err := json.MarshalIndent(rowArr, "", " "); err != nil {
                log.Fatalf("JSON marshaling failed: %s", err)
        } else {
                fmt.Printf("json.MarshalIndent:\n%s\n\n", data)
        }
}

func main() {
        initDB()
        defer db.Close()

        // this example shows how to dynamically assign a list of field name to the rows.Scan() function.
        optArr := map[string]interface{}{}
        optArr["idOrder"] = 1
        testSelectMultipleRowsV3(optArr)
}

Sample output:

# go run main.go

Type: int       int     string
Row: 1  1       2016-05-06 20:41:06

Type: int       int     string
Row: 2  2       2016-05-06 20:41:35

json.MarshalIndent:
[
 {
  "IdOrder": 1,
  "Uid": 1,
  "Changed": "2016-05-06 20:41:06"
 },
 {
  "IdOrder": 2,
  "Uid": 2,
  "Changed": "2016-05-06 20:41:35"
 }
]