1. 技术文章
  2. Openvswitch原理与代码分析(3): openvswitch内核模块的加载

Openvswitch原理与代码分析(3): openvswitch内核模块的加载

由 穿精又带淫゛_ 提交于 2019-11-28 07:21:34

 

 

上一节我们讲了ovs-vswitchd，其中虚拟网桥初始化的时候，对调用内核模块来添加虚拟网卡。

 

我们从openvswitch内核模块的加载过程，来看这个过程。

 

在datapath/datapath.c中会调用module_init(dp_init);来初始化内核模块。

 

  1. static int __init dp_init(void)
  2. {
  3.    int err;
  4.  
  5.    BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > FIELD_SIZEOF(struct sk_buff, cb));
  6.  
  7.    pr_info("Open vSwitch switching datapath %s\n", VERSION);
  8.  
  9.    err = compat_init();
  10.    if (err)
  11.       goto error;
  12.  
  13.    err = action_fifos_init();
  14.    if (err)
  15.       goto error_compat_exit;
  16.  
  17.    err = ovs_internal_dev_rtnl_link_register();
  18.    if (err)
  19.       goto error_action_fifos_exit;
  20.  
  21.    err = ovs_flow_init();
  22.    if (err)
  23.       goto error_unreg_rtnl_link;
  24.  
  25.    err = ovs_vport_init();
  26.    if (err)
  27.       goto error_flow_exit;
  28.  
  29.    err = register_pernet_device(&ovs_net_ops);
  30.    if (err)
  31.       goto error_vport_exit;
  32.  
  33.    err = register_netdevice_notifier(&ovs_dp_device_notifier);
  34.    if (err)
  35.       goto error_netns_exit;
  36.  
  37.    err = ovs_netdev_init();
  38.    if (err)
  39.       goto error_unreg_notifier;
  40.  
  41.    err = dp_register_genl();
  42.    if (err < 0)
  43.       goto error_unreg_netdev;
  44.  
  45.    return 0;
  46.  
  47. error_unreg_netdev:
  48.    ovs_netdev_exit();
  49. error_unreg_notifier:
  50.    unregister_netdevice_notifier(&ovs_dp_device_notifier);
  51. error_netns_exit:
  52.    unregister_pernet_device(&ovs_net_ops);
  53. error_vport_exit:
  54.    ovs_vport_exit();
  55. error_flow_exit:
  56.    ovs_flow_exit();
  57. error_unreg_rtnl_link:
  58.    ovs_internal_dev_rtnl_link_unregister();
  59. error_action_fifos_exit:
  60.    action_fifos_exit();
  61. error_compat_exit:
  62.    compat_exit();
  63. error:
  64.    return err;
  65. }

 

其中比较重要的是调用了dp_register_genl()，这个就是注册netlink函数，从而ovs-vswitchd可以通过netlink调用内核。

 

  1. static int dp_register_genl(void)
  2. {
  3.    int err;
  4.    int i;
  5.  
  6.    for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
  7.  
  8.       err = genl_register_family(dp_genl_families[i]);
  9.       if (err)
  10.          goto error;
  11.    }
  12.  
  13.    return 0;
  14.  
  15. error:
  16.    dp_unregister_genl(i);
  17.    return err;
  18. }

 

这里dp_genl_families由四个netlink的family组成

 

  1. static struct genl_family *dp_genl_families[] = {
  2.    &dp_datapath_genl_family,
  3.    &dp_vport_genl_family,
  4.    &dp_flow_genl_family,
  5.    &dp_packet_genl_family,
  6. };

 

其中分别定义了以下的操作：

 

Family名称

.name

.ops

dp_datapath_genl_family
  1. static struct genl_family dp_datapath_genl_family = {
  2.    .id = GENL_ID_GENERATE,
  3.    .hdrsize = sizeof(struct ovs_header),
  4.    .name = OVS_DATAPATH_FAMILY,
  5.    .version = OVS_DATAPATH_VERSION,
  6.    .maxattr = OVS_DP_ATTR_MAX,
  7.    .netnsok = true,
  8.    .parallel_ops = true,
  9.    .ops = dp_datapath_genl_ops,
  10.    .n_ops = ARRAY_SIZE(dp_datapath_genl_ops),
  11.    .mcgrps = &ovs_dp_datapath_multicast_group,
  12.    .n_mcgrps = 1,
  13. };
  1. static struct genl_ops dp_datapath_genl_ops[] = {
  2.    { .cmd = OVS_DP_CMD_NEW,
  3.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  4.      .policy = datapath_policy,
  5.      .doit = ovs_dp_cmd_new
  6.    },
  7.    { .cmd = OVS_DP_CMD_DEL,
  8.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  9.      .policy = datapath_policy,
  10.      .doit = ovs_dp_cmd_del
  11.    },
  12.    { .cmd = OVS_DP_CMD_GET,
  13.      .flags = 0, /* OK for unprivileged users. */
  14.      .policy = datapath_policy,
  15.      .doit = ovs_dp_cmd_get,
  16.      .dumpit = ovs_dp_cmd_dump
  17.    },
  18.    { .cmd = OVS_DP_CMD_SET,
  19.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  20.      .policy = datapath_policy,
  21.      .doit = ovs_dp_cmd_set,
  22.    },
  23. };

dp_vport_genl_family
  1. struct genl_family dp_vport_genl_family = {
  2.    .id = GENL_ID_GENERATE,
  3.    .hdrsize = sizeof(struct ovs_header),
  4.    .name = OVS_VPORT_FAMILY,
  5.    .version = OVS_VPORT_VERSION,
  6.    .maxattr = OVS_VPORT_ATTR_MAX,
  7.    .netnsok = true,
  8.    .parallel_ops = true,
  9.    .ops = dp_vport_genl_ops,
  10.    .n_ops = ARRAY_SIZE(dp_vport_genl_ops),
  11.    .mcgrps = &ovs_dp_vport_multicast_group,
  12.    .n_mcgrps = 1,
  13. };
  1. static struct genl_ops dp_vport_genl_ops[] = {
  2.    { .cmd = OVS_VPORT_CMD_NEW,
  3.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  4.      .policy = vport_policy,
  5.      .doit = ovs_vport_cmd_new
  6.    },
  7.    { .cmd = OVS_VPORT_CMD_DEL,
  8.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  9.      .policy = vport_policy,
  10.      .doit = ovs_vport_cmd_del
  11.    },
  12.    { .cmd = OVS_VPORT_CMD_GET,
  13.      .flags = 0, /* OK for unprivileged users. */
  14.      .policy = vport_policy,
  15.      .doit = ovs_vport_cmd_get,
  16.      .dumpit = ovs_vport_cmd_dump
  17.    },
  18.    { .cmd = OVS_VPORT_CMD_SET,
  19.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  20.      .policy = vport_policy,
  21.      .doit = ovs_vport_cmd_set,
  22.    },
  23. };

dp_flow_genl_family
  1. static struct genl_family dp_flow_genl_family = {
  2.    .id = GENL_ID_GENERATE,
  3.    .hdrsize = sizeof(struct ovs_header),
  4.    .name = OVS_FLOW_FAMILY,
  5.    .version = OVS_FLOW_VERSION,
  6.    .maxattr = OVS_FLOW_ATTR_MAX,
  7.    .netnsok = true,
  8.    .parallel_ops = true,
  9.    .ops = dp_flow_genl_ops,
  10.    .n_ops = ARRAY_SIZE(dp_flow_genl_ops),
  11.    .mcgrps = &ovs_dp_flow_multicast_group,
  12.    .n_mcgrps = 1,
  13. };
  1. static struct genl_ops dp_flow_genl_ops[] = {
  2.    { .cmd = OVS_FLOW_CMD_NEW,
  3.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  4.      .policy = flow_policy,
  5.      .doit = ovs_flow_cmd_new
  6.    },
  7.    { .cmd = OVS_FLOW_CMD_DEL,
  8.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  9.      .policy = flow_policy,
  10.      .doit = ovs_flow_cmd_del
  11.    },
  12.    { .cmd = OVS_FLOW_CMD_GET,
  13.      .flags = 0, /* OK for unprivileged users. */
  14.      .policy = flow_policy,
  15.      .doit = ovs_flow_cmd_get,
  16.      .dumpit = ovs_flow_cmd_dump
  17.    },
  18.    { .cmd = OVS_FLOW_CMD_SET,
  19.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  20.      .policy = flow_policy,
  21.      .doit = ovs_flow_cmd_set,
  22.    },
  23. };

dp_packet_genl_family
  1. static struct genl_family dp_packet_genl_family = {
  2.    .id = GENL_ID_GENERATE,
  3.    .hdrsize = sizeof(struct ovs_header),
  4.    .name = OVS_PACKET_FAMILY,
  5.    .version = OVS_PACKET_VERSION,
  6.    .maxattr = OVS_PACKET_ATTR_MAX,
  7.    .netnsok = true,
  8.    .parallel_ops = true,
  9.    .ops = dp_packet_genl_ops,
  10.    .n_ops = ARRAY_SIZE(dp_packet_genl_ops),
  11. };
  1. static struct genl_ops dp_packet_genl_ops[] = {
  2.    { .cmd = OVS_PACKET_CMD_EXECUTE,
  3.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  4.      .policy = packet_policy,
  5.      .doit = ovs_packet_cmd_execute
  6.    }
  7. };

 

如上一节中，ovs-vswitchd启动的时候，将虚拟网卡添加到虚拟交换机上的时候，会调用netlink的OVS_VPORT_CMD_NEW命令，因而会调用函数ovs_vport_cmd_new。

会调用static struct vport *new_vport(const struct vport_parms *parms)

会调用struct vport *ovs_vport_add(const struct vport_parms *parms)里面会调用vport = ops->create(parms);

ops是什么呢？在dp_init函数中会调用ovs_netdev_init，它会调用ovs_vport_ops_register(&ovs_netdev_vport_ops);

  1. static struct vport_ops ovs_netdev_vport_ops = {
  2.    .type = OVS_VPORT_TYPE_NETDEV,
  3.    .create = netdev_create,
  4.    .destroy = netdev_destroy,
  5.    .send = dev_queue_xmit,
  6. };

 

所以ops->create会调用netdev_create，它会调用ovs_netdev_link，其中有下面的代码：

  1. err = netdev_rx_handler_register(vport->dev, netdev_frame_hook,
  2.                 vport);

 

注册一个方法叫做netdev_frame_hook，每当网卡收到包的时候，就调用这个方法。

 

 

上一节我们讲了ovs-vswitchd，其中虚拟网桥初始化的时候，对调用内核模块来添加虚拟网卡。

 

我们从openvswitch内核模块的加载过程，来看这个过程。

 

在datapath/datapath.c中会调用module_init(dp_init);来初始化内核模块。

 

  1. static int __init dp_init(void)
  2. {
  3.    int err;
  4.  
  5.    BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > FIELD_SIZEOF(struct sk_buff, cb));
  6.  
  7.    pr_info("Open vSwitch switching datapath %s\n", VERSION);
  8.  
  9.    err = compat_init();
  10.    if (err)
  11.       goto error;
  12.  
  13.    err = action_fifos_init();
  14.    if (err)
  15.       goto error_compat_exit;
  16.  
  17.    err = ovs_internal_dev_rtnl_link_register();
  18.    if (err)
  19.       goto error_action_fifos_exit;
  20.  
  21.    err = ovs_flow_init();
  22.    if (err)
  23.       goto error_unreg_rtnl_link;
  24.  
  25.    err = ovs_vport_init();
  26.    if (err)
  27.       goto error_flow_exit;
  28.  
  29.    err = register_pernet_device(&ovs_net_ops);
  30.    if (err)
  31.       goto error_vport_exit;
  32.  
  33.    err = register_netdevice_notifier(&ovs_dp_device_notifier);
  34.    if (err)
  35.       goto error_netns_exit;
  36.  
  37.    err = ovs_netdev_init();
  38.    if (err)
  39.       goto error_unreg_notifier;
  40.  
  41.    err = dp_register_genl();
  42.    if (err < 0)
  43.       goto error_unreg_netdev;
  44.  
  45.    return 0;
  46.  
  47. error_unreg_netdev:
  48.    ovs_netdev_exit();
  49. error_unreg_notifier:
  50.    unregister_netdevice_notifier(&ovs_dp_device_notifier);
  51. error_netns_exit:
  52.    unregister_pernet_device(&ovs_net_ops);
  53. error_vport_exit:
  54.    ovs_vport_exit();
  55. error_flow_exit:
  56.    ovs_flow_exit();
  57. error_unreg_rtnl_link:
  58.    ovs_internal_dev_rtnl_link_unregister();
  59. error_action_fifos_exit:
  60.    action_fifos_exit();
  61. error_compat_exit:
  62.    compat_exit();
  63. error:
  64.    return err;
  65. }

 

其中比较重要的是调用了dp_register_genl()，这个就是注册netlink函数，从而ovs-vswitchd可以通过netlink调用内核。

 

  1. static int dp_register_genl(void)
  2. {
  3.    int err;
  4.    int i;
  5.  
  6.    for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
  7.  
  8.       err = genl_register_family(dp_genl_families[i]);
  9.       if (err)
  10.          goto error;
  11.    }
  12.  
  13.    return 0;
  14.  
  15. error:
  16.    dp_unregister_genl(i);
  17.    return err;
  18. }

 

这里dp_genl_families由四个netlink的family组成

 

  1. static struct genl_family *dp_genl_families[] = {
  2.    &dp_datapath_genl_family,
  3.    &dp_vport_genl_family,
  4.    &dp_flow_genl_family,
  5.    &dp_packet_genl_family,
  6. };

 

其中分别定义了以下的操作：

 

Family名称

.name

.ops

dp_datapath_genl_family
  1. static struct genl_family dp_datapath_genl_family = {
  2.    .id = GENL_ID_GENERATE,
  3.    .hdrsize = sizeof(struct ovs_header),
  4.    .name = OVS_DATAPATH_FAMILY,
  5.    .version = OVS_DATAPATH_VERSION,
  6.    .maxattr = OVS_DP_ATTR_MAX,
  7.    .netnsok = true,
  8.    .parallel_ops = true,
  9.    .ops = dp_datapath_genl_ops,
  10.    .n_ops = ARRAY_SIZE(dp_datapath_genl_ops),
  11.    .mcgrps = &ovs_dp_datapath_multicast_group,
  12.    .n_mcgrps = 1,
  13. };
  1. static struct genl_ops dp_datapath_genl_ops[] = {
  2.    { .cmd = OVS_DP_CMD_NEW,
  3.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  4.      .policy = datapath_policy,
  5.      .doit = ovs_dp_cmd_new
  6.    },
  7.    { .cmd = OVS_DP_CMD_DEL,
  8.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  9.      .policy = datapath_policy,
  10.      .doit = ovs_dp_cmd_del
  11.    },
  12.    { .cmd = OVS_DP_CMD_GET,
  13.      .flags = 0, /* OK for unprivileged users. */
  14.      .policy = datapath_policy,
  15.      .doit = ovs_dp_cmd_get,
  16.      .dumpit = ovs_dp_cmd_dump
  17.    },
  18.    { .cmd = OVS_DP_CMD_SET,
  19.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  20.      .policy = datapath_policy,
  21.      .doit = ovs_dp_cmd_set,
  22.    },
  23. };

dp_vport_genl_family
  1. struct genl_family dp_vport_genl_family = {
  2.    .id = GENL_ID_GENERATE,
  3.    .hdrsize = sizeof(struct ovs_header),
  4.    .name = OVS_VPORT_FAMILY,
  5.    .version = OVS_VPORT_VERSION,
  6.    .maxattr = OVS_VPORT_ATTR_MAX,
  7.    .netnsok = true,
  8.    .parallel_ops = true,
  9.    .ops = dp_vport_genl_ops,
  10.    .n_ops = ARRAY_SIZE(dp_vport_genl_ops),
  11.    .mcgrps = &ovs_dp_vport_multicast_group,
  12.    .n_mcgrps = 1,
  13. };
  1. static struct genl_ops dp_vport_genl_ops[] = {
  2.    { .cmd = OVS_VPORT_CMD_NEW,
  3.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  4.      .policy = vport_policy,
  5.      .doit = ovs_vport_cmd_new
  6.    },
  7.    { .cmd = OVS_VPORT_CMD_DEL,
  8.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  9.      .policy = vport_policy,
  10.      .doit = ovs_vport_cmd_del
  11.    },
  12.    { .cmd = OVS_VPORT_CMD_GET,
  13.      .flags = 0, /* OK for unprivileged users. */
  14.      .policy = vport_policy,
  15.      .doit = ovs_vport_cmd_get,
  16.      .dumpit = ovs_vport_cmd_dump
  17.    },
  18.    { .cmd = OVS_VPORT_CMD_SET,
  19.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  20.      .policy = vport_policy,
  21.      .doit = ovs_vport_cmd_set,
  22.    },
  23. };

dp_flow_genl_family
  1. static struct genl_family dp_flow_genl_family = {
  2.    .id = GENL_ID_GENERATE,
  3.    .hdrsize = sizeof(struct ovs_header),
  4.    .name = OVS_FLOW_FAMILY,
  5.    .version = OVS_FLOW_VERSION,
  6.    .maxattr = OVS_FLOW_ATTR_MAX,
  7.    .netnsok = true,
  8.    .parallel_ops = true,
  9.    .ops = dp_flow_genl_ops,
  10.    .n_ops = ARRAY_SIZE(dp_flow_genl_ops),
  11.    .mcgrps = &ovs_dp_flow_multicast_group,
  12.    .n_mcgrps = 1,
  13. };
  1. static struct genl_ops dp_flow_genl_ops[] = {
  2.    { .cmd = OVS_FLOW_CMD_NEW,
  3.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  4.      .policy = flow_policy,
  5.      .doit = ovs_flow_cmd_new
  6.    },
  7.    { .cmd = OVS_FLOW_CMD_DEL,
  8.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  9.      .policy = flow_policy,
  10.      .doit = ovs_flow_cmd_del
  11.    },
  12.    { .cmd = OVS_FLOW_CMD_GET,
  13.      .flags = 0, /* OK for unprivileged users. */
  14.      .policy = flow_policy,
  15.      .doit = ovs_flow_cmd_get,
  16.      .dumpit = ovs_flow_cmd_dump
  17.    },
  18.    { .cmd = OVS_FLOW_CMD_SET,
  19.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  20.      .policy = flow_policy,
  21.      .doit = ovs_flow_cmd_set,
  22.    },
  23. };

dp_packet_genl_family
  1. static struct genl_family dp_packet_genl_family = {
  2.    .id = GENL_ID_GENERATE,
  3.    .hdrsize = sizeof(struct ovs_header),
  4.    .name = OVS_PACKET_FAMILY,
  5.    .version = OVS_PACKET_VERSION,
  6.    .maxattr = OVS_PACKET_ATTR_MAX,
  7.    .netnsok = true,
  8.    .parallel_ops = true,
  9.    .ops = dp_packet_genl_ops,
  10.    .n_ops = ARRAY_SIZE(dp_packet_genl_ops),
  11. };
  1. static struct genl_ops dp_packet_genl_ops[] = {
  2.    { .cmd = OVS_PACKET_CMD_EXECUTE,
  3.      .flags = GENL_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  4.      .policy = packet_policy,
  5.      .doit = ovs_packet_cmd_execute
  6.    }
  7. };

 

如上一节中，ovs-vswitchd启动的时候，将虚拟网卡添加到虚拟交换机上的时候，会调用netlink的OVS_VPORT_CMD_NEW命令，因而会调用函数ovs_vport_cmd_new。

会调用static struct vport *new_vport(const struct vport_parms *parms)

会调用struct vport *ovs_vport_add(const struct vport_parms *parms)里面会调用vport = ops->create(parms);

ops是什么呢？在dp_init函数中会调用ovs_netdev_init，它会调用ovs_vport_ops_register(&ovs_netdev_vport_ops);

  1. static struct vport_ops ovs_netdev_vport_ops = {
  2.    .type = OVS_VPORT_TYPE_NETDEV,
  3.    .create = netdev_create,
  4.    .destroy = netdev_destroy,
  5.    .send = dev_queue_xmit,
  6. };

 

所以ops->create会调用netdev_create，它会调用ovs_netdev_link，其中有下面的代码：

  1. err = netdev_rx_handler_register(vport->dev, netdev_frame_hook,
  2.                 vport);

 

注册一个方法叫做netdev_frame_hook，每当网卡收到包的时候，就调用这个方法。

 

标签
openvswitch
flow
ovs
易学教程内所有资源均来自网络或用户发布的内容，如有违反法律规定的内容欢迎反馈！
该文章没有解决你所遇到的问题?点击提问,说说你的问题,让更多的人一起探讨吧!