聊聊kingbus的starRaft

序

本文主要研究一下kingbus的starRaft

starRaft

kingbus/server/server.go

func (s *KingbusServer) starRaft(cfg config.RaftNodeConfig) error {
	var (
		etcdRaftNode etcdraft.Node
		id           types.ID
		cl           *membership.RaftCluster
		remotes      []*membership.Member
		appliedIndex uint64
	)

	prt, err := rafthttp.NewRoundTripper(transport.TLSInfo{}, DialTimeout)
	if err != nil {
		return err
	}

	store, err := storage.NewDiskStorage(cfg.DataDir, cfg.ReserveDataSize)
	if err != nil {
		log.Log.Fatalf("NewKingbusServer:NewDiskStorage error,err:%s,dir:%s", err.Error(), cfg.DataDir)
	}

	//store, err := storage.NewMemoryStorage(cfg.DataDir)
	//if err != nil {
	//	log.Log.Fatalf("NewKingbusServer:NewMemoryStorage error,err:%s,dir:%s", err.Error(), cfg.DataDir)
	//}

	defer func() {
		//close storage when occur error
		if err != nil {
			store.Close()
		}
	}()

	logExist := utils.ExistLog(cfg.DataDir)
	switch {
	case !logExist && !cfg.NewCluster:
		if err = cfg.VerifyJoinExisting(); err != nil {
			return err
		}
		cl, err = membership.NewClusterFromURLsMap(cfg.InitialPeerURLsMap)
		if err != nil {
			return err
		}
		remotePeerURLs := membership.GetRemotePeerURLs(cl, cfg.Name)
		existingCluster, gerr := membership.GetClusterFromRemotePeers(remotePeerURLs, prt)
		if gerr != nil {
			return fmt.Errorf("cannot fetch cluster info from peer urls: %v", gerr)
		}
		if err = membership.ValidateClusterAndAssignIDs(cl, existingCluster); err != nil {
			return fmt.Errorf("error validating peerURLs %s: %v", existingCluster, err)
		}

		remotes = existingCluster.Members()
		cl.SetID(existingCluster.GetID())
		cl.SetStore(store)
		id, etcdRaftNode = startEtcdRaftNode(cfg, store, cl, nil)
	case !logExist && cfg.NewCluster:
		if err = cfg.VerifyBootstrap(); err != nil {
			return err
		}
		cl, err = membership.NewClusterFromURLsMap(cfg.InitialPeerURLsMap)
		if err != nil {
			return err
		}
		m := cl.MemberByName(cfg.Name)
		if membership.IsMemberBootstrapped(cl, cfg.Name, prt, DialTimeout) {
			return fmt.Errorf("member %s has already been bootstrapped", m.ID)
		}

		cl.SetStore(store)
		id, etcdRaftNode = startEtcdRaftNode(cfg, store, cl, cl.MemberIDs())
	case logExist:
		if err = utils.IsDirWriteable(cfg.DataDir); err != nil {
			return fmt.Errorf("cannot write to member directory: %v", err)
		}
		//node restart, read states from storage
		//get applied index
		appliedIndex = raft.MustGetAppliedIndex(store)
		cfg.AppliedIndex = appliedIndex
		id, etcdRaftNode, cl = restartEtcdNode(cfg, store)
		cl.SetStore(store)
	default:
		return fmt.Errorf("unsupported bootstrap config")
	}

	s.raftNode = raft.NewNode(
		raft.NodeConfig{
			IsIDRemoved: func(id uint64) bool { return cl.IsIDRemoved(types.ID(id)) },
			Node:        etcdRaftNode,
			Heartbeat:   cfg.HeartbeatMs,
			Storage:     store,
		},
	)
	//committedIndex,term will update by fsm(UpdateCommittedIndex,SetTerm)
	//set appliedIndex when applyEntries will check the entry continuity
	s.raftNode.SetAppliedIndex(appliedIndex)

	s.id = id
	s.wait = wait.New()
	s.reqIDGen = idutil.NewGenerator(uint16(id), time.Now())
	s.stopping = make(chan struct{})
	s.errorc = make(chan error)
	s.applyBroadcast = utils.NewBroadcast()
	s.stats = stats.NewServerStats(cfg.Name, id.String())
	s.lstats = stats.NewLeaderStats(id.String())
	s.store = store

	tr := &rafthttp.Transport{
		TLSInfo:     transport.TLSInfo{},
		DialTimeout: DialTimeout,
		ID:          id,
		URLs:        cfg.PeerURLs,
		ClusterID:   cl.GetID(),
		Raft:        s,
		ServerStats: s.stats,
		LeaderStats: s.lstats,
		ErrorC:      s.errorc,
	}
	if err = tr.Start(); err != nil {
		return err
	}
	// add all remotes into transport
	//Add remotes to rafthttp, who help newly joined members catch up the
	//progress of the cluster. It supports basic message sending to remote, and
	//has no stream connection for simplicity. remotes will not be used
	//after the latest peers have been added into rafthttp.
	for _, m := range remotes {
		if m.ID != id {
			tr.AddRemote(m.ID, m.PeerURLs)
		}
	}
	for _, m := range cl.Members() {
		if m.ID != id {
			tr.AddPeer(m.ID, m.PeerURLs)
		}
	}
	s.raftNode.Transport = tr
	s.cluster = cl

	return nil
}

• starRaft方法先通过rafthttp.NewRoundTripper创建http.RoundTripper，之后通过storage.NewDiskStorage创建DiskStorage，之后根据logExist及cfg.NewCluster做不同处理；若二者都为false则更新membership.RaftCluster的id为存在的cluster的id，然后执行startEtcdRaftNode；若cfg.NewCluster为true则使用cl.MemberIDs()来执行startEtcdRaftNode；若logExist为true则执行restartEtcdNode；最后创建rafthttp.Transport，执行tr.Start()、tr.AddRemote、tr.AddPeer

startEtcdRaftNode

kingbus/server/server.go

func startEtcdRaftNode(cfg config.RaftNodeConfig, store storage.Storage, cl *membership.RaftCluster, ids []types.ID) (
	id types.ID, n etcdraft.Node) {
	member := cl.MemberByName(cfg.Name)
	peers := make([]etcdraft.Peer, len(ids))

	for i, id := range ids {
		ctx, err := json.Marshal((*cl).Member(id))
		if err != nil {
			log.Log.Panicf("marshal member should never fail: %v", err)
		}
		peers[i] = etcdraft.Peer{ID: uint64(id), Context: ctx}
	}
	id = member.ID
	log.Log.Infof("starting member %s in cluster %s", id, cl.GetID())

	c := &etcdraft.Config{
		ID:                        uint64(id),
		ElectionTick:              int(cfg.ElectionTimeoutMs / cfg.HeartbeatMs),
		HeartbeatTick:             1,
		Storage:                   store,
		MaxSizePerMsg:             cfg.MaxRequestBytes,
		MaxInflightMsgs:           maxInflightMsgs,
		CheckQuorum:               true,
		PreVote:                   cfg.PreVote,
		DisableProposalForwarding: true,
		Logger:                    log.Log,
	}

	n = etcdraft.StartNode(c, peers)
	raft.AdvanceTicks(n, c.ElectionTick)
	return id, n
}

• startEtcdRaftNode方法通过指定的ids创建peers，之后执行etcdraft.StartNode及raft.AdvanceTicks

restartEtcdNode

kingbus/server/server.go

func restartEtcdNode(cfg config.RaftNodeConfig, store storage.Storage) (
	types.ID, etcdraft.Node, *membership.RaftCluster) {
	cl, err := membership.GetRaftClusterFromStorage(store)
	if err != nil {
		if err != nil {
			log.Log.Panic("GetRaftClusterFromStorage error:%s", err.Error())
		}
	}

	log.Log.Debugf("restartEtcdNode:get raft cluster from storage,cluster:%v", cl.String())

	//get id from raftCluster
	member := cl.MemberByName(cfg.Name)
	if member == nil {
		log.Log.Fatalf("restartEtcdNode:member not in raft cluster,cluster:%v,memberName:%s",
			cl.String(), cfg.Name)
	}
	c := &etcdraft.Config{
		ID:                        uint64(member.ID),
		ElectionTick:              int(cfg.ElectionTimeoutMs / cfg.HeartbeatMs),
		HeartbeatTick:             1,
		Applied:                   cfg.AppliedIndex, //set appliedIndex
		Storage:                   store,
		MaxSizePerMsg:             cfg.MaxRequestBytes,
		MaxInflightMsgs:           maxInflightMsgs,
		CheckQuorum:               true,
		PreVote:                   cfg.PreVote,
		DisableProposalForwarding: true,
		Logger:                    log.Log,
	}

	n := etcdraft.RestartNode(c)
	return member.ID, n, cl
}

• restartEtcdNode方法通过membership.GetRaftClusterFromStorage(store)获取RaftCluster，之后通过cl.MemberByName(cfg.Name)获取Member，然后使用member.ID构造etcdraft.Config，最后根据etcdraft.Config执行etcdraft.RestartNode

小结

starRaft方法先通过rafthttp.NewRoundTripper创建http.RoundTripper，之后通过storage.NewDiskStorage创建DiskStorage，之后根据logExist及cfg.NewCluster做不同处理；若二者都为false则更新membership.RaftCluster的id为存在的cluster的id，然后执行startEtcdRaftNode；若cfg.NewCluster为true则使用cl.MemberIDs()来执行startEtcdRaftNode；若logExist为true则执行restartEtcdNode；最后创建rafthttp.Transport，执行tr.Start()、tr.AddRemote、tr.AddPeer

doc

• server.go

来源：oschina

链接：https://my.oschina.net/go4it/blog/4310623