容器化RDS｜计算存储分离架构下的Split-Brain

以阿里巴巴为例, 考虑到今时今日它的规模, 如果能够实现数据库服务的离线(ODPS)/在线集群的混合部署, 意义极其重大. 关键问题在于, 离线(ODPS)计算和在线计算对实时性要求不同, 硬件配置也不同, 尤其是本地存储介质, ●离线(ODPS)以机械磁盘为主 ●在线以 SSD / Flash 为主. 如果采用本地存储作为数据库实例的存储介质, 试想一下, 一个 Storage Qos 要求是 Flash 的数据库实例无法调度到离线计算集群, 哪怕离线计算集群 CPU, Memory 有大量空闲. 计算存储分离为实现离线(ODPS)/在线集群的混合部署提供了可能.

当集群中某个 Node 不可用后, 借助 Kubernetes 的原生组件Node Controller, Scheduler和原生API Statefulset即可将数据库实例调度到其他可用节点, 以实现数据库实例的高可用.

补充一句, 借助 ETCD 集群的高可用强一致, 得以保证 Kubernetes 集群元信息的一致性, ●ETCD 基于 Raft 算法实现 ●Raft 算法是一种基于消息传递(state machine replicated)且具有高度容错(fault tolerance)特性的一致性算法(consensus algorithm). ●Raft 是大名鼎鼎的 Paxos 的简化版本 ●如果对于 Raft 算法的实现有兴趣,可以看看 https://github.com/goraft/raft 所有感兴趣一致性算法的同学, 都值得花精力学习. 基于 goraft/raft, 我实现了Network Partition Failures/Recovery TestCase, 收获不小.

Kubelet 细节如下: ●通过 API Server 更新集群信息 if kl.kubeClient != nil { // Start syncing node status immediately, this may set up things the runtime needs to run. go wait.Until(kl.syncNodeStatus, kl.nodeStatusUpdateFrequency, wait.NeverStop) } ●定期(nodeStatusUpdateFrequency)更新对应节点状态 nodeStatusUpdateFrequency 默认时间为 10 秒, 测试时设置的是8s obj.NodeStatusUpdateFrequency = metav1.Duration{Duration: 10 * time.Second} ●更新如下信息: func (kl *Kubelet) defaultNodeStatusFuncs() []func(*v1.Node) error { // initial set of node status update handlers, can be modified by Option's withoutError := func(f func(*v1.Node)) func(*v1.Node) error { return func(n *v1.Node) error { f(n) return nil } } return []func(*v1.Node) error{ kl.setNodeAddress, withoutError(kl.setNodeStatusInfo), withoutError(kl.setNodeOODCondition), withoutError(kl.setNodeMemoryPressureCondition), withoutError(kl.setNodeDiskPressureCondition), withoutError(kl.setNodeReadyCondition), withoutError(kl.setNodeVolumesInUseStatus), withoutError(kl.recordNodeSchedulableEvent), } } ●通过 kubectl 可以获得节点的信息

●定期(nodeMonitorPeriod)查看一次节点状态 // Incorporate the results of node status pushed from kubelet to master. go wait.Until(func() { if err := nc.monitorNodeStatus(); err != nil { glog.Errorf("Error monitoring node status: %v", err) } }, nc.nodeMonitorPeriod, wait.NeverStop) ●nodeMonitorPeriod 默认 5秒, 测试时4s NodeMonitorPeriod: metav1.Duration{Duration: 5 * time.Second}, ●当超过 NodeMonitorGracePeriod 时间后, 节点状态没有更新将节点状态设置成 unknown if nc.now().After(savedNodeStatus.probeTimestamp.Add(gracePeriod)) { // NodeReady condition was last set longer ago than gracePeriod, so update it to Unknown // (regardless of its current value) in the master. if currentReadyCondition == nil { glog.V(2).Infof("node %v is never updated by kubelet", node.Name) node.Status.Conditions = append(node.Status.Conditions, v1.NodeCondition{ Type: v1.NodeReady, Status: v1.ConditionUnknown, Reason: "NodeStatusNeverUpdated", Message: fmt.Sprintf("Kubelet never posted node status."), LastHeartbeatTime: node.CreationTimestamp, LastTransitionTime: nc.now(), }) } else { glog.V(4).Infof("node %v hasn't been updated for %+v. Last ready condition is: %+v", node.Name, nc.now().Time.Sub(savedNodeStatus.probeTimestamp.Time), observedReadyCondition) if observedReadyCondition.Status != v1.ConditionUnknown { currentReadyCondition.Status = v1.ConditionUnknown currentReadyCondition.Reason = "NodeStatusUnknown" currentReadyCondition.Message = "Kubelet stopped posting node status." // LastProbeTime is the last time we heard from kubelet. currentReadyCondition.LastHeartbeatTime = observedReadyCondition.LastHeartbeatTime currentReadyCondition.LastTransitionTime = nc.now() } }

●在podEvictionTimeout 后, 认为该节点上 pods 需要开始驱逐 if observedReadyCondition.Status == v1.ConditionUnknown { if nc.useTaintBasedEvictions { // We want to update the taint straight away if Node is already tainted with the UnreachableTaint if taintutils.TaintExists(node.Spec.Taints, NotReadyTaintTemplate) { taintToAdd := *UnreachableTaintTemplate if !util.SwapNodeControllerTaint(nc.kubeClient, []*v1.Taint{&taintToAdd}, []*v1.Taint{NotReadyTaintTemplate}, node) { glog.Errorf("Failed to instantly swap UnreachableTaint to NotReadyTaint. Will try again in the next cycle.") } } else if nc.markNodeForTainting(node) { glog.V(2).Infof("Node %v is unresponsive as of %v. Adding it to the Taint queue.", node.Name, decisionTimestamp, ) } } else { if decisionTimestamp.After(nc.nodeStatusMap[node.Name].probeTimestamp.Add(nc.podEvictionTimeout)) { if nc.evictPods(node) { glog.V(2).Infof("Node is unresponsive. Adding Pods on Node %s to eviction queues: %v is later than %v + %v", node.Name, decisionTimestamp, nc.nodeStatusMap[node.Name].readyTransitionTimestamp, nc.podEvictionTimeout-gracePeriod, ) } } } } ●放到驱逐数组中 // evictPods queues an eviction for the provided node name, and returns false if the node is already // queued for eviction. func (nc *Controller) evictPods(node *v1.Node) bool { nc.evictorLock.Lock() defer nc.evictorLock.Unlock() return nc.zonePodEvictor[utilnode.GetZoneKey(node)].Add(node.Name, string(node.UID)) }

●驱逐 goroutine if nc.useTaintBasedEvictions { // Handling taint based evictions. Because we don't want a dedicated logic in TaintManager for NC-originated // taints and we normally don't rate limit evictions caused by taints, we need to rate limit adding taints. go wait.Until(nc.doNoExecuteTaintingPass, scheduler.NodeEvictionPeriod, wait.NeverStop) } else { // Managing eviction of nodes: // When we delete pods off a node, if the node was not empty at the time we then // queue an eviction watcher. If we hit an error, retry deletion. go wait.Until(nc.doEvictionPass, scheduler.NodeEvictionPeriod, wait.NeverStop) } ●通过删除 pods 的方式驱逐 func (nc *Controller) doEvictionPass() { nc.evictorLock.Lock() defer nc.evictorLock.Unlock() for k := range nc.zonePodEvictor { // Function should return 'false' and a time after which it should be retried, or 'true' if it shouldn't (it succeeded). nc.zonePodEvictor[k].Try(func(value scheduler.TimedValue) (bool, time.Duration) { node, err := nc.nodeLister.Get(value.Value) if apierrors.IsNotFound(err) { glog.Warningf("Node %v no longer present in nodeLister!", value.Value) } else if err != nil { glog.Warningf("Failed to get Node %v from the nodeLister: %v", value.Value, err) } else { zone := utilnode.GetZoneKey(node) evictionsNumber.WithLabelValues(zone).Inc() } nodeUID, _ := value.UID.(string) remaining, err := util.DeletePods(nc.kubeClient, nc.recorder, value.Value, nodeUID, nc.daemonSetStore) if err != nil { utilruntime.HandleError(fmt.Errorf("unable to evict node %q: %v", value.Value, err)) return false, 0 } if remaining { glog.Infof("Pods awaiting deletion due to Controller eviction") } return true, 0 }) } }