kubernetes垃圾回收器GarbageCollector 源码分析（完）

type ObjectMeta struct {
    // ...    
    Finalizers []string
}

func (gc *GarbageCollector) runAttemptToDeleteWorker() {
    for gc.attemptToDeleteWorker() {    
    }
}

func (gc *GarbageCollector) attemptToDeleteWorker() bool {
    //从队列里取出需要尝试删除的资源
   item, quit := gc.attemptToDelete.Get()
   gc.workerLock.RLock()
   defer gc.workerLock.RUnlock()
   if quit {
       return false
   }
   defer gc.attemptToDelete.Done(item)
   n, ok := item.(*node)
   if !ok {
       utilruntime.HandleError(fmt.Errorf("expect *node, got %#v", item))
       return true
   }
  err := gc.attemptToDeleteItem(n)
  if err != nil {
       if _, ok := err.(*restMappingError); ok {
           // There are at least two ways this can happen:
           // 1. The reference is to an object of a custom type that has not yet been
           //    recognized by gc.restMapper (this is a transient error).
           // 2. The reference is to an invalid group/version. We don't currently
           //    have a way to distinguish this from a valid type we will recognize
           //    after the next discovery sync.
           // For now, record the error and retry.
           klog.V(5).Infof("error syncing item %s: %v", n, err)
       } else {
           utilruntime.HandleError(fmt.Errorf("error syncing item %s: %v", n, err))
       }
       // retry if garbage collection of an object failed.
       // 如果对象的垃圾收集失败，则重试。
       gc.attemptToDelete.AddRateLimited(item)
   } else if !n.isObserved() {
       // requeue if item hasn't been observed via an informer event yet.
       // otherwise a virtual node for an item added AND removed during watch reestablishment can get stuck in the graph and never removed.
       // see https://issue.k8s.io/56121
       klog.V(5).Infof("item %s hasn't been observed via informer yet", n.identity)
       gc.attemptToDelete.AddRateLimited(item)
   }
   return true
}

    func (gc *GarbageCollector) attemptToDeleteItem(item *node) error {
        klog.V(2).Infof("processing item %s", item.identity)
        // "being deleted" is an one-way trip to the final deletion. We'll just wait for the final deletion, and then process the object's dependents.
        // item资源被标记为正在删除,即deletionTimestamp不为nil;且不是正在删除从资源(这个从上一节可以看出,只有item被foreground方式删除时,deletingDependents才会被设置为true)
        // item在删除中,且为Orphan和Background方式删除则直接返回
        if item.isBeingDeleted() && !item.isDeletingDependents() {
            klog.V(5).Infof("processing item %s returned at once, because its DeletionTimestamp is non-nil", item.identity)
            return nil
        }
        // TODO: It's only necessary to talk to the API server if this is a
        // "virtual" node. The local graph could lag behind the real status, but in
        // practice, the difference is small.
        //根据item里的信息获取object对象体
        latest, err := gc.getObject(item.identity)
        switch {
        case errors.IsNotFound(err):
            // the GraphBuilder can add "virtual" node for an owner that doesn't
            // exist yet, so we need to enqueue a virtual Delete event to remove
            // the virtual node from GraphBuilder.uidToNode.
            klog.V(5).Infof("item %v not found, generating a virtual delete event", item.identity)
            gc.dependencyGraphBuilder.enqueueVirtualDeleteEvent(item.identity)
            // since we're manually inserting a delete event to remove this node,
            // we don't need to keep tracking it as a virtual node and requeueing in attemptToDelete
            item.markObserved()
            return nil
        case err != nil:
            return err
        }
        //uid不匹配
        if latest.GetUID() != item.identity.UID {
            klog.V(5).Infof("UID doesn't match, item %v not found, generating a virtual delete event", item.identity)
            gc.dependencyGraphBuilder.enqueueVirtualDeleteEvent(item.identity)
            // since we're manually inserting a delete event to remove this node,
            // we don't need to keep tracking it as a virtual node and requeueing in attemptToDelete
            //因为我们手动插入删除事件以删除此节点，我们不需要将其作为虚拟节点跟踪并在attemptToDelete中重新排队
            item.markObserved()
            return nil
        }
        // TODO: attemptToOrphanWorker() routine is similar. Consider merging
        // attemptToOrphanWorker() into attemptToDeleteItem() as well.
        // item的从资源正在删除中,同时删除其从资源
        if item.isDeletingDependents() {
            return gc.processDeletingDependentsItem(item)
        }
        // compute if we should delete the item
        // 获取该object里metadata.ownerReference
        // 计算我们是否应删除该项目
        ownerReferences := latest.GetOwnerReferences()
        if len(ownerReferences) == 0 {
            //没有owner的不用处理
            klog.V(2).Infof("object %s's doesn't have an owner, continue on next item", item.identity)
            return nil
        }
        //solid(owner存在,owner没被删或者终结器不为foregroundDeletion Finalizer); dangling(owner不存在)
        // waitingForDependentsDeletion(owner存在,owner的deletionTimestamp为非nil，并且有foregroundDeletion Finalizer)owner列表
        solid, dangling, waitingForDependentsDeletion, err := gc.classifyReferences(item, ownerReferences)
        if err != nil {
            return err
        }
        klog.V(5).Infof("classify references of %s.\nsolid: %#v\ndangling: %#v\nwaitingForDependentsDeletion: %#v\n", item.identity, solid, dangling, waitingForDependentsDeletion)
        switch {
        //item对象的owner存在,且不是正在删除
        case len(solid) != 0:
            klog.V(2).Infof("object %#v has at least one existing owner: %#v, will not garbage collect", solid, item.identity)
            if len(dangling) == 0 && len(waitingForDependentsDeletion) == 0 {
                return nil
            }
            klog.V(2).Infof("remove dangling references %#v and waiting references %#v for object %s", dangling, waitingForDependentsDeletion, item.identity)
            // waitingForDependentsDeletion needs to be deleted from the
            // ownerReferences, otherwise the referenced objects will be stuck with
            // the FinalizerDeletingDependents and never get deleted.
            // waitingForDependentsDeletion需要从 ownerReferences中删除，否则引用的对象将被
            // FinalizerDeletingDependents所卡住，并且永远不会被删除。
            //需要移除的ownerUids
            ownerUIDs := append(ownerRefsToUIDs(dangling), ownerRefsToUIDs(waitingForDependentsDeletion)...)
            //拼接patch请求参数
            patch := deleteOwnerRefStrategicMergePatch(item.identity.UID, ownerUIDs...)
            //发送patch请求
            _, err = gc.patch(item, patch, func(n *node) ([]byte, error) {
                return gc.deleteOwnerRefJSONMergePatch(n, ownerUIDs...)
            })
            return err
        //item对象的owner正在被删除; 且item有从资源
        case len(waitingForDependentsDeletion) != 0 && item.dependentsLength() != 0:
            deps := item.getDependents()
            // 遍历item从资源
            for _, dep := range deps {
                if dep.isDeletingDependents() {
                    // this circle detection has false positives, we need to
                    // apply a more rigorous detection if this turns out to be a
                    // problem.
                    // there are multiple workers run attemptToDeleteItem in
                    // parallel, the circle detection can fail in a race condition.
                    klog.V(2).Infof("processing object %s, some of its owners and its dependent [%s] have FinalizerDeletingDependents, to prevent potential cycle, its ownerReferences are going to be modified to be non-blocking, then the object is going to be deleted with Foreground", item.identity, dep.identity)
                    // 生成一个补丁，该补丁会取消设置item所有ownerReferences的BlockOwnerDeletion字段,避免阻塞item的owner删除
                    patch, err := item.unblockOwnerReferencesStrategicMergePatch()
                    if err != nil {
                        return err
                    }
                    //执行patch
                    if _, err := gc.patch(item, patch, gc.unblockOwnerReferencesJSONMergePatch); err != nil {
                        return err
                    }
                    break
                }
            }
            //item对象的至少一个owner具有foregroundDeletion Finalizer，并且该对象本身具有依赖项，因此它将在Foreground中删除
            klog.V(2).Infof("at least one owner of object %s has FinalizerDeletingDependents, and the object itself has dependents, so it is going to be deleted in Foreground", item.identity)
            // the deletion event will be observed by the graphBuilder, so the item
            // will be processed again in processDeletingDependentsItem. If it
            // doesn't have dependents, the function will remove the
            // FinalizerDeletingDependents from the item, resulting in the final
            // deletion of the item.
            // graphBuilder将观察删除事件，因此将在processDeletingDependentsItem中再次处理该项目。
            // 如果没有依赖项，该函数将从项中删除foregroundDeletion Finalizer，最终删除item。
            policy := metav1.DeletePropagationForeground
            return gc.deleteObject(item.identity, &policy)
        default:
            // item doesn't have any solid owner, so it needs to be garbage
            // collected. Also, none of item's owners is waiting for the deletion of
            // the dependents, so set propagationPolicy based on existing finalizers.
            // item没有任何实体所有者，因此需要收集垃圾 。此外，项目的所有者都没有等待删除
            // 依赖项，因此请根据现有的终结器设置propagationPolicy。
            var policy metav1.DeletionPropagation
            switch {
            case hasOrphanFinalizer(latest):
                // if an existing orphan finalizer is already on the object, honor it.
                //如果现有的孤儿终结器已经在对象上，请尊重它。
                policy = metav1.DeletePropagationOrphan
            case hasDeleteDependentsFinalizer(latest):
                // if an existing foreground finalizer is already on the object, honor it.
                //如果现有的前景终结器已经在对象上，请尊重它。
                policy = metav1.DeletePropagationForeground
            default:
                // otherwise, default to background.
                //否则，默认为背景。
                policy = metav1.DeletePropagationBackground
            }
            klog.V(2).Infof("delete object %s with propagation policy %s", item.identity, policy)
            //删除孤儿对象
            return gc.deleteObject(item.identity, &policy)
        }
    }

//等待其依赖项被删除的进程项
func (gc *GarbageCollector) processDeletingDependentsItem(item *node) error {
    //阻塞item资源删除的从资源列表
    blockingDependents := item.blockingDependents()
    //没有阻塞item资源删除的从资源,则移除item资源的foregroundDeletion终结器
    if len(blockingDependents) == 0 {
        klog.V(2).Infof("remove DeleteDependents finalizer for item %s", item.identity)
        return gc.removeFinalizer(item, metav1.FinalizerDeleteDependents)
    }
    //遍历阻塞item资源删除的从资源
    for _, dep := range blockingDependents {
        // 如果dep的从资源没有开始删除,则将dep加入到尝试删除队列中
        if !dep.isDeletingDependents() {
            klog.V(2).Infof("adding %s to attemptToDelete, because its owner %s is deletingDependents", dep.identity, item.identity)
            //将从资源加入删除队列
            gc.attemptToDelete.Add(dep)
        }
    }
    return nil
}

// 将latestReferences分为三类：
// solid：所有者存在，且不是waitingForDependentsDeletion
// dangling悬空：所有者不存在
// waitingForDependentsDeletion: 所有者存在，其deletionTimestamp为非nil，并且有FinalizerDeletingDependents
func (gc *GarbageCollector) classifyReferences(item *node, latestReferences []metav1.OwnerReference) (
    solid, dangling, waitingForDependentsDeletion []metav1.OwnerReference, err error) {
    //遍历该node的owner
    for _, reference := range latestReferences {
        //获取owner是否存在;isDangling为true表示不存在,发生err则最终将该item加入AddRateLimited attemptToDelete队列
        isDangling, owner, err := gc.isDangling(reference, item)
        if err != nil {
            return nil, nil, nil, err
        }
        //将不存在的owner加入dangling切片
        if isDangling {
            dangling = append(dangling, reference)
            continue
        }
        //owner存在,获取accessor
        ownerAccessor, err := meta.Accessor(owner)
        if err != nil {
            return nil, nil, nil, err
       }
        //owner正在被删除,且owner有foregroundDeletion Finalizer
        if ownerAccessor.GetDeletionTimestamp() != nil && hasDeleteDependentsFinalizer(ownerAccessor) {
            //owner将等待依赖删除;收集等待删除依赖的owner列表
            waitingForDependentsDeletion = append(waitingForDependentsDeletion, reference)
        } else {
            //owner没被删或者终结器不为foregroundDeletion Finalizer
            solid = append(solid, reference)
        }
    }
    return solid, dangling, waitingForDependentsDeletion, nil
}

// isDangling检查引用是否指向不存在的对象。如果isDangling在API服务器上查找引用的对象，它也返回其最新状态。
func (gc *GarbageCollector) isDangling(reference metav1.OwnerReference, item *node) (
    dangling bool, owner *unstructured.Unstructured, err error) {
    if gc.absentOwnerCache.Has(reference.UID) {
        klog.V(5).Infof("according to the absentOwnerCache, object %s's owner %s/%s, %s does not exist", item.identity.UID, reference.APIVersion, reference.Kind, reference.Name)
        return true, nil, nil
    }
    // TODO: we need to verify the reference resource is supported by the
    // system. If it's not a valid resource, the garbage collector should i)
    // ignore the reference when decide if the object should be deleted, and
    // ii) should update the object to remove such references. This is to
    // prevent objects having references to an old resource from being
    // deleted during a cluster upgrade.
    resource, namespaced, err := gc.apiResource(reference.APIVersion, reference.Kind)
    if err != nil {
        return false, nil, err
    }
    // TODO: It's only necessary to talk to the API server if the owner node
    // is a "virtual" node. The local graph could lag behind the real
    // status, but in practice, the difference is small.
    owner, err = gc.dynamicClient.Resource(resource).Namespace(resourceDefaultNamespace(namespaced, item.identity.Namespace)).Get(reference.Name, metav1.GetOptions{})
    switch {
    case errors.IsNotFound(err):
        gc.absentOwnerCache.Add(reference.UID)
        klog.V(5).Infof("object %s's owner %s/%s, %s is not found", item.identity.UID, reference.APIVersion, reference.Kind, reference.Name)
        return true, nil, nil
    case err != nil:
        return false, nil, err
    }
    if owner.GetUID() != reference.UID {
        klog.V(5).Infof("object %s's owner %s/%s, %s is not found, UID mismatch", item.identity.UID, reference.APIVersion, reference.Kind, reference.Name)
        gc.absentOwnerCache.Add(reference.UID)
        return true, nil, nil
    }
    return false, owner, nil
}

func (gc *GarbageCollector) runAttemptToOrphanWorker() {
    for gc.attemptToOrphanWorker() {
    }
}

// attemptToOrphanWorker将一个节点从attemptToOrphan中取出，然后根据GC维护的图找到它的依赖项，然后将其从其依赖项的
// OwnerReferences中删除，最后更新item以删除孤儿终结器。如果这些步骤中的任何一个失败，则将节点添加回attemptToOrphan。
func (gc *GarbageCollector) attemptToOrphanWorker() bool {
    item, quit := gc.attemptToOrphan.Get()
    gc.workerLock.RLock()
    defer gc.workerLock.RUnlock()
    if quit {
        return false
    }
    defer gc.attemptToOrphan.Done(item)
    owner, ok := item.(*node)
    if !ok {
        utilruntime.HandleError(fmt.Errorf("expect *node, got %#v", item))
        return true
    }
    // we don't need to lock each element, because they never get updated
    owner.dependentsLock.RLock()
    dependents := make([]*node, 0, len(owner.dependents))
    for dependent := range owner.dependents {
        dependents = append(dependents, dependent)
    }
    owner.dependentsLock.RUnlock()
    // 处理孤儿
    err := gc.orphanDependents(owner.identity, dependents)
    if err != nil {
        utilruntime.HandleError(fmt.Errorf("orphanDependents for %s failed with %v", owner.identity, err))
        gc.attemptToOrphan.AddRateLimited(item)
        return true
    }
    // update the owner, remove "orphaningFinalizer" from its finalizers list
    // 移除item的orphan终结器
    err = gc.removeFinalizer(owner, metav1.FinalizerOrphanDependents)
    if err != nil {
        utilruntime.HandleError(fmt.Errorf("removeOrphanFinalizer for %s failed with %v", owner.identity, err))
        gc.attemptToOrphan.AddRateLimited(item)
    }
    return true
}

    // dependents are copies of pointers to the owner's dependents, they don't need to be locked.
    func (gc *GarbageCollector) orphanDependents(owner objectReference, dependents []*node) error {
        errCh := make(chan error, len(dependents))
        wg := sync.WaitGroup{}
        wg.Add(len(dependents))
        for i := range dependents {
            go func(dependent *node) {
                defer wg.Done()
                // the dependent.identity.UID is used as precondition
                patch := deleteOwnerRefStrategicMergePatch(dependent.identity.UID, owner.UID)
                _, err := gc.patch(dependent, patch, func(n *node) ([]byte, error) {
                    return gc.deleteOwnerRefJSONMergePatch(n, owner.UID)
                })
                // note that if the target ownerReference doesn't exist in the
                // dependent, strategic merge patch will NOT return an error.
                if err != nil && !errors.IsNotFound(err) {
                    errCh <- fmt.Errorf("orphaning %s failed, %v", dependent.identity, err)
                }
            }(dependents[i])
        }
        wg.Wait()
        close(errCh)
        var errorsSlice []error
        for e := range errCh {
            errorsSlice = append(errorsSlice, e)
        }
        if len(errorsSlice) != 0 {
            return fmt.Errorf("failed to orphan dependents of owner %s, got errors: %s", owner, utilerrors.NewAggregate(errorsSlice).Error())
        }
        klog.V(5).Infof("successfully updated all dependents of owner %s", owner)
        return nil
    }

    func deleteOwnerRefStrategicMergePatch(dependentUID types.UID, ownerUIDs ...types.UID) []byte {
        var pieces []string
        //拼接需要删除的uid
        for _, ownerUID := range ownerUIDs {
            pieces = append(pieces, fmt.Sprintf(`{"$patch":"delete","uid":"%s"}`, ownerUID))
        }
        //拼接patch请求参数
        patch := fmt.Sprintf(`{"metadata":{"ownerReferences":[%s],"uid":"%s"}}`, strings.Join(pieces, ","), dependentUID)
        return []byte(patch)
    }