内存lru file比cache大的一种场景介绍

原创

cdh

修改于 2023-10-23 10:48:41

6730

修改于 2023-10-23 10:48:41

文章被收录于专栏：笔记+笔记+

在定位一个线上问题时发现Active(file)+Inactive(file)要比cached统计值大很多，看起来不太符合预期，正常情况下Active(file)+Inactive(file)的统计值都会同时计算到cached里，也就是一般cached的值会比Active(file)+Inactive(file)要大。

/proc/meminfo输出Cached信息内核统计方式如下，从前面meminfo信息看buffer的值并不大，也没有使用swap分区

从前面的截图可以看到AnonPages的值要比Active(anon)+Inactive(anon)大，推测是有部分anon page被统计到lru file page里，但是没有统计到lru anon中去。

搜下内核代码确实有相关的逻辑会将内存从LRU active annon移到lru inactive file的情况(但是这部分内存不会统计到cache里，这也是导致meminfo统计到的cache值比inactive file + active file小的原因)：

static void lru_lazyfree_fn(struct page *page, struct lruvec *lruvec)
{
        if (PageAnon(page) && PageSwapBacked(page) &&
            !PageSwapCache(page) && !PageUnevictable(page)) {
                bool active = PageActive(page);

                del_page_from_lru_list(page, lruvec,
                                       LRU_INACTIVE_ANON + active); //这里从LRU的active anon移除
                ClearPageActive(page);
                ClearPageReferenced(page);
                /*
                 * lazyfree pages are clean anonymous pages. They have
                 * SwapBacked flag cleared to distinguish normal anonymous
                 * pages
                 */
                ClearPageSwapBacked(page);
                add_page_to_lru_list(page, lruvec, LRU_INACTIVE_FILE);//把page加到lru INACTIVE FILE上

                __count_vm_events(PGLAZYFREE, hpage_nr_pages(page));
                count_memcg_page_event(page, PGLAZYFREE);
                update_page_reclaim_stat(lruvec, 1, 0);
        }
}

strace下业务进程看下业务进程都是如何使用内存的，看到有大量的madvise MADV_FREE的系统调用：

网上找了个madvise实例验证确认下：

# cat madvise-sample.c
#include <signal.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdio.h>
#include <sys/mman.h>
#include <unistd.h>

bool cont = true;
//static const ptrdiff_t len = 1 << 25; // 32 MB
static const ptrdiff_t len = (1*1024*1024*1024);
void
handle_signal(int sig)
{
        if (sig != SIGINT) {
                return;
        }

        cont ^= true;
}

void
wait()
{
        cont = false;
        signal(SIGINT, handle_signal);

        printf("CTRL+C to continue\n");

        while (!cont) {
                sleep(1);
        }
}

int
main(void)
{
        char *start, *end;
        void* pb;

        pb = sbrk(0);
        if (pb == (void*)-1) {
                perror("sbrk");
                return 1;
        }

        start = (char*)pb;
        end = start + len;

        // "allocate" mem by increasing the program break
        //
        if (!~brk(end)) {
                perror("brk");
                return 1;
        }
        printf("allocate mem\r\n");
        wait();

        // "touch" the memory so that we get it really utilized - at this point,
        // we should see the faults taking place, and both RSS and active anon
        // going up
        //
        for (; start < end;) {
                *(start++) = 123;
        }
        printf("using allocate mem\r\n");
        wait();

        // let the kernel know that we don't really need half of the memory we
        // allocated anymore - while this will not change RSS, it'll definitely
        // change active and inactive.
        //
/*        if (!~madvise(pb + (len >> 1), (len >> 1), MADV_FREE)) {
                perror("madvise");
                return 1;
        }*/
        if (!~madvise(pb  , len, MADV_FREE)) {
                perror("madvise");
                return 1;
        }

        printf("madvise MADV_FREE mem\r\n");

        wait();

        return 0;
}

compile it
gcc -O2 -static -o sample ./madvise-sample.c

run it in a terminal that has the current proc in a cgroup
mkdir /sys/fs/cgroup/memory/test
echo $$ > /sys/fs/cgroup/memory/test/cgroup.procs
./sample

in another terminal, observe memory.stat for that cgroup
cat /sys/fs/cgroup/memory/test/memory.stat

use CTRL+C to make sample advance - see memory.stat as you do it.

验证效果：

1.开始执行测试程序时只申请了虚拟内存，所以看到内存Active(anon)并没有涨