python源码阅读笔记之GC(二)

第0层

这是指使用glibc的malloc()方法去申请内存,当然这不是啥对象都用这个,而是取决于分配的内存大小

void *
PyObject_Malloc(size_t nbytes)
{
    block *bp;
    poolp pool;
    poolp next;
    uint size;
    /*
     * 异常处理
     */
    if (nbytes > PY_SSIZE_T_MAX)
        return NULL;
    /*
     * 需要分配的内存小于某个特定的值时
     */
    if ((nbytes - 1) < SMALL_REQUEST_THRESHOLD) {
        LOCK();
        /*
         * Most frequent paths first
         */
        size = (uint)(nbytes - 1) >> ALIGNMENT_SHIFT;
        pool = usedpools[size + size];
        if (pool != pool->nextpool) {
            /*
             * There is a used pool for this size class.
             * Pick up the head block of its free list.
             */
            ++pool->ref.count;
            bp = pool->freeblock;
            assert(bp != NULL);
            if ((pool->freeblock = *(block **)bp) != NULL) {
                UNLOCK();
                return (void *)bp;
            }
            /*
             * Reached the end of the free list, try to extend it.
             */
            if (pool->nextoffset <= pool->maxnextoffset) {
                /* There is room for another block. */
                pool->freeblock = (block*)pool +
                                  pool->nextoffset;
                pool->nextoffset += INDEX2SIZE(size);
                *(block **)(pool->freeblock) = NULL;
                UNLOCK();
                return (void *)bp;
            }
            /* Pool is full, unlink from used pools. */
            next = pool->nextpool;
            pool = pool->prevpool;
            next->prevpool = pool;
            pool->nextpool = next;
            UNLOCK();
            return (void *)bp;
        }
        /* There isn't a pool of the right size class immediately
         * available:  use a free pool.
         */
        if (usable_arenas == NULL) {
            /* No arena has a free pool:  allocate a new arena. */
#ifdef WITH_MEMORY_LIMITS
            if (narenas_currently_allocated >= MAX_ARENAS) {
                UNLOCK();
                goto redirect;
            }
#endif
            usable_arenas = new_arena();
            if (usable_arenas == NULL) {
                UNLOCK();
                goto redirect;
            }
            usable_arenas->nextarena =
                usable_arenas->prevarena = NULL;
        }
        assert(usable_arenas->address != 0);
        /* Try to get a cached free pool. */
        pool = usable_arenas->freepools;
        if (pool != NULL) {
            /* Unlink from cached pools. */
            usable_arenas->freepools = pool->nextpool;
            /* This arena already had the smallest nfreepools
             * value, so decreasing nfreepools doesn't change
             * that, and we don't need to rearrange the
             * usable_arenas list.  However, if the arena has
             * become wholly allocated, we need to remove its
             * arena_object from usable_arenas.
             */
            --usable_arenas->nfreepools;
            if (usable_arenas->nfreepools == 0) {
                /* Wholly allocated:  remove. */
                assert(usable_arenas->freepools == NULL);
                assert(usable_arenas->nextarena == NULL ||
                       usable_arenas->nextarena->prevarena ==
                       usable_arenas);
                usable_arenas = usable_arenas->nextarena;
                if (usable_arenas != NULL) {
                    usable_arenas->prevarena = NULL;
                    assert(usable_arenas->address != 0);
                }
            }
            else {
                /* nfreepools > 0:  it must be that freepools
                 * isn't NULL, or that we haven't yet carved
                 * off all the arena's pools for the first
                 * time.
                 */
                assert(usable_arenas->freepools != NULL ||
                       usable_arenas->pool_address <=
                       (block*)usable_arenas->address +
                           ARENA_SIZE - POOL_SIZE);
            }
        init_pool:
            /* Frontlink to used pools. */
            next = usedpools[size + size]; /* == prev */
            pool->nextpool = next;
            pool->prevpool = next;
            next->nextpool = pool;
            next->prevpool = pool;
            pool->ref.count = 1;
            if (pool->szidx == size) {
                /* Luckily, this pool last contained blocks
                 * of the same size class, so its header
                 * and free list are already initialized.
                 */
                bp = pool->freeblock;
                pool->freeblock = *(block **)bp;
                UNLOCK();
                return (void *)bp;
            }
            /*
             * Initialize the pool header, set up the free list to
             * contain just the second block, and return the first
             * block.
             */
            pool->szidx = size;
            size = INDEX2SIZE(size);
            bp = (block *)pool + POOL_OVERHEAD;
            pool->nextoffset = POOL_OVERHEAD + (size << 1);
            pool->maxnextoffset = POOL_SIZE - size;
            pool->freeblock = bp + size;
            *(block **)(pool->freeblock) = NULL;
            UNLOCK();
            return (void *)bp;
        }
        /* Carve off a new pool. */
        assert(usable_arenas->nfreepools > 0);
        assert(usable_arenas->freepools == NULL);
        pool = (poolp)usable_arenas->pool_address;
        assert((block*)pool <= (block*)usable_arenas->address +
                               ARENA_SIZE - POOL_SIZE);
        pool->arenaindex = usable_arenas - arenas;
        assert(&arenas[pool->arenaindex] == usable_arenas);
        pool->szidx = DUMMY_SIZE_IDX;
        usable_arenas->pool_address += POOL_SIZE;
        --usable_arenas->nfreepools;
        if (usable_arenas->nfreepools == 0) {
            assert(usable_arenas->nextarena == NULL ||
                   usable_arenas->nextarena->prevarena ==
                   usable_arenas);
            /* Unlink the arena:  it is completely allocated. */
            usable_arenas = usable_arenas->nextarena;
            if (usable_arenas != NULL) {
                usable_arenas->prevarena = NULL;
                assert(usable_arenas->address != 0);
            }
        }
        goto init_pool;
    }
    /* The small block allocator ends here. */
redirect:
    /* 异常处理
     */
    if (nbytes == 0)
        nbytes = 1;
 /*
 *大于的话直接分配,调用C的内置函数
 */
    return (void *)malloc(nbytes);
}

备注:

在某些版本这个特定的值是256

#define SMALL_REQUEST_THRESHOLD 512

第一层的分配器

如果python的对象并不都是些大对象,并且其生成是循环,这时候就不能使用malloc了

而是先保留内存空间,再去分配

内存结构:

分为3个

block -> pool -> arena

首先来看arena

struct arena_object {
    /* malloc之后的arena的地址 
 */
    uptr address;
    /* 用于对齐pool的地址 */
    block* pool_address;
    /* 空闲的pool总数
     */
    uint nfreepools;
    /* pool的总数 */
    uint ntotalpools;
    /* 连接空闲pool的单向链表 */
    struct pool_header* freepools;
    /* Whenever this arena_object is not associated with an allocated
     * arena, the nextarena member is used to link all unassociated
     * arena_objects in the singly-linked `unused_arena_objects` list.
     * The prevarena member is unused in this case.
     *
     * When this arena_object is associated with an allocated arena
     * with at least one available pool, both members are used in the
     * doubly-linked `usable_arenas` list, which is maintained in
     * increasing order of `nfreepools` values.
     *
     * Else this arena_object is associated with an allocated arena
     * all of whose pools are in use.  `nextarena` and `prevarena`
     * are both meaningless in this case.
     */
    struct arena_object* nextarena;
    struct arena_object* prevarena;
};
这个管理着arena,大小如下
#define ARENA_SIZE              (256 << 10)     /* 256KB */
arena还被arenas数组管理着
/* Array of objects used to track chunks of memory (arenas). */
static struct arena_object* arenas = NULL;
/* Number of slots currently allocated in the `arenas` vector. */
static uint maxarenas = 0;

此外的一些宏定义

#undef  uchar
#define uchar   unsigned char   /* assuming == 8 bits  */
#undef  uint
#define uint    unsigned int    /* assuming >= 16 bits */
#undef  ulong
#define ulong   unsigned long   /* assuming >= 32 bits */
/*这个用于将指针安全的转化为整数
*/
#undef uptr
#define uptr    Py_uintptr_t
/* When you say memory, my mind reasons in terms of (pointers to) blocks */
typedef uchar block;

pool结构

取决于os的页面单位,4k

#define SYSTEM_PAGE_SIZE        (4 * 1024)
#define POOL_SIZE               SYSTEM_PAGE_SIZE        /* must be 2^N */

arenade pool被划分为4k的大小,也就是说将pool的地址按照4k的倍数对齐,这也就意味着arena的地址不一定会和pool地址对齐

所以在struct里需要对齐

struct pool_header {
    union { block *_padding;
            uint count; } ref;          /* 分配到pool里的block的数量    */
    block *freeblock;                   /* block的空闲链表的开头         */
    struct pool_header *nextpool;       /* 指向下一个pool的指针(双向链表) */
    struct pool_header *prevpool;       /* 上一个的指针       ""        */
    uint arenaindex;                    /* 自己所属的arena的索引*/
    uint szidx;                         /* 分配的block的大小       */
    uint nextoffset;                    /* 到下一个block的位移        */
    uint maxnextoffset;                 /* 最大有效偏移     */
};

这个将pool连接,保持pool内部block的大小和个数

下面是生成arena的代码

static struct arena_object*
new_arena(void)
{
    struct arena_object* arenaobj;
    uint excess;        /* number of bytes above pool alignment */
    void *address;
    int err;
#ifdef PYMALLOC_DEBUG
    if (Py_GETENV("PYTHONMALLOCSTATS"))
        _PyObject_DebugMallocStats();
#endif
    if (unused_arena_objects == NULL) {
        uint i;
        uint numarenas;
        size_t nbytes;
        /* maxarenas指的是arenas数组现有的元素数量,刚开始调用时,自然maxarenas为0
         * Note that it's possible for `numarenas` to overflow.
         */
        numarenas = maxarenas ? maxarenas << 1 : INITIAL_ARENA_OBJECTS;
        if (numarenas <= maxarenas)
            return NULL;                /* overflow */
#if SIZEOF_SIZE_T <= SIZEOF_INT
        if (numarenas > PY_SIZE_MAX / sizeof(*arenas))
            return NULL;                /* overflow */
#endif
        nbytes = numarenas * sizeof(*arenas);
        arenaobj = (struct arena_object *)realloc(arenas, nbytes);
        if (arenaobj == NULL)
            return NULL;
        arenas = arenaobj;
        /* We might need to fix pointers that were copied.  However,
         * new_arena only gets called when all the pages in the
         * previous arenas are full.  Thus, there are *no* pointers
         * into the old array. Thus, we don't have to worry about
         * invalid pointers.  Just to be sure, some asserts:
         */
        assert(usable_arenas == NULL);
        assert(unused_arena_objects == NULL);
        /* Put the new arenas on the unused_arena_objects list. */
        for (i = maxarenas; i < numarenas; ++i) {
            arenas[i].address = 0;              /* mark as unassociated */
            arenas[i].nextarena = i < numarenas - 1 ?
                                   &arenas[i+1] : NULL;
        }
        /* Update globals. */
        unused_arena_objects = &arenas[maxarenas];
        maxarenas = numarenas;
    }
    /* Take the next available arena object off the head of the list. */
    assert(unused_arena_objects != NULL);
    arenaobj = unused_arena_objects;
    unused_arena_objects = arenaobj->nextarena;
    assert(arenaobj->address == 0);
#ifdef ARENAS_USE_MMAP
    address = mmap(NULL, ARENA_SIZE, PROT_READ|PROT_WRITE,
                   MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
    err = (address == MAP_FAILED);
#else
    address = malloc(ARENA_SIZE);
    err = (address == 0);
#endif    
    if (err) {
        /* The allocation failed: return NULL after putting the
         * arenaobj back.
         */
        arenaobj->nextarena = unused_arena_objects;
        unused_arena_objects = arenaobj;
        return NULL;
    }
    arenaobj->address = (uptr)address;
    ++narenas_currently_allocated;
#ifdef PYMALLOC_DEBUG
    ++ntimes_arena_allocated;
    if (narenas_currently_allocated > narenas_highwater)
        narenas_highwater = narenas_currently_allocated;
#endif
    arenaobj->freepools = NULL;
    /* pool_address <- first pool-aligned address in the arena
       nfreepools <- number of whole pools that fit after alignment */
    arenaobj->pool_address = (block*)arenaobj->address;
    arenaobj->nfreepools = ARENA_SIZE / POOL_SIZE;
    assert(POOL_SIZE * arenaobj->nfreepools == ARENA_SIZE);
    excess = (uint)(arenaobj->address & POOL_SIZE_MASK);
    if (excess != 0) {
        --arenaobj->nfreepools;
        arenaobj->pool_address += POOL_SIZE - excess;
    }
    arenaobj->ntotalpools = arenaobj->nfreepools;
    return arenaobj;
}

unused_arena_objects这个指的是

static struct arena_object* unused_arena_objects = NULL;

表示的是现在未使用的arena_object的单向链表(包含新生成和已废弃的)

原文发布于微信公众号 - 鸿的学习笔记(shujuxuexizhilu)

原文发表时间:2017-07-18

本文参与腾讯云自媒体分享计划,欢迎正在阅读的你也加入,一起分享。

发表于

我来说两句

0 条评论
登录 后参与评论

相关文章

来自专栏岑玉海

RavenDb学习(七) 异步工作以及维度查询

1、异步执行 var entity = new Company {Name = "Async Company #2", Id = "companies/2"}...

2935
来自专栏代码拾遗

JPA 详解

Java Persistence API(JPA)是将Java对象和关系型数据库对象映射起来规范。实现这个规范后开发者可以使用相同的代码可以在任意的数据库中执行...

1662
来自专栏木宛城主

Thinking In Design Pattern——Query Object模式

什么是Query Object模式 Query Object的架构设计 Query Object在服务层的应用 测试 Query Obj...

2156
来自专栏码匠的流水账

聊聊pg jdbc statement的maxRows参数

postgresql-9.4.1212.jre7-sources.jar!/org/postgresql/core/v3/QueryExecutorImpl.j...

912
来自专栏Spark生态圈

[Spark SQL] 源码解析之Optimizer

optimizer 以及之后的模块都只会在触发了action操作后才会执行。优化器是用来将Resolved LogicalPlan转化为optimized Lo...

992
来自专栏跟着阿笨一起玩NET

ZPL打印中文信息

  相信各位在实际的项目中,需要开发打条码模块的也会有不少,很多同行肯定也一直觉得斑马打印机很不错,但是ZPL打印中文字符很麻烦。如果购买字体卡,或者通过COD...

4131
来自专栏陈满iOS

iOS开发·runtime原理与实践: 关联对象篇(Associated Object)(应用场景:为分类添加“属性”,为UI控件关联事件Block体,为了不重复获得某种数据)

分类(category)与关联对象(Associated Object)作为objective-c的扩展机制的两个特性:分类,可以通过它来扩展方法;Associ...

3632
来自专栏跟着阿笨一起玩NET

ASP.NET 存储过程操作

存储过程是存放在数据库服务器上的预先编译好的sql语句。使用存储过程,可以直接在数据库中存储并运行功能强大的任务。存储过程在第一应用程序执行时进行语法检查和编...

1311
来自专栏24K纯开源

RegQueryValueEx正确使用方法

      项目中需要读取注册表中的HKEY_CLASSES_ROOT主键下一个子键的值,看了看MSDN的说明,有RegOpenKeyEx和RegQueryVa...

2508
来自专栏Jerry的SAP技术分享

如何在ABAP里用函数式编程思想打印出非波拉契Fibonacci(数列)

在JavaScript里可以用ES6提供的FunctionGenerator这种黑科技来打印非波拉契数列,具体细节参考我这篇文章。

1083

扫码关注云+社区

领取腾讯云代金券