Linux进程的内存管理之缺页异常

ENTRY(vectors)
 kernel_ventry 1, sync_invalid   // Synchronous EL1t
 kernel_ventry 1, irq_invalid   // IRQ EL1t
 kernel_ventry 1, fiq_invalid   // FIQ EL1t
 kernel_ventry 1, error_invalid  // Error EL1t

 kernel_ventry 1, sync    // Synchronous EL1h
 kernel_ventry 1, irq    // IRQ EL1h
 kernel_ventry 1, fiq_invalid   // FIQ EL1h
 kernel_ventry 1, error_invalid  // Error EL1h

 kernel_ventry 0, sync    // Synchronous 64-bit EL0
 kernel_ventry 0, irq    // IRQ 64-bit EL0
 kernel_ventry 0, fiq_invalid   // FIQ 64-bit EL0
 kernel_ventry 0, error_invalid  // Error 64-bit EL0

#ifdef CONFIG_COMPAT
 kernel_ventry 0, sync_compat, 32  // Synchronous 32-bit EL0
 kernel_ventry 0, irq_compat, 32  // IRQ 32-bit EL0
 kernel_ventry 0, fiq_invalid_compat, 32 // FIQ 32-bit EL0
 kernel_ventry 0, error_invalid_compat, 32 // Error 32-bit EL0
#else
 kernel_ventry 0, sync_invalid, 32  // Synchronous 32-bit EL0
 kernel_ventry 0, irq_invalid, 32  // IRQ 32-bit EL0
 kernel_ventry 0, fiq_invalid, 32  // FIQ 32-bit EL0
 kernel_ventry 0, error_invalid, 32  // Error 32-bit EL0
#endif
END(vectors)

el1_sync:
 kernel_entry 1
 mrs x1, esr_el1   // read the syndrome register
 lsr x24, x1, #ESR_ELx_EC_SHIFT // exception class
 cmp x24, #ESR_ELx_EC_DABT_CUR // data abort in EL1
 b.eq el1_da
 cmp x24, #ESR_ELx_EC_IABT_CUR // instruction abort in EL1
 b.eq el1_ia
 cmp x24, #ESR_ELx_EC_SYS64  // configurable trap
 b.eq el1_undef
 cmp x24, #ESR_ELx_EC_SP_ALIGN // stack alignment exception
 b.eq el1_sp_pc
 cmp x24, #ESR_ELx_EC_PC_ALIGN // pc alignment exception
 b.eq el1_sp_pc
 cmp x24, #ESR_ELx_EC_UNKNOWN // unknown exception in EL1
 b.eq el1_undef
 cmp x24, #ESR_ELx_EC_BREAKPT_CUR // debug exception in EL1
 b.ge el1_dbg
 b el1_inv

asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr,
      struct pt_regs *regs)
{
 const struct fault_info *inf = esr_to_fault_info(esr);
 struct siginfo info;

 if (!inf->fn(addr, esr, regs))
  return;

 pr_alert("Unhandled fault: %s (0x%08x) at 0x%016lx\n",
   inf->name, esr, addr);

 mem_abort_decode(esr);

 info.si_signo = inf->sig;
 info.si_errno = 0;
 info.si_code  = inf->code;
 info.si_addr  = (void __user *)addr;
 arm64_notify_die("", regs, &info, esr);
}

static const struct fault_info fault_info[] = {
 { do_bad,  SIGBUS,  0,  "ttbr address size fault" },
 { do_bad,  SIGBUS,  0,  "level 1 address size fault" },
 { do_bad,  SIGBUS,  0,  "level 2 address size fault" },
 { do_bad,  SIGBUS,  0,  "level 3 address size fault" },
 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" },
 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" },
 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" },
 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" },
 { do_bad,  SIGBUS,  0,  "unknown 8"   },
 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" },
 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" },
 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" },
  ......
}

static int __kprobes do_translation_fault(unsigned long addr,
       unsigned int esr,
       struct pt_regs *regs)
{
 if (addr < TASK_SIZE)
  return do_page_fault(addr, esr, regs); //用户空间

 do_bad_area(addr, esr, regs); //内核空间或非法空间
 return 0;
}

static int __do_page_fault(struct mm_struct *mm, unsigned long addr,
      unsigned int mm_flags, unsigned long vm_flags,
      struct task_struct *tsk)
{
 struct vm_area_struct *vma;
 int fault;

 vma = find_vma(mm, addr);
 fault = VM_FAULT_BADMAP; //没有找到vma区域，说明addr还没有在进程的地址空间中
 if (unlikely(!vma))
  goto out;
 if (unlikely(vma->vm_start > addr))
  goto check_stack;

 /*
  * Ok, we have a good vm_area for this memory access, so we can handle
  * it.
  */
good_area://一个好的vma
 /*
  * Check that the permissions on the VMA allow for the fault which
  * occurred.
  */
 if (!(vma->vm_flags & vm_flags)) {//权限检查
  fault = VM_FAULT_BADACCESS; 
  goto out;
 }

 //重新建立物理页面到VMA的映射关系
 return handle_mm_fault(vma, addr & PAGE_MASK, mm_flags);

check_stack:
 if (vma->vm_flags & VM_GROWSDOWN && !expand_stack(vma, addr))
  goto good_area;
out:
 return fault;
}

static int __handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
  unsigned int flags)
{
  ......
 //查找页全局目录，获取地址对应的表项
 pgd = pgd_offset(mm, address);
 //查找页四级目录表项，没有则创建
 p4d = p4d_alloc(mm, pgd, address);
 if (!p4d)
  return VM_FAULT_OOM;

 //查找页上级目录表项，没有则创建
 vmf.pud = pud_alloc(mm, p4d, address);
 ......
 //查找页中级目录表项，没有则创建
 vmf.pmd = pmd_alloc(mm, vmf.pud, address);
  ......
 //处理pte页表
 return handle_pte_fault(&vmf);
}

static int handle_pte_fault(struct vm_fault *vmf)
{
  ......
 //页表项不存在
 if (!vmf->pte) {
  //判断是否是匿名页
  if (vma_is_anonymous(vmf->vma))
   //处理匿名页   
   return do_anonymous_page(vmf); //malloc/mmap分配了vma，但是没有进行映射处理，在首次访问时触发
  else
   //处理文件页，匿名共享页
   return do_fault(vmf); 
 }

 //页表项存在，但页不在内存中
 if (!pte_present(vmf->orig_pte))
  return do_swap_page(vmf);

 if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
  return do_numa_page(vmf); //NUMA自动平衡处理
  ......
 if (vmf->flags & FAULT_FLAG_WRITE) {
  if (!pte_write(entry))
   return do_wp_page(vmf); //页在内存中，但是没有写权限位，写时复制
  entry = pte_mkdirty(entry);
 }
  ......
}

/*
 * Empty_zero_page is a special page that is used for zero-initialized data
 * and COW.
 */
unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] __page_aligned_bss;
EXPORT_SYMBOL(empty_zero_page);

 //是否为写内存导致的缺页异常
 if (!(vmf->flags & FAULT_FLAG_WRITE) &&
   !mm_forbids_zeropage(vma->vm_mm)) {
  //异常由读操作触发，并允许使用0页。设置页表项的值映射到０页。
  entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
      vma->vm_page_prot));
  vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
    vmf->address, &vmf->ptl);
  if (!pte_none(*vmf->pte))
   goto unlock;
  ret = check_stable_address_space(vma->vm_mm);
  if (ret)
   goto unlock;
  /* Deliver the page fault to userland, check inside PT lock */
  if (userfaultfd_missing(vma)) {
   pte_unmap_unlock(vmf->pte, vmf->ptl);
   return handle_userfault(vmf, VM_UFFD_MISSING);
  }
  goto setpte;
 }
  ......
setpte:
 //设置页表项
 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);

 /* No need to invalidate - it was non-present before */
 //更新cpu的tlb页表缓存
 update_mmu_cache(vma, vmf->address, vmf->pte);

 //分配一个高端可迁移的被０填充的物理页
 page = alloc_zeroed_user_highpage_movable(vma, vmf->address);
 if (!page)
  goto oom;
  ......
 //设置page的标志位，PG_uptodate
 __SetPageUptodate(page);

 //根据vma的权限位和page描述符，生成页表项
 entry = mk_pte(page, vma->vm_page_prot);
 if (vma->vm_flags & VM_WRITE)
  //如果有写权限，设置页表项值为脏且可写
  entry = pte_mkwrite(pte_mkdirty(entry));
  ......
 //增加匿名页面计数
 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
 //建立物理页到虚拟页的反向映射，添加到rmap链表中
 page_add_new_anon_rmap(page, vma, vmf->address, false);
 mem_cgroup_commit_charge(page, memcg, false, false);
 //把物理页添加到lru链表中
 lru_cache_add_active_or_unevictable(page, vma);

int do_swap_page(struct vm_fault *vmf)
{
  ......
 //根据pte找到swap entry, swap entry和pte有一个对应关系
 entry = pte_to_swp_entry(vmf->orig_pte);
  ......
 if (!page)
  //根据entry从swap缓存中查找页, 在swapcache里面寻找entry对应的page
  //Lookup a swap entry in the swap cache
  page = lookup_swap_cache(entry, vma_readahead ? vma : NULL,
      vmf->address);
 //没有找到页
 if (!page) {
  if (vma_readahead)
   page = do_swap_page_readahead(entry,
    GFP_HIGHUSER_MOVABLE, vmf, &swap_ra);
  else
   //如果swapcache里面找不到就在swap area里面找，分配新的内存页并从swap area中读入
   page = swapin_readahead(entry,
    GFP_HIGHUSER_MOVABLE, vma, vmf->address);
  ......
 //获取一个pte的entry，重新建立映射
 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
   &vmf->ptl);
  ......
 //anonpage数加1，匿名页从swap空间交换出来，所以加1
 //swap page个数减1，由page和VMA属性创建一个新的pte
 inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
 dec_mm_counter_fast(vma->vm_mm, MM_SWAPENTS);
 pte = mk_pte(page, vma->vm_page_prot);
  ......
 flush_icache_page(vma, page);
 if (pte_swp_soft_dirty(vmf->orig_pte))
  pte = pte_mksoft_dirty(pte);
 //将新生成的PTE entry添加到硬件页表中
 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, pte);
 vmf->orig_pte = pte;
 //根据page是否为swapcache
 if (page == swapcache) {
  //如果是，将swap缓存页用作anon页，添加反向映射rmap中
  do_page_add_anon_rmap(page, vma, vmf->address, exclusive);
  mem_cgroup_commit_charge(page, memcg, true, false);
  //并添加到active链表中
  activate_page(page);
 //如果不是
 } else { /* ksm created a completely new copy */
  //使用新页面并复制swap缓存页，添加反向映射rmap中
  page_add_new_anon_rmap(page, vma, vmf->address, false);
  mem_cgroup_commit_charge(page, memcg, false, false);
  //并添加到lru链表中
  lru_cache_add_active_or_unevictable(page, vma);
 }

 //释放swap entry
 swap_free(entry);
  ......
 if (vmf->flags & FAULT_FLAG_WRITE) {
  //有写请求则写时复制
  ret |= do_wp_page(vmf);
  if (ret & VM_FAULT_ERROR)
   ret &= VM_FAULT_ERROR;
  goto out;
 }
  ......
  return ret;
}

static int do_wp_page(struct vm_fault *vmf)
 __releases(vmf->ptl)
{
 struct vm_area_struct *vma = vmf->vma;

 //从页表项中得到页帧号，再得到页描述符，发生异常时地址所在的page结构
 vmf->page = vm_normal_page(vma, vmf->address, vmf->orig_pte);
 if (!vmf->page) {
  //没有page结构是使用页帧号的特殊映射
  /*
   * VM_MIXEDMAP !pfn_valid() case, or VM_SOFTDIRTY clear on a
   * VM_PFNMAP VMA.
   *
   * We should not cow pages in a shared writeable mapping.
   * Just mark the pages writable and/or call ops->pfn_mkwrite.
   */
  if ((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
         (VM_WRITE|VM_SHARED))
   //处理共享可写映射
   return wp_pfn_shared(vmf);

  pte_unmap_unlock(vmf->pte, vmf->ptl);
  //处理私有可写映射
  return wp_page_copy(vmf);
 }

 /*
  * Take out anonymous pages first, anonymous shared vmas are
  * not dirty accountable.
  */
 if (PageAnon(vmf->page) && !PageKsm(vmf->page)) {
  int total_map_swapcount;
  if (!trylock_page(vmf->page)) {
   //添加原来页的引用计数，方式被释放
   get_page(vmf->page);
   //释放页表锁
   pte_unmap_unlock(vmf->pte, vmf->ptl);
   lock_page(vmf->page);
   vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
     vmf->address, &vmf->ptl);
   if (!pte_same(*vmf->pte, vmf->orig_pte)) {
    unlock_page(vmf->page);
    pte_unmap_unlock(vmf->pte, vmf->ptl);
    put_page(vmf->page);
    return 0;
   }
   put_page(vmf->page);
  }
  //单身匿名页面的处理
  if (reuse_swap_page(vmf->page, &total_map_swapcount)) {
   if (total_map_swapcount == 1) {
    /*
     * The page is all ours. Move it to
     * our anon_vma so the rmap code will
     * not search our parent or siblings.
     * Protected against the rmap code by
     * the page lock.
     */
    page_move_anon_rmap(vmf->page, vma);
   }
   unlock_page(vmf->page);
   wp_page_reuse(vmf);
   return VM_FAULT_WRITE;
  }
  unlock_page(vmf->page);
 } else if (unlikely((vma->vm_flags & (VM_WRITE|VM_SHARED)) ==
     (VM_WRITE|VM_SHARED))) {
  //共享可写，不需要复制物理页，设置页表权限即可
  return wp_page_shared(vmf);
 }

 /*
  * Ok, we need to copy. Oh, well..
  */
 get_page(vmf->page);

 pte_unmap_unlock(vmf->pte, vmf->ptl);
 //私有可写，复制物理页，将虚拟页映射到物理页
 return wp_page_copy(vmf);
}