前往小程序,Get更优阅读体验!
立即前往
首页
学习
活动
专区
工具
TVP
发布
社区首页 >专栏 >tampercfg内核模块导致机器频繁crash

tampercfg内核模块导致机器频繁crash

原创
作者头像
johnazhang
发布2022-08-10 12:13:57
1.7K1
发布2022-08-10 12:13:57
举报
文章被收录于专栏:Linux问题笔记Linux问题笔记

该问题,疑似rootkit或者哪个软件捆绑安装的模块(apache?),不过谷歌并搜不到相关信息。

【如何判断】

  1. 查看dmesg,堆栈有tampercore、tampercfg内核模块打印

2. 如果没有直接打印出tampercore、tampercfg,但是每次重启都是不同的位置踩内存(比如BUG、gpf等)

打开vmcore 后执行mod -t命令可以看到tampercfg、tampercore模块:

代码语言:javascript
复制
crash> mod -t
NAME        TAINTS
tampercfg   OE
tampercore  OE

【解决方案】

这个内核模块用rmmod是无法完全清理掉的,过一段时间又会自己装回来(目前暂无环境继续分析是谁装回来的)。

所以建议直接备份重装、并使用强密码

【其他一些细节】

这个模块会hook系统调用:

代码语言:javascript
复制
crash> p sys_call_table
sys_call_table = $2 = 
/*
* 提示:该行代码过长,系统自动注释不进行高亮。一键复制会移除系统注释 
* {0xffffffff95c4de60, 0xffffffffc0a007d0, 0xffffffffc0a00d30, 0xffffffff95c4c1f0, 0xffffffff95c534d0, 0xffffffff95c53500, 0xffffffff95c534e0, 0xffffffff95c64e40, 0xffffffff95c4cc80, 0xffffffff95a31ef0, 0xffffffff95bfea30, 0xffffffff95bfcce0, 0xffffffff95bfbf40, 0xffffffff95ab5430, 0xffffffff95ab42f0, 0xffffffff961934b0, 0xffffffff95c62ad0, 0xffffffff95c4e040, 0xffffffff95c4e100, 0xffffffff95c4e620, 0xffffffff95c4e730, 0xffffffff95c4b6f0, 0xffffffff95c57c30, 0xffffffff95c644a0, 0xffffffff95ada780, 0xffffffff95bffae0, 0xffffffff95c00200, 0xffffffff95bf74d0, 0xffffffff95becf50, 0xffffffff95cf50c0, 0xffffffff95cf58e0, 0xffffffff95cf5150, 0xffffffff95c6ec30, 0xffffffff95c6ebd0, 0xffffffff95ab57d0, 0xffffffff95acb600, 0xffffffff95aa3430, 0xffffffff95aafa30, 0xffffffff95aa3810, 0xffffffff95ab8bd0, 0xffffffff95c4ee80, 0xffffffff96035190, 0xffffffff960355b0, 0xffffffff96035590, 0xffffffff960355e0, 0xffffffff96035610, 0xffffffff96035960, 0xffffffff96035bd0, 0xffffffff96035830, 0xffffffff960354c0, 0xffffffff960354d0, 0xffffffff960355c0, 0xffffffff960355d0, 0xffffffff96035280, 0xffffffff96035640, 0xffffffff96035740, 0xffffffff96193270, 0xffffffff961932e0, 0xffffffff96193350, 0xffffffff96193430, 0xffffffff95aa2b70, 0xffffffff95aa2e10, 0xffffffff95ab4bb0, 0xffffffff95ab91e0, 0xffffffff95cf3720, 0xffffffff95cf3a60, 0xffffffff95cf37c0, 0xffffffff95cf5930, 0xffffffff95cf0320, 0xffffffff95cf07e0, 0xffffffff95cf0d70, 0xffffffff95cf03a0, 0xffffffff95c61200, 0xffffffff95cad390, 0xffffffff95c83230, 0xffffffff95c83250, 0xffffffff95c4b3c0, 0xffffffff95c4b3f0, 0xffffffff95c630a0, 0xffffffff95c699a0, 0xffffffff95c4b710, 0xffffffff95c4b7e0, 0xffffffffc0a00ad0, 0xffffffffc0a01140, 0xffffffffc0a00e80, 0xffffffffc0a00790, 0xffffffff95c60e70, 0xffffffffc0a009a0, 0xffffffff95c60b40, 0xffffffff95c53660, 0xffffffff95c4bb30, 0xffffffff95c4b9f0, 0xffffffff95c4bc70, 0xffffffff95c4bcc0, 0xffffffff95c4bc90, 0xffffffff95ab9e30, 0xffffffff95aa4090, 0xffffffff95ab9a30, 0xffffffff95ab9de0, 0xffffffff95aba3c0, 0xffffffff95ab8dd0, 0xffffffff95aab8d0, 0xffffffff95ab8c50, 0xffffffff95a9e0e0, 0xffffffff95ab8cb0, 0xffffffff95ab8420, 0xffffffff95ab8190, 0xffffffff95ab8c80, 0xffffffff95ab8ce0, 0xffffffff95ab8e60, 0xffffffff95ab8c20, 0xffffffff95ab9090, 0xffffffff95ab9120, 0xffffffff95ab8270, 0xffffffff95ab8010, 0xffffffff95acea40, 0xffffffff95aceb80, 0xffffffff95ab8540, 0xffffffff95ab8750, 0xffffffff95ab87f0, 0xffffffff95ab8980, 0xffffffff95ab9020, 0xffffffff95ab8a20, 0xffffffff95ab8b00, 0xffffffff95ab90b0, 0xffffffff95aaa070, 0xffffffff95aaa210, 0xffffffff95ab4410, 0xffffffff95ab4ba0, 0xffffffff95ab4c20, 0xffffffff95ab5890, 0xffffffff95ab4fd0, 0xffffffff95c83760, 0xffffffff95c60870, 0xffffffff95ac78b0, 0xffffffff95a9b7e0, 0xffffffff95c84840, 0xffffffff95c846c0, 0xffffffff95c846e0, 0xffffffff95c6f2b0, 0xffffffff95ab79d0, 0xffffffff95ab7750, 0xffffffff95ada070, 0xffffffff95ada2e0, 0xffffffff95ada040, 0xffffffff95ada260, 0xffffffff95ada850, 0xffffffff95ada890, 0xffffffff95ada8d0, 0xffffffff95bf8880, 0xffffffff95bf88e0, 0xffffffff95bf8960, 0xffffffff95bf8b00, 0xffffffff95c4c240, 0xffffffff95a30440, 0xffffffff95c73e70, 0xffffffff95aa9bf0, 0xffffffff95ab9e80, 0xffffffff95a2c1f0, 0xffffffff95aa4350, 0xffffffff95ab9cd0, 0xffffffff95c4b8b0, 0xffffffff95c83080, 0xffffffff95b21530, 0xffffffff95aa4260, 0xffffffff95c73ce0, 0xffffffff95c71cb0, 0xffffffff95c0e380, 0xffffffff95c0dca0, 0xffffffff95ab7fb0, 0xffffffff95ab9410, 0xffffffff95ab95e0, 0xffffffff961933c0, 0xffffffff95a2fc70, 0xffffffff95ac78b0, 0xffffffff95b1f2f0, 0xffffffff95b1c320, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95cc20c0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95ab8c00, 0xffffffff95bcbd10, 0xffffffff95c76ee0, 0xffffffff95c76fe0, 0xffffffff95c770d0, 0xffffffff95c771e0, 0xffffffff95c772a0, 0xffffffff95c77350, 0xffffffff95c77420, 0xffffffff95c774d0, 0xffffffff95c77570, 0xffffffff95c77640, 0xffffffff95c77710, 0xffffffff95c777e0, 0xffffffff95ab4bf0, 0xffffffff95aa3fc0, 0xffffffff95b15bb0, 0xffffffff95adbf00, 0xffffffff95ada690, 0xffffffff95ac78b0, 0xffffffff95ca45c0, 0xffffffff95ca4950, 0xffffffff95ca5470, 0xffffffff95ca52f0, 0xffffffff95ca5310, 0xffffffff95ac78b0, 0xffffffff95ce0c50, 0xffffffff95c9ca00, 0xffffffff95ac78b0, 0xffffffff95ac78b0, 0xffffffff95bea0e0, 0xffffffff95c631d0, 0xffffffff95a9aba0, 0xffffffff95ab4060, 0xffffffff95cf3a50, 0xffffffff95bc30e0, 0xffffffff95ac5460, 0xffffffff95ac5ae0, 0xffffffff95ac5960, 0xffffffff95ac5a70, 0xffffffff95ac5cb0, 0xffffffff95ac5ec0, 0xffffffff95ac5f70, 0xffffffff95ac6030, 0xffffffff95ac60e0, 0xffffffff95aa2c30, 0xffffffff95c9d6b0, 0xffffffff95c9ca30, 0xffffffff95ab4bc0, 0xffffffff95c839d0, 0xffffffff95ac78b0, 0xffffffff95c1ce30, 0xffffffff95c1b950, 0xffffffff95c1b970, 0xffffffff95cf73f0, 0xffffffff95cf7790, 0xffffffff95cf7910, 0xffffffff95cf7c90, 0xffffffff95cf81d0, 0xffffffff95cf8620, 0xffffffff95b23560, 0xffffffff95aa2c80, 0xffffffff95cfc1c0, 0xffffffff95cfc3e0, 0xffffffff95cfdc40, 0xffffffff95c93dc0, 0xffffffff95c940f0, 0xffffffff95c993e0, 0xffffffff95c99400, 0xffffffff95c99750...}
*/
crash> sym 0xffffffff95c4de60
ffffffff95c4de60 (T) sys_read /usr/src/debug/kernel-3.10.0-1127.19.1.el7/linux-3.10.0-1127.19.1.el7.x86_64/fs/read_write.c: 569
crash> sym 0xffffffffc0a00d30
ffffffffc0a00d30 (t) tamperhack_open [tampercore] 

启动加载日志可以看到:

代码语言:javascript
复制
crash> log | grep tainting
[   15.908686] tampercfg: module verification failed: signature and/or required key missing - tainting kernel

一次踩内存的分析举例:

代码语言:javascript
复制
crash> bt
PID: 3141   TASK: ffff9413fba320e0  CPU: 3   COMMAND: "YDService"
 #0 [ffff9413fba7b960] machine_kexec at ffffffff95a66254 
 #1 [ffff9413fba7b9c0] __crash_kexec at ffffffff95b22ef2 
 #2 [ffff9413fba7ba90] crash_kexec at ffffffff95b22fe0 
 #3 [ffff9413fba7baa8] oops_end at ffffffff9618a798 
 #4 [ffff9413fba7bad0] no_context at ffffffff95a75d74 
 #5 [ffff9413fba7bb20] __bad_area_nosemaphore at ffffffff95a76042 
 #6 [ffff9413fba7bb70] bad_area_nosemaphore at ffffffff95a76164 
 #7 [ffff9413fba7bb80] __do_page_fault at ffffffff9618d750 
 #8 [ffff9413fba7bbf0] trace_do_page_fault at ffffffff9618da26 
 #9 [ffff9413fba7bc30] do_async_page_fault at ffffffff9618cfa2 
#10 [ffff9413fba7bc50] async_page_fault at ffffffff961897a8 
    [exception RIP: exit_creds+27]
    RIP: ffffffff95acd8ab  RSP: ffff9413fba7bd08  RFLAGS: 00010246
    RAX: 0000000000000000  RBX: ffff9414409f20e0  RCX: 0000000000000000
    RDX: ffffffff966c0ce0  RSI: ffff9414409f20e0  RDI: 0000000000000000
    RBP: ffff9413fba7bd10   R8: 0000000000000000   R9: ffff94146b229c00
    R10: 00000000000002b1  R11: 0000000000000000  R12: ffff9414409f20e0
    R13: ffff9414409f20e0  R14: 0000000000000001  R15: ffffa23101b3a170
    ORIG_RAX: ffffffffffffffff  CS: 0010  SS: 0018
#11 [ffff9413fba7bd18] __put_task_struct at ffffffff95a9839a 
#12 [ffff9413fba7bd38] wake_up_q at ffffffff95adb960 
#13 [ffff9413fba7bd60] futex_wake_op at ffffffff95b13193 
#14 [ffff9413fba7be48] do_futex at ffffffff95b1578a 
#15 [ffff9413fba7bed8] sys_futex at ffffffff95b15c30 
#16 [ffff9413fba7bf50] system_call_fastpath at ffffffff96192ed2 
    RIP: 00007f49c18200f6  RSP: 00007f4990060de8  RFLAGS: 00000202
    RAX: 00000000000000ca  RBX: 00000000025cdbe0  RCX: 0000000000000000
    RDX: 0000000000000001  RSI: 0000000000000085  RDI: 00000000025cdc5c
    RBP: 0000000000511800   R8: 00000000025cdc58   R9: 0000000004000001
    R10: 0000000000000001  R11: 0000000000000283  R12: 0000000000000400
    R13: 00007f4990060e50  R14: 00000000025b8620  R15: 0000000000000000
    ORIG_RAX: 00000000000000ca  CS: 0033  SS: 002b
	
crash> dis -r ffffffff95acd8ab
0xffffffff95acd890 <exit_creds>:        nopl   0x0(%rax,%rax,1) [FTRACE NOP]
0xffffffff95acd895 <exit_creds+5>:      push   %rbp
0xffffffff95acd896 <exit_creds+6>:      mov    %rsp,%rbp
0xffffffff95acd899 <exit_creds+9>:      push   %rbx
0xffffffff95acd89a <exit_creds+10>:     mov    0x670(%rdi),%rax
0xffffffff95acd8a1 <exit_creds+17>:     mov    %rdi,%rbx
0xffffffff95acd8a4 <exit_creds+20>:     mov    0x668(%rdi),%rdi
0xffffffff95acd8ab <exit_creds+27>:     mov    (%rax),%eax

rax、rdi是0,对应的task结构体被置空
	
crash> wake_q_head ffff9413fba7bdd0
struct wake_q_head {
  first = 0xffff9414409f30f8, 
  lastp = 0xffff9414409f30f8
}
crash> wake_q_node 0xffff9414409f30f8
struct wake_q_node {
  next = 0x0
}

这里为0不符合预期

对应代码:
void wake_up_q(struct wake_q_head *head)
{       
        struct wake_q_node *node = head->first;

        while (node != WAKE_Q_TAIL) {
                struct task_struct *task;

                task = container_of(node, struct task_struct, wake_q);
                BUG_ON(!task);	//注意,这里通过了BUG检查,说明这时候还不是0
                /* task can safely be re-inserted now */
                node = node->next;
                task->wake_q.next = NULL;
                
                /*
                 * wake_up_process() implies a wmb() to pair with the queueing
                 * in wake_q_add() so as not to miss wakeups.
                 */
                wake_up_process(task);
                put_task_struct(task);	//这里进入这个函数后,就再没有task为0的检查,后面被并发改成0了
        }
}

所以就是这个内核模块本身写的也有点问题,会去并发的改task结构体,导致其他的正常执行逻辑里,task为0跪了。

原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。

如有侵权,请联系 cloudcommunity@tencent.com 删除。

原创声明:本文系作者授权腾讯云开发者社区发表,未经许可,不得转载。

如有侵权,请联系 cloudcommunity@tencent.com 删除。

评论
登录后参与评论
0 条评论
热度
最新
推荐阅读
相关产品与服务
云服务器
云服务器(Cloud Virtual Machine,CVM)提供安全可靠的弹性计算服务。 您可以实时扩展或缩减计算资源,适应变化的业务需求,并只需按实际使用的资源计费。使用 CVM 可以极大降低您的软硬件采购成本,简化 IT 运维工作。
领券
问题归档专栏文章快讯文章归档关键词归档开发者手册归档开发者手册 Section 归档