一则golang 内存溢出分析
由于自己的服务器配置比较差，上传一篇较长博客后，进存kill，显示内存溢出。每次重启后不久，都会进存kill掉的现象出现。 问题 上传了一篇比较长的文章后，goblog后台直接被killed掉，查看日志，内存溢出:
2019/11/15 15:38:04.729 [I] [router.go:266] /root/Project/src/goblog/src/controller/admin no changed
2019/11/15 15:38:04.729 [I] [router.go:266] /root/Project/src/goblog/src/controller no changed
[goblog] 2019/11/15 15:38:04.733401 user_mapper.go:57 [INFO] call GetByCondition rows:1
2019/11/15 15:38:05.109 [I] [asm_amd64.s:1333] http server Running on http://:9090
2019/11/15 15:38:05.127 [I] [asm_amd64.s:1333] Admin server Running on 127.0.0.1:8088
2019/11/15 15:38:08 Gse dictionary loaded finished.
fatal error: runtime: out of memory
runtime stack:
runtime.throw(0xbeb588, 0x16)
/root/go/src/runtime/panic.go:608 +0x72
runtime.sysMap(0xc050000000, 0x4000000, 0x138fb58)
/root/go/src/runtime/mem_linux.go:156 +0xc7
runtime.(*mheap).sysAlloc(0x1376320, 0x4000000, 0x1376338, 0x7f4c067ddb48)
/root/go/src/runtime/malloc.go:619 +0x1c7
runtime.(*mheap).grow(0x1376320, 0x2, 0x0)
/root/go/src/runtime/mheap.go:920 +0x42
runtime.(*mheap).allocSpanLocked(0x1376320, 0x2, 0x138fb68, 0x800)
/root/go/src/runtime/mheap.go:848 +0x337
runtime.(*mheap).alloc_m(0x1376320, 0x2, 0x75, 0x7f4c069e5fff)
/root/go/src/runtime/mheap.go:692 +0x119
runtime.(*mheap).alloc.func1()
/root/go/src/runtime/mheap.go:759 +0x4c
runtime.(*mheap).alloc(0x1376320, 0x2, 0x7f4c06010075, 0x7f4c067ddab0)
/root/go/src/runtime/mheap.go:758 +0x8a
runtime.(*mcentral).grow(0x1379398, 0x0)
/root/go/src/runtime/mcentral.go:232 +0x94
runtime.(*mcentral).cacheSpan(0x1379398, 0x7f4c067ddab0)
/root/go/src/runtime/mcentral.go:106 +0x2f8
runtime.(*mcache).refill(0x7f4c1358d000, 0x7f4c0cbd1875)
/root/go/src/runtime/mcache.go:122 +0x95
runtime.(*mcache).nextFree.func1()
/root/go/src/runtime/malloc.go:749 +0x32
runtime.systemstack(0x45a029)
/root/go/src/runtime/asm_amd64.s:351 +0x66
runtime.mstart()
/root/go/src/runtime/proc.go:1229
goroutine 65 [running]:
runtime.systemstack_switch()
/root/go/src/runtime/asm_amd64.s:311 fp=0xc01af6b400 sp=0xc01af6b3f8 pc=0x45a120
runtime.(*mcache).nextFree(0x7f4c1358d000, 0x203075, 0xc04fffa000, 0x7f4c067ddab0, 0xc01af6b401)
/root/go/src/runtime/malloc.go:748 +0xb6 fp=0xc01af6b458 sp=0xc01af6b400 pc=0x40d506
runtime.mallocgc(0x4000, 0x0, 0x7f4c0ced8b00, 0xc04fffa000)
/root/go/src/runtime/malloc.go:903 +0x793 fp=0xc01af6b4f8 sp=0xc01af6b458 pc=0x40de53
runtime.rawstring(0x3fca, 0x0, 0x0, 0x0, 0x0, 0x0)
/root/go/src/runtime/string.go:258 +0x4f fp=0xc01af6b528 sp=0xc01af6b4f8 pc=0x4491df
runtime.rawstringtmp(0x0, 0x3fca, 0xc005a5b4f0, 0x0, 0x1370e00, 0x3fc9, 0x7f4c1358d000)
/root/go/src/runtime/string.go:123 +0x72 fp=0xc01af6b568 sp=0xc01af6b528 pc=0x448bb2
runtime.concatstrings(0x0, 0xc01af6b648, 0x2, 0x2, 0x0, 0xc04ab3a830)
/root/go/src/runtime/string.go:49 +0xfd fp=0xc01af6b600 sp=0xc01af6b568 pc=0x44868d
runtime.concatstring2(0x0, 0xc04fffa000, 0x3fc9, 0xc045f674c9, 0x1, 0xc005a5b400, 0x0)
/root/go/src/runtime/string.go:58 +0x47 fp=0xc01af6b640 sp=0xc01af6b600 pc=0x4488b7
github.com/go-ego/riot.(*Engine).ForSplitData(0x1370740, 0xc048a70000, 0x52cf, 0x52cf, 0x52cf, 0x52cf, 0x52cf)
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:53 +0x141 fp=0xc01af6b710 sp=0xc01af6b640 pc=0x9e4671
github.com/go-ego/riot.(*Engine).splitData(0x1370740, 0x12cf0d9, 0x1, 0x2ab8b572, 0xc045f48a80, 0x6321, 0x0, 0x0, 0x0, 0x0, ...)
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:108 +0x32d fp=0xc01af6b8e8 sp=0xc01af6b710 pc=0x9e4e0d
github.com/go-ego/riot.(*Engine).segmenterData(0x1370740, 0x12cf0d9, 0x1, 0x2ab8b572, 0xc045f48a80, 0x6321, 0x0, 0x0, 0x0, 0x0, ...)
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:186 +0x2d2 fp=0xc01af6bb00 sp=0xc01af6b8e8 pc=0x9e54d2
github.com/go-ego/riot.(*Engine).makeTokensMap(0x1370740, 0x12cf0d9, 0x1, 0x2ab8b572, 0xc045f48a80, 0x6321, 0x0, 0x0, 0x0, 0x0, ...)
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:214 +0x4a4 fp=0xc01af6bc70 sp=0xc01af6bb00 pc=0x9e6284
github.com/go-ego/riot.(*Engine).segmenterWorker(0x1370740)
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/segment.go:261 +0x1b5 fp=0xc01af6bfd8 sp=0xc01af6bc70 pc=0x9e6465
runtime.goexit()
/root/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc01af6bfe0 sp=0xc01af6bfd8 pc=0x45c201
created by github.com/go-ego/riot.(*Engine).Init
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/engine.go:331 +0x52c
goroutine 1 [chan receive]:
github.com/astaxie/beego.(*App).Run(0xc000118130, 0x0, 0x0, 0x0)
/root/Project/pkg/mod/github.com/astaxie/beego@v1.9.2/app.go:221 +0x262
github.com/astaxie/beego.Run(0x0, 0x0, 0x0)
/root/Project/pkg/mod/github.com/astaxie/beego@v1.9.2/beego.go:67 +0x62
main.main()
/root/Project/src/goblog/main.go:12 +0x32
goroutine 5 [syscall]:
os/signal.signal_recv(0x0)
/root/go/src/runtime/sigqueue.go:139 +0x9c
os/signal.loop()
/root/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
/root/go/src/os/signal/signal_unix.go:29 +0x41
goroutine 6 [select]:
goblog/src/logs.async()
/root/Project/src/goblog/src/logs/logs.go:224 +0x108
created by goblog/src/logs.InitLogs
/root/Project/src/goblog/src/logs/logs.go:89 +0x761
goroutine 7 [chan receive]:
goblog/src/logs.(*logs).logCut(0xc000088550, 0xc00000e688)
/root/Project/src/goblog/src/logs/logs.go:111 +0x2a7
created by goblog/src/logs.InitLogs
/root/Project/src/goblog/src/logs/logs.go:91 +0x724
goroutine 8 [select]:
database/sql.(*DB).connectionOpener(0xc0000f00c0, 0xcb23e0, 0xc000061d40)
/root/go/src/database/sql/sql.go:1001 +0xe8
created by database/sql.OpenDB
/root/go/src/database/sql/sql.go:671 +0x15d
goroutine 9 [select]:
database/sql.(*DB).connectionResetter(0xc0000f00c0, 0xcb23e0, 0xc000061d40)
/root/go/src/database/sql/sql.go:1014 +0xfb
created by database/sql.OpenDB
/root/go/src/database/sql/sql.go:672 +0x193
goroutine 13 [select]:
github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher.func1(0xc000062d20, 0xc0000f0180, 0xc00002a5a0)
/root/Project/pkg/mod/github.com/go-sql-driver/mysql@v1.4.0/connection_go18.go:179 +0xbf
created by github.com/go-sql-driver/mysql.(*mysqlConn).startWatcher
/root/Project/pkg/mod/github.com/go-sql-driver/mysql@v1.4.0/connection_go18.go:176 +0xbe
goroutine 14 [select]:
database/sql.(*DB).connectionCleaner(0xc0000f00c0, 0x1a3185c5000)
/root/go/src/database/sql/sql.go:899 +0x37d
created by database/sql.(*DB).startCleanerLocked
/root/go/src/database/sql/sql.go:886 +0xa7
goroutine 19 [chan receive]:
github.com/go-ego/riot.(*Engine).Store(0x1370740)
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/engine.go:212 +0x48e
github.com/go-ego/riot.(*Engine).Init(0x1370740, 0x0, 0x3, 0xbc5db6, 0x2, 0x0, 0x0, 0x0, 0x0, 0x1, ...)
/root/Project/pkg/mod/github.com/go-ego/riot@v0.0.0-20181222142112-75a2cab10779/engine.go:351 +0x720
goblog/src/service.init.2.func1()
/root/Project/src/goblog/src/service/search_service.go:55 +0xba
created by goblog/src/component.GoRoutine
/root/Project/src/goblog/src/component/routine_component.go:9 +0x33
goroutine 63 [IO wait]:
internal/poll.runtime_pollWait(0x7f4c13531d60, 0x72, 0x0)
/root/go/src/runtime/netpoll.go:173 +0x66
internal/poll.(*pollDesc).wait(0xc0000ef998, 0x72, 0xc000060000, 0x0, 0x0)
/root/go/src/internal/poll/fd_poll_runtime.go:85 +0x9a
internal/poll.(*pollDesc).waitRead(0xc0000ef998, 0xffffffffffffff00, 0x0, 0x0)
/root/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Accept(0xc0000ef980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/root/go/src/internal/poll/fd_unix.go:384 +0x1a0
net.(*netFD).accept(0xc0000ef980, 0x1370e00, 0x30, 0x30)
/root/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0001c84b0, 0xc000052d18, 0x7f4c1358d000, 0xc000153500)
/root/go/src/net/tcpsock_posix.go:139 +0x2e
net.(*TCPListener).AcceptTCP(0xc0001c84b0, 0x40e238, 0x30, 0xb67640)
/root/go/src/net/tcpsock.go:247 +0x47
net/http.tcpKeepAliveListener.Accept(0xc0001c84b0, 0xb67640, 0xc000df7da0, 0xaedd40, 0x135c100)
/root/go/src/net/http/server.go:3232 +0x2f
net/http.(*Server).Serve(0xc0000ff860, 0xcb21e0, 0xc0001c84b0, 0x0, 0x0)
/root/go/src/net/http/server.go:2826 +0x22f
net/http.(*Server).ListenAndServe(0xc0000ff860, 0xc0000ff860, 0x26)
/root/go/src/net/http/server.go:2764 +0xb6
net/http.ListenAndServe(0xc000e00450, 0xe, 0x0, 0x0, 0x1, 0xc000e00450)
/root/go/src/net/http/server.go:3004 +0x74
github.com/astaxie/beego.(*adminApp).Run(0xc00000e648)
/root/Project/pkg/mod/github.com/astaxie/beego@v1.9.2/admin.go:399 +0x358
created by github.com/astaxie/beego.registerAdmin
/root/Project/pkg/mod/github.com/astaxie/beego@v1.9.2/hooks.go:89 +0x67
...分析
猜测是线程数配置的问题，查看engine.go的Init方法中各参数的作用，如下:
func (engine *Engine) Init(options types.EngineOpts) {
// 将线程数设置为CPU数
// runtime.GOMAXPROCS(runtime.NumCPU())
// runtime.GOMAXPROCS(128)
// 初始化初始参数
if engine.initialized {
log.Fatal("Do not re-initialize the engine.")
}
options = engine.initDef(options)
options.Init()
engine.initOptions = options
engine.initialized = true
if !options.NotUseGse {
if !engine.loaded {
// 载入分词器词典
engine.segmenter.LoadDict(options.GseDict)
engine.loaded = true
}
// 初始化停用词
engine.stopTokens.Init(options.StopTokenFile)
}
// 初始化索引器和排序器
for shard := 0; shard < options.NumShards; shard++ {
engine.indexers = append(engine.indexers, core.Indexer{})
engine.indexers[shard].Init(*options.IndexerOpts)
engine.rankers = append(engine.rankers, core.Ranker{})
engine.rankers[shard].Init(options.IDOnly)
}
// 初始化分词器通道
engine.segmenterChan = make(
chan segmenterReq, options.NumGseThreads)
// 初始化索引器通道
engine.Indexer(options)
// 初始化排序器通道
engine.Ranker(options)
// engine.CheckMem(engine.initOptions.UseStore)
engine.CheckMem()
// 初始化持久化存储通道
if engine.initOptions.UseStore {
engine.InitStore()
}
// 启动分词器
for iThread := 0; iThread < options.NumGseThreads; iThread++ {
go engine.segmenterWorker()
}
// 启动索引器和排序器
for shard := 0; shard < options.NumShards; shard++ {
go engine.indexerAddDoc(shard)
go engine.indexerRemoveDoc(shard)
go engine.rankerAddDoc(shard)
go engine.rankerRemoveDoc(shard)
for i := 0; i < options.NumIndexerThreads; i++ {
go engine.indexerLookup(shard)
}
for i := 0; i < options.NumRankerThreads; i++ {
go engine.rankerRank(shard)
}
}
// 启动持久化存储工作协程
if engine.initOptions.UseStore {
engine.Store()
}
atomic.AddUint64(&engine.numDocsStored, engine.numIndexingReqs)
}它的第331行代码为:
// 启动分词器
for iThread := 0; iThread < options.NumGseThreads; iThread++ {
go engine.segmenterWorker()
}推测是riot组件的设置问题，于是查看search_service.go文件的init方法:
func init() {
SearchBiz = searchService{}
engineType = make(map[string]SearchEngine)
//标签搜索
engineType[TAG] = TagSearchEngine{}
//栏目搜索
engineType[CATEGORY] = CategorySearchEngine{}
//博文搜索
engineType[ARTICLES_FULL_TEXT] = ArticlesSearchEngine{}
//归档搜索
engineType[PLACE_OF_FILE] = PlaceOfFileSearchEngine{}
runtime.GOMAXPROCS(runtime.NumCPU())
fmt.Println("start init search====================")
component.GoRoutine(func() {
gob.Register(ArticlesScoringFields{})
opts := types.EngineOpts{
Using: 3,
GseDict: "zh",
UseStore: true,
StoreFolder: "./indexer",
StoreShards: 4,
StoreEngine: "",
}
fullTextSearcher.Init(opts)
//data,_ := json.Marshal(opts)
//fmt.Println(data)
fullTextSearcher.Flush()
})
fmt.Println("end init search====================")
}于是查看EngineOpts方件:
type EngineOpts struct {
// 是否使用分词器
// 默认使用，否则在启动阶段跳过 GseDict 和 StopTokenFile 设置
// 如果你不需要在引擎内分词，可以将这个选项设为 true
// 注意，如果你不用分词器，那么在调用 IndexDoc 时,
// DocIndexData 中的 Content 会被忽略
// Not use the gse segment
NotUseGse bool `toml:"not_use_gse"`
// new, 分词规则
Using int `toml:"using"`
// 半角逗号 "," 分隔的字典文件，具体用法见
// gse.Segmenter.LoadDict 函数的注释
GseDict string `toml:"gse_dict"`
PinYin bool `toml:"pin_yin"`
// 停用词文件
StopTokenFile string `toml:"stop_file"`
// Gse search mode
GseMode bool `toml:"gse_mode"`
Hmm bool `toml:"hmm"`
Model string `toml:"model"`
// 分词器线程数
// NumSegmenterThreads int
NumGseThreads int
// 索引器和排序器的 shard 数目
// 被检索/排序的文档会被均匀分配到各个 shard 中
NumShards int
// 索引器的信道缓冲长度
IndexerBufLen int
// 索引器每个shard分配的线程数
NumIndexerThreads int
// 排序器的信道缓冲长度
RankerBufLen int
// 排序器每个 shard 分配的线程数
NumRankerThreads int
// 索引器初始化选项
IndexerOpts *IndexerOpts
// 默认的搜索选项
DefRankOpts *RankOpts
// 是否使用持久数据库，以及数据库文件保存的目录和裂分数目
StoreOnly bool `toml:"store_only"`
UseStore bool `toml:"use_store"`
StoreFolder string `toml:"store_folder"`
StoreShards int `toml:"store_shards"`
StoreEngine string `toml:"store_engine"`
IDOnly bool `toml:"id_only"`
}修改成:
component.GoRoutine(func() {
gob.Register(ArticlesScoringFields{})
opts := types.EngineOpts{
NumGseThreads:2,
NumIndexerThreads:2,
NumRankerThreads:2,
Using: 3,
GseDict: "zh",
UseStore: true,
StoreFolder: "./indexer",
StoreShards: 4,
StoreEngine: "",
}
fullTextSearcher.Init(opts)
//data,_ := json.Marshal(opts)
//fmt.Println(data)
fullTextSearcher.Flush()
})重启服务，问题解决。
总结
将相同的代码和配置放到一台性能较好的机器上不会复现上述问题，应该是riot组件默认设置对机器要求比较高，改低一些，问题解决。
另外还可以通过net/http/pprof查看golang内存情况，详见:https://segmentfault.com/a/1190000019929993
本文参与 腾讯云自媒体分享计划，分享自微信公众号。
原始发表：2019-11-18，如有侵权请联系 cloudcommunity@tencent.com 删除
评论
登录后参与评论
推荐阅读
目录