索引生命周期管理
原创
索引生命周期管理通用步骤
创建索引管理策略
这个策略可以应用于索引模板，在创建索引时，关联到索引模板，按新的策略进行管理。
可以分为hot,warm,cold,frezen,delete多个阶段，每个阶段可以对索引实施不同的策略
PUT _ilm/policy/my-lifecycle-policy
{
"policy": {
"phases": {
"hot": {
"min_age": "0",
"actions": {
"set_priority": {
"priority": 100
},
"rollover": {
"max_age": "3d",
"max_docs": 3,
"max_size": "2gb"
}
}
},
"warm": {
"min_age": "1d",
"actions": {
"set_priority": {
"priority": 50
},
"allocate": {
"number_of_replicas": 0
},
"shrink": {
"number_of_shards": 1
},
"forcemerge": {
"max_num_segments": 1
}
}
}
}
}
}创建组件模板
索引模板可以由多个组件模板组成，可以根据索引的setting,mapping等配置多个组件，这样组合起来更灵活。
创建mapping组件模板
PUT _component_template/my-mappings
{
"template": {
"mappings": {
"properties": {
"@timestamp": {
"type": "date",
"format": "date_optional_time||epoch_millis"
},
"message": {
"type": "wildcard"
}
}
}
},
"_meta": {
"description": "Mappings for @timestamp and message fields",
"my-custom-meta-field": "More arbitrary metadata"
}
}创建setting组件模板
这里setting组件模板会关联索引管理策略:my-lifecycle-policy
PUT _component_template/my-settings
{
"template": {
"settings": {
"index.lifecycle.name": "my-lifecycle-policy"
}
},
"_meta": {
"description": "Settings for ILM",
"my-custom-meta-field": "More arbitrary metadata"
}
}创建索引模板
通过composed_of整合多个组件模板，让创建索引模板更加灵活，这里同时配置了data_stream功能，目前这个功能用于关联datastream的创建，这里可以测试两种方式。
1、通过datastream创建数据流对象（下面会讲到）。
2、通过索引创建（这里可以自行测试一下）
PUT _index_template/my-index-template
{
"index_patterns": [
"my-data-stream*"
],
"data_stream": {},
"composed_of": [
"my-mappings",
"my-settings"
],
"priority": 500,
"_meta": {
"description": "Template for my time series data",
"my-custom-meta-field": "More arbitrary metadata"
}
}配置集群参数
更加快和容易触发索引的滚动
PUT _cluster/settings
{
"transient": {
"indices.lifecycle.poll_interval": "10s"
}
}配置时间字段
当索引字段定义中没有自动产生时间，需要额外配置一个ingest
PUT _ingest/pipeline/add-timestamp
{
"processors": [
{
"set": {
"field": "@timestamp",
"value": "{{_ingest.timestamp}}"
}
}
]
}数据流管理阶段
创建数据流
PUT _data_stream/my-data-stream插入单条数据
POST /my-data-stream/_doc/?pipeline=add-timestamp
{
"user": {
"id": "8a4f500d"
},
"message": "Login successful"
}批量插入数据
PUT /my-data-stream/_bulk?pipeline=add-timestamp&refresh
{"create":{ }}
{ "user": { "id": "vlb44hny" }, "message": "Login attempt failed" }
{"create":{ }}
{ "user": { "id": "8a4f500d" }, "message": "Login successful" }
{"create":{ }}
{ "user": { "id": "l7gk7f82" }, "message": "Logout successful" }
GET my-data-stream/_ilm/explain
GET my-data-stream/_search?filter_path=**.hits
DELETE _data_stream/my-data-stream
#获取后备索引
GET _data_stream/my-data-stream
POST _index_template/_simulate_index/my-data-stream
#搜索数据流
GET my-data-stream/_search?filter_path=**.hits
#滚动数据流
POST /my-data-stream/_rollover/
#数据流统计信息
GET /_data_stream/my-data-stream/_stats?human=true
POST /.ds-my-data-stream-2022.11.17-000002/_open
#删除数据流
DELETE _data_stream/demo-ds
POST /_reindex
{
"source": {
"index": "archive"
},
"dest": {
"index": "my-data-stream",
"op_type": "create" #这是重点
}
}
POST /my-data-stream/_update_by_query
{
"query": {
"match": {
"user.id": "l7gk7f82"
}
},
"script": {
"source": "ctx._source.user.id = params.new_id",
"params": {
"new_id": "XgdX0NoX"
}
}
}
POST /my-data-stream/_delete_by_query
{
"query": {
"match": {
"user.id": "vlb44hny"
}
}
}
GET /my-data-stream/_search
{
"seq_no_primary_term": true,
"query": {
"match": {
"user.id": "yWIumJd7"
}
}
}
PUT /.ds-my-data-stream-2099-03-08-000003/_doc/bfspvnIBr7VVZlfp2lqX?if_seq_no=0&if_primary_term=1
{
"@timestamp": "2099-03-08T11:06:07.000Z",
"user": {
"id": "8a4f500d"
},
"message": "Login successful"
}
DELETE /.ds-my-data-stream-2099.03.08-000003/_doc/bfspvnIBr7VVZlfp2lqX
#如何添加字段映射到数据流
#1、更新索引模板
PUT /_index_template/my-index-template
{
"index_patterns": [ "my-data-stream*" ],
"data_stream": { },
"priority": 500,
"template": {
"mappings": {
"properties": {
"message": {
"type": "text"
}
}
}
}
}
#2、更新字段到数据流的所有后备索引
PUT /my-data-stream/_mapping
{
"properties": {
"message": {
"type": "text"
}
}
}
#3、仅更新到当前写入的索引
PUT /my-data-stream/_mapping?write_index_only=true
{
"properties": {
"message": {
"type": "text"
}
}
}
PUT /_index_template/my-index-template
{
"index_patterns": [ "my-data-stream*" ],
"data_stream": { },
"priority": 500,
"template": {
"mappings": {
"properties": {
"host": {
"properties": {
"ip": {
"type": "ip",
"ignore_malformed": true
}
}
}
}
}
}
}
PUT /_index_template/my-index-template
{
"index_patterns": [ "my-data-stream*" ],
"data_stream": { },
"priority": 500,
"template": {
"settings": {
"index.refresh_interval": "30s"
}
}
}
GET /_resolve/index/new-data-stream*数据流日常管理操作
PUT _data_stream/my-data-stream
##查询数据
GET my-data-stream/_search?filter_path=**.hits
##数据流索引策略
GET my-data-stream/_ilm/explain
#获取后备索引
GET _data_stream/my-data-stream
DELETE _data_stream/my-data-stream
POST _index_template/_simulate_index/my-data-stream
#滚动数据流
POST /my-data-stream/_rollover/
#数据流统计信息
GET /_data_stream/my-data-stream/_stats?human=true
POST /.ds-my-data-stream-2022.11.17-000002/_open
#删除数据流
DELETE _data_stream/my-data-stream
POST /_reindex
{
"source": {
"index": "archive"
},
"dest": {
"index": "my-data-stream",
"op_type": "create" #这是重点
}
}
POST /my-data-stream/_update_by_query
{
"query": {
"match": {
"user.id": "l7gk7f82"
}
},
"script": {
"source": "ctx._source.user.id = params.new_id",
"params": {
"new_id": "XgdX0NoX"
}
}
}
POST /my-data-stream/_delete_by_query
{
"query": {
"match": {
"user.id": "vlb44hny"
}
}
}
GET /my-data-stream/_search
{
"seq_no_primary_term": true,
"query": {
"match": {
"user.id": "yWIumJd7"
}
}
}
PUT /.ds-my-data-stream-2099-03-08-000003/_doc/bfspvnIBr7VVZlfp2lqX?if_seq_no=0&if_primary_term=1
{
"@timestamp": "2099-03-08T11:06:07.000Z",
"user": {
"id": "8a4f500d"
},
"message": "Login successful"
}
DELETE /.ds-my-data-stream-2099.03.08-000003/_doc/bfspvnIBr7VVZlfp2lqX
#如何添加字段映射到数据流
#1、更新索引模板
PUT /_index_template/my-index-template
{
"index_patterns": [ "my-data-stream*" ],
"data_stream": { },
"priority": 500,
"template": {
"mappings": {
"properties": {
"message": {
"type": "text"
}
}
}
}
}
#2、更新字段到数据流的所有后备索引
PUT /my-data-stream/_mapping
{
"properties": {
"message": {
"type": "text"
}
}
}
#3、仅更新到当前写入的索引
PUT /my-data-stream/_mapping?write_index_only=true
{
"properties": {
"message": {
"type": "text"
}
}
}
PUT /_index_template/my-index-template
{
"index_patterns": [ "my-data-stream*" ],
"data_stream": { },
"priority": 500,
"template": {
"mappings": {
"properties": {
"host": {
"properties": {
"ip": {
"type": "ip",
"ignore_malformed": true
}
}
}
}
}
}
}
PUT /_index_template/my-index-template
{
"index_patterns": [ "my-data-stream*" ],
"data_stream": { },
"priority": 500,
"template": {
"settings": {
"index.refresh_interval": "30s"
}
}
}
GET /_resolve/index/new-data-stream*索引生命周期管理转为数据流管理
PUT _component_template/my-settings
{
"template": {
"settings": {
"index.lifecycle.prefer_ilm":"true"
}
}
}
PUT _data_stream/my-data-stream/_lifecycle
{
"data_retention": "7d"
}
GET _data_stream/my-data-stream
POST my-data-stream/_rollover数据流管理转回索引生命周期管理
PUT _data_stream/my-data-stream/_lifecycle
{
"data_retention": "7d",
"enabled": false
}
POST my-data-stream/_rollover
GET _data_stream/my-data-stream原创声明：本文系作者授权腾讯云开发者社区发表，未经许可，不得转载。
如有侵权，请联系 cloudcommunity@tencent.com 删除。
原创声明：本文系作者授权腾讯云开发者社区发表，未经许可，不得转载。
如有侵权，请联系 cloudcommunity@tencent.com 删除。
评论
登录后参与评论
推荐阅读
目录
腾讯云开发者
