一般来讲一个高并发高性能的系统,日志是非常庞大的,随时可能高达几个T,一台服务器的硬盘极有可能装不下,而Elasticsearch的集群可以分布在不同的机器上,而又对整个集群作为一个整体,对其大容量的内容进行存储以及它的最牛掰的能力--检索.
首先在配置文件中做如下配置
elasticsearch:
clusterName: aubin-cluster
clusterNodes: 192.168.5.182:9300
@Getter
@Setter
@Configuration
@ConfigurationProperties(prefix = "elasticsearch")
public class ElasticSearchConfig {
private String clusterName;
private String clusterNodes;
/**
* 使用elasticsearch实现类时才触发
*
* @return
*/
@Bean
@ConditionalOnBean(value = EsLogServiceImpl.class)
public TransportClient getESClient() {
// 设置集群名字
Settings settings = Settings.builder().put("cluster.name", this.clusterName).build();
TransportClient client = new PreBuiltTransportClient(settings);
try {
// 读取的ip列表是以逗号分隔的
for (String clusterNode : this.clusterNodes.split(",")) {
String ip = clusterNode.split(":")[0];
String port = clusterNode.split(":")[1];
client.addTransportAddress(new TransportAddress(InetAddress.getByName(ip), Integer.parseInt(port)));
}
} catch (UnknownHostException e) {
e.printStackTrace();
}
return client;
}
}
再定义一个操作接口
public interface LogService {
void save(Log log);
Page<Log> findLogs(Map<String, Object> params);
}
具体实现类为
@Service
public class EsLogServiceImpl implements LogService, ApplicationContextAware {
private static final Logger logger = LoggerFactory.getLogger(EsLogServiceImpl.class);
private static final String INDEX = "index_logs";
private static final String TYPE = "type_logs";
@Autowired
private TransportClient client;
@Async
@Override
public void save(Log log) {
if (log.getCreateTime() == null) {
log.setCreateTime(new Date());
}
if (log.getFlag() == null) {
log.setFlag(Boolean.TRUE);
}
logger.info("{}", log);
String string = JSONObject.toJSONString(log);
IndexRequestBuilder builder = client.prepareIndex(INDEX, TYPE).setSource(string, XContentType.JSON);
builder.execute();
}
@Override
public Page<Log> findLogs(Map<String, Object> params) {
SearchRequestBuilder builder = client.prepareSearch().setIndices(INDEX).setTypes(TYPE);
if (!CollectionUtils.isEmpty(params)) {
BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
// 用户名模糊匹配
String username = MapUtils.getString(params, "username");
if (StringUtils.isNoneBlank(username)) {
queryBuilder.must(QueryBuilders.wildcardQuery("username", "*" + username + "*"));
}
// 模块精确匹配
String module = MapUtils.getString(params, "module");
if (StringUtils.isNoneBlank(module)) {
queryBuilder.must(QueryBuilders.matchQuery("module", module));
}
String flag = MapUtils.getString(params, "flag");
if (StringUtils.isNoneBlank(flag)) {
Boolean bool = Boolean.FALSE;
if ("1".equals(flag) || "true".equalsIgnoreCase(flag)) {
bool = Boolean.TRUE;
}
queryBuilder.must(QueryBuilders.matchQuery("flag", bool));
}
// 大于等于开始日期,格式yyyy-MM-dd
String beginTime = MapUtils.getString(params, "beginTime");
if (StringUtils.isNoneBlank(beginTime)) {
// 转化为0点0分0秒
Long timestamp = toTimestamp(beginTime + "T00:00:00");
queryBuilder.must(QueryBuilders.rangeQuery("createTime").from(timestamp));
}
// 小于等于结束日期,格式yyyy-MM-dd
String endTime = MapUtils.getString(params, "endTime");
if (StringUtils.isNoneBlank(endTime)) {
// 转化为23点59分59秒
Long timestamp = toTimestamp(endTime + "T23:59:59");
queryBuilder.must(QueryBuilders.rangeQuery("createTime").to(timestamp));
}
if (queryBuilder != null) {
builder.setPostFilter(queryBuilder);
}
}
builder.addSort("createTime", SortOrder.DESC);
PageUtil.pageParamConver(params, true);
Integer start = MapUtils.getInteger(params, PageUtil.START);
if (start != null) {
builder.setFrom(start);
}
Integer length = MapUtils.getInteger(params, PageUtil.LENGTH);
if (length != null) {
builder.setSize(length);
}
SearchResponse searchResponse = builder.get();
SearchHits searchHits = searchResponse.getHits();
// 总数量
Long total = searchHits.getTotalHits();
int size = searchHits.getHits().length;
List<Log> list = new ArrayList<>(size);
if (size > 0) {
searchHits.forEach(hit -> {
String val = hit.getSourceAsString();
list.add(JSONObject.parseObject(val, Log.class));
});
}
return new Page<>(total.intValue(), list);
}
private Long toTimestamp(String str) {
LocalDateTime localDateTime = LocalDateTime.parse(str);
Date date = Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant());
return date.getTime();
}
private static ApplicationContext applicationContext = null;
@Override
public void setApplicationContext(ApplicationContext context) throws BeansException {
applicationContext = context;
}
/**
* 初始化日志es索引
*/
@PostConstruct
public void initIndex() {
LogService logService = applicationContext.getBean(LogService.class);
// 日志实现是否采用elasticsearch
boolean flag = (logService instanceof EsLogServiceImpl);
if (!flag) {
return;
}
try {
// 判断索引是否存在
IndicesExistsResponse indicesExistsResponse = client.admin().indices()
.exists(new IndicesExistsRequest(INDEX)).get();
if (indicesExistsResponse.isExists()) {
return;
}
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
CreateIndexRequestBuilder requestBuilder = client.admin().indices().prepareCreate(INDEX);
CreateIndexResponse createIndexResponse = requestBuilder.execute().actionGet();
if (createIndexResponse.isAcknowledged()) {
logger.info("索引:{},创建成功", INDEX);
} else {
logger.error("索引:{},创建失败", INDEX);
}
}
}