API方式上传PB（Protobuf）日志至腾讯云日志服务

Cheng_Blog

发布于 2022-02-25 09:39:46

9450

发布于 2022-02-25 09:39:46

文章被收录于专栏：Cheng's BlogCheng's Blog

点击进入腾讯日志服务官方文档查看

一、生成PB（Protobuf）上传日志类

使用插件的话,首先要下载安装Protobuf Support插件。

安装后重启IDEA即可。

接着需要对Protobuf Support插件进行配置.

maven的pom文件添加下面的代码

 <!--pb文件处理-->
        <dependency>
            <groupId>com.google.protobuf</groupId>
            <artifactId>protobuf-java</artifactId>
            <version>3.18.0</version>
        </dependency>
        <dependency>
            <groupId>com.google.protobuf</groupId>
            <artifactId>protobuf-java-util</artifactId>
            <version>3.18.0</version>
        </dependency>
        <dependency>
            <groupId>net.jpountz.lz4</groupId>
            <artifactId>lz4</artifactId>
            <version>1.3.0</version>
        </dependency>
        <!--pb文件处理-->
        <!--腾讯云服务-->
        <dependency>
            <groupId>com.tencentcloudapi</groupId>
            <artifactId>tencentcloud-sdk-java</artifactId>
            <version>3.1.361</version>
        </dependency>
        <dependency>
            <groupId>com.tencentcloudapi</groupId>
            <artifactId>tencentcloud-sdk-java-common</artifactId>
            <version>3.1.361</version>
        </dependency>
        <!--腾讯云服务-->

<extensions>
	<extension>
		<groupId>kr.motd.maven</groupId>
		<artifactId>os-maven-plugin</artifactId>
		<version>1.5.0.Final</version>
	</extension>
</extensions>

                        <plugin>
				<groupId>org.xolstice.maven.plugins</groupId>
				<artifactId>protobuf-maven-plugin</artifactId>
				<version>0.5.0</version>
				<configuration>
					<protocArtifact>
						com.google.protobuf:protoc:3.1.0:exe:${os.detected.classifier}
					</protocArtifact>
					<pluginId>grpc-java</pluginId>
					<pluginArtifact>
						io.grpc:protoc-gen-grpc-java:1.11.0:exe:${os.detected.classifier}
					</pluginArtifact>
				</configuration>
				<executions>
					<execution>
						<goals>
							<goal>compile</goal>
							<goal>compile-custom</goal>
						</goals>
					</execution>
				</executions>
			</plugin>

现在开始写.proto文件,在src/main下面新建proto文件夹,在proto里面新建一个文件,注意后缀是.proto,我这里是cls.proto,

package cls;
//syntax = "proto3";
//proto类生成路径
option java_package = "com.site.blog.my.core.entity.log";
option java_outer_classname = "Cls";
message Log
{
  message Content
  {
    required string key   = 1; // 每组字段的 key
    required string value = 2; // 每组字段的 value
  }
  required int64   time     = 1; // 时间戳，UNIX时间格式
  repeated Content contents = 2; // 一条日志里的多个kv组合
}

message LogTag
{
  required string key       = 1;
  required string value     = 2;
}

message LogGroup
{
  repeated Log    logs        = 1; // 多条日志合成的日志数组
  optional string contextFlow = 2; // 目前暂无效用
  optional string filename    = 3; // 日志文件名
  optional string source      = 4; // 日志来源，一般使用机器IP
  repeated LogTag logTags     = 5;
}

message LogGroupList
{
  repeated LogGroup logGroupList = 1; // 日志组列表
}

编写好.proto文件后,使用插件将proto文件转换为java文件

双击 protobuf:compile 即可

出现SUCCESS信息表示转换成功

然后在target/generated-sources/protobuf目录中即可找到生成的java文件，复制到src/main/java中你需要使用的位置即可。

二、上传pb日志工具类

1. 日志类型枚举

/**
 * 日志类型
 *
 * @author Houchengen
 * @date 2021/09/01
 */
public enum LogType {
    /**
     * info日志
     */
    INFO(1, "INFO"),
    /**
     * debug日志
     */
    DEBUG(2, "DEBUG"),
    /**
     * 错误日志
     */
    ERROR(3, "ERROR");


    LogType(Integer origin, String label) {
        this.origin = origin;
        this.label = label;
    }

    /**
     * 性质代码
     */
    private Integer origin;
    /**
     * 性质中文
     */
    private String label;

    public Integer getOrigin() {
        return origin;
    }

    public void setOrigin(Integer origin) {
        this.origin = origin;
    }

    public String getLabel() {
        return label;
    }

    public void setLabel(String label) {
        this.label = label;
    }


    public static LogType getEnumByKey(Integer origin) {
        for (LogType type : LogType.values()) {
            if (origin.equals(type.origin)) {
                return type;
            }
        }
        return null;
    }
}

1. 日志上传类

import com.site.blog.my.core.entity.log.Cls;
import com.site.blog.my.core.entity.log.LogType;
import com.tencentcloudapi.common.CommonClient;
import com.tencentcloudapi.common.Credential;
import com.tencentcloudapi.common.exception.TencentCloudSDKException;
import com.tencentcloudapi.common.profile.ClientProfile;
import com.tencentcloudapi.common.profile.HttpProfile;
import com.tencentcloudapi.common.profile.Region;
import net.jpountz.lz4.LZ4Compressor;
import net.jpountz.lz4.LZ4Factory;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.stereotype.Component;

import java.util.HashMap;
import java.util.Map;

/**
 * 发送腾讯日志工具类
 *
 * @author Houchengen
 * @date 2021/09/22
 */
@EnableAsync
@Component
public class SendTencentLogUtil {


    //腾讯云账户密钥 secretId.
    public static final String SECRET_ID = "*****************";
    //腾讯云账户密钥 secretKey.
    public static final String SECRET_KEY = "*********************";

    public static final String ENDPOINT = "cls.tencentcloudapi.com";
    public static final String ROOT_DOMAIN = "tencentcloudapi.com";
    public static final String X_CLS_TOPIC_ID = "X-CLS-TopicId";
    public static final String X_CLS_HASH_KEY = "X-CLS-HashKey";
    public static final String X_CLS_COMPRESS_TYPE = "X-CLS-CompressType";
    public static final String LZ_4 = "lz4";
	//你的topicId
    public static final String TOPIC = "************";
    public static final String PRODUCT_NAME_CLS = "cls";
    public static final String UPLOAD_LOG_URL = "UploadLog";
    public static final String VERSION = "2020-10-16";


    /**
     * 发送日志
     *
     * @param logType 日志类型
     * @param params  参数
     */
    @Async
    public void sendLog(LogType logType, Map<String, String> params) {
        Credential cred = new Credential(SECRET_ID, SECRET_KEY);

        //region 设置http选项中的参数
        HttpProfile httpProfile = new HttpProfile();
        // post请求  (默认为post请求)
        httpProfile.setReqMethod(HttpProfile.REQ_POST);
        // 指定接入地域域名(默认就近接入)
        httpProfile.setEndpoint(ENDPOINT);
        httpProfile.setRootDomain(ROOT_DOMAIN);
        // 在外网互通的网络环境下支持http协议(默认是https协议),请选择(https:// or http://)
        httpProfile.setProtocol(HttpProfile.REQ_HTTPS);
        // 设置读取超时时间，单位为秒(默认0秒)
        httpProfile.setReadTimeout(0);
        // 设置写入超时时间，单位为秒(默认0秒)
        httpProfile.setWriteTimeout(0);
        // 请求连接超时时间，单位为秒(默认60秒)
        httpProfile.setConnTimeout(HttpProfile.TM_MINUTE);
        //endregion

        //region 设置client选项中的参数
        ClientProfile clientProfile = new ClientProfile();
        clientProfile.setHttpProfile(httpProfile);
        // 指定签名算法默认（"TC3-HMAC-SHA256"），它更安全但是会轻微降低性能。
        clientProfile.setSignMethod(ClientProfile.SIGN_TC3_256);
        //打印日志，默认是false
        clientProfile.setDebug(false);
        //endregion

        //region headers 和 body 信息
        HashMap<String, String> headers = new HashMap<>();
        headers.put(X_CLS_TOPIC_ID, TOPIC);
        headers.put(X_CLS_HASH_KEY, "");
        // body lz4 压缩
        headers.put(X_CLS_COMPRESS_TYPE, LZ_4);
        byte[] body = compressedByte(getBodyInfo(logType, params));
        //endregion
        try {
            CommonClient client = new CommonClient(PRODUCT_NAME_CLS, VERSION, cred, Region.Chengdu.getValue(), clientProfile);
            String resp = client.callOctetStream(UPLOAD_LOG_URL, headers, body);
            System.out.println(resp);
        } catch (TencentCloudSDKException te) {
            System.out.println(te.getMessage());
        }
    }

    /**
     * pom依赖:
     * <dependency>
     * <groupId>net.jpountz.lz4</groupId>
     * <artifactId>lz4</artifactId>
     * <version>1.3.0</version>
     * </dependency>
     * <p>
     * 进行lz4压缩
     *
     * @param srcByte
     * @return
     */
    public static byte[] compressedByte(byte[] srcByte) {
        LZ4Factory factory = LZ4Factory.fastestInstance();
        LZ4Compressor compressor = factory.fastCompressor();
        return compressor.compress(srcByte);
    }

    /**
     * pom依赖:
     * <dependency>
     * <groupId>com.google.protobuf</groupId>
     * <artifactId>protobuf-java</artifactId>
     * <version>3.15.3</version>
     * </dependency
     * <p>
     * 构造http body 信息
     * protobuf 序列化为 byte[]
     *
     * @return byte[]
     */
    public static byte[] getBodyInfo(LogType logType, Map<String, String> params) {
        Cls.Log.Builder logBuilder = Cls.Log.newBuilder();
        logBuilder.setTime(System.currentTimeMillis() / 1000L);
        logBuilder.addContents(Cls.Log.Content.newBuilder()
                .setKey("type")
                .setValue(logType.getLabel())
                .build());
        params.forEach((k, v) -> {
            if (v == null) {
                v = "null";
            }
            logBuilder.addContents(Cls.Log.Content.newBuilder()
                    .setKey(k)
                    .setValue(v)
                    .build());
        });
        Cls.Log log = logBuilder.build();
        Cls.LogGroup logGroup = Cls.LogGroup.newBuilder()
                .addLogs(log)
                .build();

        Cls.LogGroupList logGroupList = Cls.LogGroupList.newBuilder()
                .addLogGroupList(logGroup)
                .build();

        //这里我们将封装有数据的对象实例，转换为字节数组，用于数据传输、存储等
        return logGroupList.toByteArray();
    }
}

三、使用示例

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.site.blog.my.core.entity.log.LogType;
import com.site.blog.my.core.util.SendTencentLogUtil;
import org.aspectj.lang.JoinPoint;
import org.aspectj.lang.annotation.AfterReturning;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.annotation.Before;
import org.aspectj.lang.annotation.Pointcut;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.context.request.RequestContextHolder;
import org.springframework.web.context.request.ServletRequestAttributes;

import javax.servlet.http.HttpServletRequest;
import java.util.Arrays;
import java.util.HashMap;


/**
 * 请求日志上传切面
 *
 * @author Houchengen
 * @date 2021/09/22
 */
@Aspect
@Component
public class RequestLogAspect {

    @Autowired
    private SendTencentLogUtil sendTencentLogUtil;

    @Pointcut("execution(* com.site.blog.my.core.controller..*(..))")
    public void executeController() {
    }

    @Before("executeController()")
    public void doBefore(JoinPoint joinPoint) {
        //获取请求报文头部元数据
        ServletRequestAttributes requestAttributes = (ServletRequestAttributes) RequestContextHolder.getRequestAttributes();
        //获取请求对象
        HttpServletRequest request = requestAttributes.getRequest();
        HashMap<String, String> map = new HashMap<>();
        map.put("params", Arrays.toString(joinPoint.getArgs()));
        map.put("method", request.getMethod());
        map.put("class_method", joinPoint.getSignature().getDeclaringTypeName() +
                "." + joinPoint.getSignature().getName());
        map.put("requestUrl", request.getRequestURL().toString());
        sendTencentLogUtil.sendLog(LogType.INFO, map);
    }

    @AfterReturning(returning = "ret", pointcut = "executeController()")
    public void doAfter(Object ret) {
        HashMap<String, String> map = new HashMap<>();
        map.put("responseBody", JSON.toJSONString(ret, SerializerFeature.WriteMapNullValue));
        sendTencentLogUtil.sendLog(LogType.INFO, map);
    }

}