cd /opt/TLS/k8s/ssl
cat > ca-config.json << EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF
cat > ca-csr.json << EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
#查看已生成的证书文件
[root@k8s-master ssl]# ll
total 20
-rw-r--r-- 1 root root 294 Apr 3 13:37 ca-config.json
-rw-r--r-- 1 root root 1001 Apr 3 13:38 ca.csr
-rw-r--r-- 1 root root 264 Apr 3 13:37 ca-csr.json
-rw------- 1 root root 1675 Apr 3 13:38 ca-key.pem
-rw-r--r-- 1 root root 1310 Apr 3 13:38 ca.pem
#这里生成了ca.pem和ca-key.pem两个文件
cat > server-csr.json << EOF
{
"CN": "kubernetes",
"hosts": [
"10.0.0.1",
"127.0.0.1",
"192.168.190.147",
"192.168.190.148",
"kubernetes",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "k8s",
"OU": "System"
}
]
}
EOF
#上述文件hosts字段中IP为所有Master IP,一个都不能少!为了方便后期扩容可以多写几个预留的IP
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server
[root@k8s-master ssl]# ll
total 36
-rw-r--r-- 1 root root 294 Apr 3 13:37 ca-config.json
-rw-r--r-- 1 root root 1001 Apr 3 13:38 ca.csr
-rw-r--r-- 1 root root 264 Apr 3 13:37 ca-csr.json
-rw------- 1 root root 1675 Apr 3 13:38 ca-key.pem
-rw-r--r-- 1 root root 1310 Apr 3 13:38 ca.pem
-rw-r--r-- 1 root root 1261 Apr 3 13:55 server.csr
-rw-r--r-- 1 root root 557 Apr 3 13:55 server-csr.json
-rw------- 1 root root 1675 Apr 3 13:55 server-key.pem
-rw-r--r-- 1 root root 1627 Apr 3 13:55 server.pem
#这里生成了server.pem和server-key.pem两个文件
cat > kube-apiserver.conf << EOF
KUBE_APISERVER_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/opt/kubernetes/logs \\
--insecure-port=0 \\
--etcd-servers=https://192.168.190.147:2379,https://192.168.190.148:2379 \\
--bind-address=192.168.190.147 \\
--secure-port=6443 \\
--advertise-address=192.168.190.147 \\
--allow-privileged=true \\
--service-cluster-ip-range=10.0.0.0/24 \\
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \\
--authorization-mode=RBAC,Node \\
--enable-bootstrap-token-auth=true \\
--token-auth-file=/opt/kubernetes/cfg/token.csv \\
--service-node-port-range=30000-32767 \\
--kubelet-client-certificate=/opt/kubernetes/ssl/server.pem \\
--kubelet-client-key=/opt/kubernetes/ssl/server-key.pem \\
--kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname,InternalDNS,ExternalDNS \\
--tls-cert-file=/opt/kubernetes/ssl/server.pem \\
--tls-private-key-file=/opt/kubernetes/ssl/server-key.pem \\
--client-ca-file=/opt/kubernetes/ssl/ca.pem \\
--service-account-key-file=/opt/kubernetes/ssl/ca-key.pem \\
--service-account-issuer=api \\
--service-account-signing-key-file=/opt/kubernetes/ssl/server-key.pem \\
--etcd-cafile=/opt/etcd/ssl/ca.pem \\
--etcd-certfile=/opt/etcd/ssl/server.pem \\
--etcd-keyfile=/opt/etcd/ssl/server-key.pem \\
--requestheader-client-ca-file=/opt/kubernetes/ssl/ca.pem \\
--proxy-client-cert-file=/opt/kubernetes/ssl/server.pem \\
--proxy-client-key-file=/opt/kubernetes/ssl/server-key.pem \\
--requestheader-allowed-names=kubernetes \\
--requestheader-extra-headers-prefix=X-Remote-Extra- \\
--requestheader-group-headers=X-Remote-Group \\
--requestheader-username-headers=X-Remote-User \\
--enable-aggregator-routing=true \\
--audit-log-maxage=30 \\
--audit-log-maxbackup=3 \\
--audit-log-maxsize=100 \\
--audit-log-path=/opt/kubernetes/logs/k8s-audit.log"
EOF
# 上面两个\\ 第一个是转义符,第二个是换行符,使用转义符是为了使用EOF保留换行符。
# • --logtostderr:启用日志
# • ---v:日志等级
# • --log-dir:日志目录
# • --etcd-servers:etcd集群地址
# • --bind-address:监听地址
# • --secure-port:https安全端口
# • --advertise-address:集群通告地址
# • --allow-privileged:启用授权
# • --service-cluster-ip-range:Service虚拟IP地址段
# • --enable-admission-plugins:准入控制模块
# • --authorization-mode:认证授权,启用RBAC授权和节点自管理
# • --enable-bootstrap-token-auth:启用TLS bootstrap机制
# • --token-auth-file:bootstrap token文件
# • --service-node-port-range:Service nodeport类型默认分配端口范围
# • --kubelet-client-xxx:apiserver访问kubelet客户端证书
# • --tls-xxx-file:apiserver https证书
# • 1.20以上版本必须加的参数:--service-account-issuer,--service-account-signing-key-file
# • --etcd-xxxfile:连接Etcd集群证书
# • --audit-log-xxx:审计日志
# • 启动聚合层相关配置:
# • --requestheader-client-ca-file,--proxy-client-cert-file,--proxy-client-key-file,
# • --requestheader-allowed-names,--requestheader-extra-headers-prefix,
# • --requestheader-group-headers,--requestheader-username-headers,
# • --enable-aggregator-routing
TLS Bootstraping:Master apiserver启用TLS认证后,Node节点kubelet和kube-proxy要与kube-apiserver进行通信,必须使用CA签发的有效证书才可以,当Node节点很多时,这种客户端证书颁发需要大量工作,同样也会增加集群扩展复杂度。为了简化流程,Kubernetes引入了TLS bootstraping机制来自动颁发客户端证书,kubelet会以一个低权限用户自动向apiserver申请证书,kubelet的证书由apiserver动态签署。所以强烈建议在Node上使用这种方式,目前主要用于kubelet,kube-proxy还是由我们统一颁发一个证书。
#创建token文件
cat > token.csv << EOF
c47ffb939f5ca36231d9e3121a252940,kubelet-bootstrap,10001,"system:node-bootstrapper"
EOF
# 格式:token,用户名,UID,用户组
# token也可自行生成替换:
# head -c 16 /dev/urandom | od -An -t x | tr -d ' '
这里这一串的 token,不用迟疑,写下去就是。
cat > kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-apiserver.conf
ExecStart=/opt/kubernetes/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
#查看上述命令生成的相关文件
[root@k8s-master cfg]# ll
total 12
-rw-r--r-- 1 root root 1815 Apr 3 13:57 kube-apiserver.conf
-rw-r--r-- 1 root root 286 Apr 3 14:06 kube-apiserver.service
-rw-r--r-- 1 root root 84 Apr 3 13:57 token.csv
#创建kubernetes目录
mkdir -p /opt/kubernetes/{bin,cfg,ssl,logs}
#拷贝证书文件
scp -r /opt/TLS/k8s/ssl/*pem /opt/kubernetes/ssl/
#拷贝配置文件
scp -r /opt/TLS/k8s/cfg/token.csv /opt/kubernetes/cfg/
scp /opt/TLS/k8s/cfg/kube-apiserver.conf /opt/kubernetes/cfg/kube-apiserver.conf
#拷贝管理文件
scp /opt/TLS/k8s/cfg/kube-apiserver.service /usr/lib/systemd/system/kube-apiserver.service
#拷贝可执行文件
scp /opt/TLS/download/kubernetes/server/bin/{kube-apiserver,kube-scheduler,kube-controller-manager} /opt/kubernetes/bin
scp /opt/TLS/download/kubernetes/server/bin/kubectl /usr/local/bin/
#核对证书文件
[root@k8s-master cfg]# ll /opt/kubernetes/ssl/
total 16
-rw------- 1 root root 1675 Apr 3 14:11 ca-key.pem
-rw-r--r-- 1 root root 1310 Apr 3 14:11 ca.pem
-rw------- 1 root root 1675 Apr 3 14:11 server-key.pem
-rw-r--r-- 1 root root 1627 Apr 3 14:11 server.pem
#核对配置文件
[root@k8s-master cfg]# ll /opt/kubernetes/cfg/token.csv
-rw-r--r-- 1 root root 84 Apr 3 14:11 /opt/kubernetes/cfg/token.csv
[root@k8s-master cfg]# ll /opt/kubernetes/cfg/kube-apiserver.conf
-rw-r--r-- 1 root root 1815 Apr 3 14:12 /opt/kubernetes/cfg/kube-apiserver.conf
#核对管理文件
[root@k8s-master cfg]# ll /usr/lib/systemd/system/kube-apiserver.service
-rw-r--r-- 1 root root 286 Apr 3 14:11 /usr/lib/systemd/system/kube-apiserver.service
#核对可执行文件
[root@k8s-master cfg]# ll /opt/kubernetes/bin/{kube-apiserver,kube-scheduler,kube-controller-manager}
-rwxr-xr-x 1 root root 131301376 Apr 3 14:12 /opt/kubernetes/bin/kube-apiserver
-rwxr-xr-x 1 root root 121110528 Apr 3 14:12 /opt/kubernetes/bin/kube-controller-manager
-rwxr-xr-x 1 root root 49618944 Apr 3 14:12 /opt/kubernetes/bin/kube-scheduler
[root@k8s-master cfg]# ll /usr/local/bin/kubectl
-rwxr-xr-x 1 root root 46592000 Apr 3 14:12 /usr/local/bin/kubectl
[root@k8s-master cfg]# systemctl daemon-reload && systemctl start kube-apiserver && systemctl enable kube-apiserver && systemctl status kube-apiserver
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-apiserver.service to /usr/lib/systemd/system/kube-apiserver.service.
● kube-apiserver.service - Kubernetes API Server
Loaded: loaded (/usr/lib/systemd/system/kube-apiserver.service; enabled; vendor preset: disabled)
Active: active (running) since Sun 2022-04-03 14:14:54 CST; 111ms ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 11765 (kube-apiserver)
CGroup: /system.slice/kube-apiserver.service
└─11765 /opt/kubernetes/bin/kube-apiserver --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --insecure-port=0 --etcd-servers=https://192.168.190.147:2379,https://192.168.190.148:2379
Apr 03 14:14:54 k8s-master systemd[1]: Started Kubernetes API Server.
cd /opt/TLS/k8s/cfg
cat > kube-controller-manager.conf << EOF
KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/opt/kubernetes/logs \\
--leader-elect=true \\
--kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubeconfig \\
--bind-address=127.0.0.1 \\
--allocate-node-cidrs=true \\
--cluster-cidr=10.244.0.0/16 \\
--service-cluster-ip-range=10.0.0.0/24 \\
--cluster-signing-cert-file=/opt/kubernetes/ssl/ca.pem \\
--cluster-signing-key-file=/opt/kubernetes/ssl/ca-key.pem \\
--root-ca-file=/opt/kubernetes/ssl/ca.pem \\
--service-account-private-key-file=/opt/kubernetes/ssl/ca-key.pem \\
--cluster-signing-duration=87600h0m0s"
EOF
# • --kubeconfig:连接apiserver配置文件
# • --leader-elect:当该组件启动多个时,自动选举(HA)
# • --cluster-signing-cert-file/--cluster-signing-key-file:自动为kubelet颁发证书的CA,与apiserver保持一致
cd /opt/TLS/k8s/ssl
cat > kube-controller-manager-csr.json << EOF
{
"CN": "system:kube-controller-manager",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:masters",
"OU": "System"
}
]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
....
[root@k8s-master ssl]# ll kube-controller-manager*
-rw-r--r-- 1 root root 1045 Apr 3 14:19 kube-controller-manager.csr
-rw-r--r-- 1 root root 255 Apr 3 14:18 kube-controller-manager-csr.json
-rw------- 1 root root 1679 Apr 3 14:19 kube-controller-manager-key.pem
-rw-r--r-- 1 root root 1436 Apr 3 14:19 kube-controller-manager.pem
#这里生成了kube-controller-manager.pem和kube-controller-manager-key.pem文件
# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/opt/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://192.168.190.147:6443 \
--kubeconfig=/opt/TLS/k8s/cfg/kube-controller-manager.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kube-controller-manager \
--client-certificate=./kube-controller-manager.pem \
--client-key=./kube-controller-manager-key.pem \
--embed-certs=true \
--kubeconfig=/opt/TLS/k8s/cfg/kube-controller-manager.kubeconfig
# 设置上下文参数
kubectl config set-context default \
--cluster=kubernetes \
--user=kube-controller-manager \
--kubeconfig=/opt/TLS/k8s/cfg/kube-controller-manager.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=/opt/TLS/k8s/cfg/kube-controller-manager.kubeconfig
cd /opt/TLS/k8s/cfg
cat > kube-controller-manager.service << EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-controller-manager.conf
ExecStart=/opt/kubernetes/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
#分发证书文件
scp -r /opt/TLS/k8s/ssl/kube-controller-manager*.pem /opt/kubernetes/ssl/
#分发配置文件
scp -r /opt/TLS/k8s/cfg/kube-controller-manager.conf /opt/kubernetes/cfg/
#分发管理文件
scp /opt/TLS/k8s/cfg/kube-controller-manager.service /usr/lib/systemd/system/kube-controller-manager.service
#分发kubeconfig文件
scp /opt/TLS/k8s/cfg/kube-controller-manager.kubeconfig /opt/kubernetes/cfg/kube-controller-manager.kubeconfig
#核对证书文件
[root@k8s-master cfg]# ll /opt/kubernetes/ssl/kube-controller-manager*.pem
-rw------- 1 root root 1679 Apr 3 14:30 /opt/kubernetes/ssl/kube-controller-manager-key.pem
-rw-r--r-- 1 root root 1436 Apr 3 14:30 /opt/kubernetes/ssl/kube-controller-manager.pem
#核对配置文件
[root@k8s-master cfg]# ll /opt/kubernetes/cfg/kube-controller-manager.conf
-rw-r--r-- 1 root root 582 Apr 3 14:30 /opt/kubernetes/cfg/kube-controller-manager.conf
#核对管理文件
[root@k8s-master cfg]# ll /usr/lib/systemd/system/kube-controller-manager.service
-rw-r--r-- 1 root root 321 Apr 3 14:30 /usr/lib/systemd/system/kube-controller-manager.service
#核对kubeconfig文件
[root@k8s-master cfg]# ll /opt/kubernetes/cfg/kube-controller-manager.kubeconfig
-rw------- 1 root root 6279 Apr 3 14:30 /opt/kubernetes/cfg/kube-controller-manager.kubeconfig
[root@k8s-master cfg]# systemctl daemon-reload && systemctl start kube-controller-manager && systemctl enable kube-controller-manager && systemctl status kube-controller-manager
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-controller-manager.service to /usr/lib/systemd/system/kube-controller-manager.service.
● kube-controller-manager.service - Kubernetes Controller Manager
Loaded: loaded (/usr/lib/systemd/system/kube-controller-manager.service; enabled; vendor preset: disabled)
Active: active (running) since Sun 2022-04-03 14:33:09 CST; 111ms ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 11872 (kube-controller)
CGroup: /system.slice/kube-controller-manager.service
└─11872 /opt/kubernetes/bin/kube-controller-manager --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --leader-elect=true --kubeconfig=/opt/kubernetes/cfg/kube-controller-manager.kubec...
Apr 03 14:33:09 k8s-master systemd[1]: Started Kubernetes Controller Manager.
cd /opt/TLS/k8s/cfg/
cat > kube-scheduler.conf << EOF
KUBE_SCHEDULER_OPTS="--logtostderr=false \\
--v=2 \\
--log-dir=/opt/kubernetes/logs \\
--leader-elect \\
--kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig \\
--bind-address=127.0.0.1"
EOF
cd /opt/TLS/k8s/ssl
cat > kube-scheduler-csr.json << EOF
{
"CN": "system:kube-scheduler",
"hosts": [],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "system:masters",
"OU": "System"
}
]
}
EOF
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler
[root@k8s-master ssl]# ll kube-scheduler*
-rw-r--r-- 1 root root 1029 Apr 3 14:37 kube-scheduler.csr
-rw-r--r-- 1 root root 245 Apr 3 14:37 kube-scheduler-csr.json
-rw------- 1 root root 1675 Apr 3 14:37 kube-scheduler-key.pem
-rw-r--r-- 1 root root 1424 Apr 3 14:37 kube-scheduler.pem
#这里生成了kube-scheduler.pem和kube-scheduler-key.pem文件
# 设置集群参数
kubectl config set-cluster kubernetes \
--certificate-authority=/opt/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=https://192.168.190.147:6443 \
--kubeconfig=/opt/TLS/k8s/cfg/kube-scheduler.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kube-scheduler \
--client-certificate=./kube-scheduler.pem \
--client-key=./kube-scheduler-key.pem \
--embed-certs=true \
--kubeconfig=/opt/TLS/k8s/cfg/kube-scheduler.kubeconfig
# 设置上下文参数
kubectl config set-context default \
--cluster=kubernetes \
--user=kube-scheduler \
--kubeconfig=/opt/TLS/k8s/cfg/kube-scheduler.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=/opt/TLS/k8s/cfg/kube-scheduler.kubeconfig
cd /opt/TLS/k8s/cfg
cat > kube-scheduler.service << EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
[Service]
EnvironmentFile=/opt/kubernetes/cfg/kube-scheduler.conf
ExecStart=/opt/kubernetes/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF
#分发配置文件
scp /opt/TLS/k8s/cfg/kube-scheduler.conf /opt/kubernetes/cfg/kube-scheduler.conf
#分发证书文件
scp /opt/TLS/k8s/ssl/kube-scheduler*.pem /opt/kubernetes/ssl/
#分发kubeconfig文件
scp /opt/TLS/k8s/cfg/kube-scheduler.kubeconfig /opt/kubernetes/cfg/kube-scheduler.kubeconfig
#分发管理文件
scp /opt/TLS/k8s/cfg/kube-scheduler.service /usr/lib/systemd/system/kube-scheduler.service
#核对配置文件
[root@k8s-master cfg]# ll /opt/kubernetes/cfg/kube-scheduler.conf
-rw-r--r-- 1 root root 188 Apr 3 14:44 /opt/kubernetes/cfg/kube-scheduler.conf
#核对证书文件
[root@k8s-master cfg]# ll /opt/kubernetes/ssl/kube-scheduler*.pem
-rw------- 1 root root 1675 Apr 3 14:45 /opt/kubernetes/ssl/kube-scheduler-key.pem
-rw-r--r-- 1 root root 1424 Apr 3 14:45 /opt/kubernetes/ssl/kube-scheduler.pem
#核对kubeconfig文件
[root@k8s-master cfg]# ll /opt/kubernetes/cfg/kube-scheduler.kubeconfig
-rw------- 1 root root 6241 Apr 3 14:45 /opt/kubernetes/cfg/kube-scheduler.kubeconfig
#核对管理文件
[root@k8s-master cfg]# ll /usr/lib/systemd/system/kube-scheduler.service
-rw-r--r-- 1 root root 285 Apr 3 14:45 /usr/lib/systemd/system/kube-scheduler.service
systemctl daemon-reload && systemctl start kube-scheduler && systemctl enable kube-scheduler && systemctl status kube-scheduler
Created symlink from /etc/systemd/system/multi-user.target.wants/kube-scheduler.service to /usr/lib/systemd/system/kube-scheduler.service.
● kube-scheduler.service - Kubernetes Scheduler
Loaded: loaded (/usr/lib/systemd/system/kube-scheduler.service; enabled; vendor preset: disabled)
Active: active (running) since Sun 2022-04-03 14:48:19 CST; 113ms ago
Docs: https://github.com/kubernetes/kubernetes
Main PID: 11972 (kube-scheduler)
CGroup: /system.slice/kube-scheduler.service
└─11972 /opt/kubernetes/bin/kube-scheduler --logtostderr=false --v=2 --log-dir=/opt/kubernetes/logs --leader-elect --kubeconfig=/opt/kubernetes/cfg/kube-scheduler.kubeconfig --bind-address=12...
Apr 03 14:48:19 vm01 systemd[1]: Started Kubernetes Scheduler.
Apr 03 14:48:19 vm01 kube-scheduler[11972]: Flag --logtostderr has been deprecated, will be removed in a future release, see https://github.com/kubernetes/enhancements/tree/master/keps/sig...k8s-components
Apr 03 14:48:19 vm01 kube-scheduler[11972]: Flag --log-dir has been deprecated, will be removed in a future release, see https://github.com/kubernetes/enhancements/tree/master/keps/sig-ins...k8s-components
Hint: Some lines were ellipsized, use -l to show in full.
至此,Master节点上的三个组件(Apiserver、ControllerManager、Scheduler)已部署并启动成功。