基于AWS EKS的K8S实践 - 如何打通云企业网集群内外服务调用

shysh95

发布于 2023-08-23 10:05:53

2950

发布于 2023-08-23 10:05:53

文章被收录于专栏：shysh95shysh95

集群内服务的暴露方式？

service
ingress

service 通常用作集群内服务之前的通信，ingress 通常用于暴露给集群外的服务使用。

由于我们这里的需求是将集群内的服务暴露给集群外的服务使用，所以我们这里选择 ingress 。

ingress controller 如何选择？

单纯的 ingress 是没有任何实际作用的，ingress 需要搭配 ingress controller 才会有意义，我们这里的需求是将集群内的服务暴露给我们其他的服务使用，本质上这里还是要通过内网进行访问，我们这里选择的是 ingress alb controller，并且 ALB 的类型是 internal，也就是不面向互联网的类型。

安装 ingress controller

创建身份提供商，这里需要填入EKS的提供商URL（该URL可以从EKS控制台拿到），然后获取指纹，受众固定填写sts.amazonaws.com，如下图：

2. 创建一个role，假设名字为aws-test-eks-alb-controller-role，信任实体填写以下内容：

{
 
"Version": "2012-10-17",
 
"Statement": [
 
{
 
"Effect": "Allow",
 
"Principal": {
 
"Federated": "arn:aws:iam::xxxxxx:oidc-provider/oidc.eks.ap-southeast-3.amazonaws.com/idxxxxxx"
 
},
 
"Action": "sts:AssumeRoleWithWebIdentity",
 
"Condition": {
 
"StringEquals": {
 
"oidc.eks.ap-southeast-3.amazonaws.com/id/xxxxx:sub": "system:serviceaccount:kube-system:aws-load-balancer-controller",
 
"oidc.eks.ap-southeast-3.amazonaws.com/id/xxxxx:aud": "sts.amazonaws.com"
 
}
 
}
 
}
 
]
 
}

3. 创建一个Policy，Policy内容如下，Policy创建成功以后附加到上面的role上：

{
 
"Version": "2012-10-17",
 
"Statement": [
 
{
 
"Effect": "Allow",
 
"Action": [
 
"iam:CreateServiceLinkedRole"
 
],
 
"Resource": "*",
 
"Condition": {
 
"StringEquals": {
 
"iam:AWSServiceName": "elasticloadbalancing.amazonaws.com"
 
}
 
}
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"ec2:DescribeAccountAttributes",
 
"ec2:DescribeAddresses",
 
"ec2:DescribeAvailabilityZones",
 
"ec2:DescribeInternetGateways",
 
"ec2:DescribeVpcs",
 
"ec2:DescribeVpcPeeringConnections",
 
"ec2:DescribeSubnets",
 
"ec2:DescribeSecurityGroups",
 
"ec2:DescribeInstances",
 
"ec2:DescribeNetworkInterfaces",
 
"ec2:DescribeTags",
 
"ec2:GetCoipPoolUsage",
 
"ec2:DescribeCoipPools",
 
"elasticloadbalancing:DescribeLoadBalancers",
 
"elasticloadbalancing:DescribeLoadBalancerAttributes",
 
"elasticloadbalancing:DescribeListeners",
 
"elasticloadbalancing:DescribeListenerCertificates",
 
"elasticloadbalancing:DescribeSSLPolicies",
 
"elasticloadbalancing:DescribeRules",
 
"elasticloadbalancing:DescribeTargetGroups",
 
"elasticloadbalancing:DescribeTargetGroupAttributes",
 
"elasticloadbalancing:DescribeTargetHealth",
 
"elasticloadbalancing:DescribeTags"
 
],
 
"Resource": "*"
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"cognito-idp:DescribeUserPoolClient",
 
"acm:ListCertificates",
 
"acm:DescribeCertificate",
 
"iam:ListServerCertificates",
 
"iam:GetServerCertificate",
 
"waf-regional:GetWebACL",
 
"waf-regional:GetWebACLForResource",
 
"waf-regional:AssociateWebACL",
 
"waf-regional:DisassociateWebACL",
 
"wafv2:GetWebACL",
 
"wafv2:GetWebACLForResource",
 
"wafv2:AssociateWebACL",
 
"wafv2:DisassociateWebACL",
 
"shield:GetSubscriptionState",
 
"shield:DescribeProtection",
 
"shield:CreateProtection",
 
"shield:DeleteProtection"
 
],
 
"Resource": "*"
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"ec2:AuthorizeSecurityGroupIngress",
 
"ec2:RevokeSecurityGroupIngress"
 
],
 
"Resource": "*"
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"ec2:CreateSecurityGroup"
 
],
 
"Resource": "*"
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"ec2:CreateTags"
 
],
 
"Resource": "arn:aws:ec2:*:*:security-group/*",
 
"Condition": {
 
"StringEquals": {
 
"ec2:CreateAction": "CreateSecurityGroup"
 
},
 
"Null": {
 
"aws:RequestTag/elbv2.k8s.aws/cluster": "false"
 
}
 
}
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"ec2:CreateTags",
 
"ec2:DeleteTags"
 
],
 
"Resource": "arn:aws:ec2:*:*:security-group/*",
 
"Condition": {
 
"Null": {
 
"aws:RequestTag/elbv2.k8s.aws/cluster": "true",
 
"aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
 
}
 
}
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"ec2:AuthorizeSecurityGroupIngress",
 
"ec2:RevokeSecurityGroupIngress",
 
"ec2:DeleteSecurityGroup"
 
],
 
"Resource": "*",
 
"Condition": {
 
"Null": {
 
"aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
 
}
 
}
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"elasticloadbalancing:CreateLoadBalancer",
 
"elasticloadbalancing:CreateTargetGroup"
 
],
 
"Resource": "*",
 
"Condition": {
 
"Null": {
 
"aws:RequestTag/elbv2.k8s.aws/cluster": "false"
 
}
 
}
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"elasticloadbalancing:CreateListener",
 
"elasticloadbalancing:DeleteListener",
 
"elasticloadbalancing:CreateRule",
 
"elasticloadbalancing:DeleteRule"
 
],
 
"Resource": "*"
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"elasticloadbalancing:AddTags",
 
"elasticloadbalancing:RemoveTags"
 
],
 
"Resource": [
 
"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
 
"arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
 
"arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
 
],
 
"Condition": {
 
"Null": {
 
"aws:RequestTag/elbv2.k8s.aws/cluster": "true",
 
"aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
 
}
 
}
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"elasticloadbalancing:AddTags",
 
"elasticloadbalancing:RemoveTags"
 
],
 
"Resource": [
 
"arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*",
 
"arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*",
 
"arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*",
 
"arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*"
 
]
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"elasticloadbalancing:ModifyLoadBalancerAttributes",
 
"elasticloadbalancing:SetIpAddressType",
 
"elasticloadbalancing:SetSecurityGroups",
 
"elasticloadbalancing:SetSubnets",
 
"elasticloadbalancing:DeleteLoadBalancer",
 
"elasticloadbalancing:ModifyTargetGroup",
 
"elasticloadbalancing:ModifyTargetGroupAttributes",
 
"elasticloadbalancing:DeleteTargetGroup"
 
],
 
"Resource": "*",
 
"Condition": {
 
"Null": {
 
"aws:ResourceTag/elbv2.k8s.aws/cluster": "false"
 
}
 
}
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"elasticloadbalancing:AddTags"
 
],
 
"Resource": [
 
"arn:aws:elasticloadbalancing:*:*:targetgroup/*/*",
 
"arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*",
 
"arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*"
 
],
 
"Condition": {
 
"StringEquals": {
 
"elasticloadbalancing:CreateAction": [
 
"CreateTargetGroup",
 
"CreateLoadBalancer"
 
]
 
},
 
"Null": {
 
"aws:RequestTag/elbv2.k8s.aws/cluster": "false"
 
}
 
}
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"elasticloadbalancing:RegisterTargets",
 
"elasticloadbalancing:DeregisterTargets"
 
],
 
"Resource": "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*"
 
},
 
{
 
"Effect": "Allow",
 
"Action": [
 
"elasticloadbalancing:SetWebAcl",
 
"elasticloadbalancing:ModifyListener",
 
"elasticloadbalancing:AddListenerCertificates",
 
"elasticloadbalancing:RemoveListenerCertificates",
 
"elasticloadbalancing:ModifyRule"
 
],
 
"Resource": "*"
 
}
 
]
 
}

4. 在kube-system的namespace下创建一个SA，SA的名称是aws-load-balancer-controller，SA的YAML文件定义如下：

apiVersion: v1
 
kind: ServiceAccount
 
metadata:
 
  annotations:
 
    eks.amazonaws.com/role-arn: arn:aws:iam::xxxxx:role/aws-test-eks-alb-controller-role
 
  labels:
 
    app.kubernetes.io/component: controller
 
    app.kubernetes.io/name: aws-load-balancer-controller
 
  name: aws-load-balancer-controller
 
namespace: kube-system

5. 获取ingress aws alb contorller的资源清单文件，修改后进行应用

wget https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.5.1/v2_5_1_full.yaml

对资源文件中的内容进行修改，首先删除ServiceAccount（内容如下图），第三步我们已经创建过了。

其次我们需要修改Deployment中cluster-name变成我们的ESK集群名称，我的集群名称是test-eks，所以修改后的信息如下图：

最后应用我们的资源清单文件：

kubectl apply -f v2_5_1_full.yaml

6. 获取ingress class类文件，并应用该资源文件

wget https://github.com/kubernetes-sigs/aws-load-balancer-controller/releases/download/v2.5.1/v2_5_1_ingclass.yaml
 


 
kubectl apply -f v2_5_1_ingclass.yaml

ingress原理

通过ingress让集群外的服务访问我们集群内的服务，通过上图可以看出大约需要4个组件

Ingress Controller：这里我们选用的是aws的alb
Ingress：具体的ingress，这里面会配置路由规则，我这里是按照环境建立ingress
Service：服务，用来对Pod做负载均衡
Pod：具体的服务，这里的Pod我们通常不单独部署，一般通过Deployment组件进行维护

Service配置

假设我们的Pod已经配置好了，这里我们看一下Service的资源清单文件配置，如下：

kind: Service
 
apiVersion: v1
 
metadata:
 
  name: test
 
namespace: application-test
 
  annotations:
 
    alb.ingress.kubernetes.io/healthcheck-interval-seconds: '8'
 
    alb.ingress.kubernetes.io/healthcheck-path: /sys/health/check
 
    alb.ingress.kubernetes.io/healthcheck-timeout-seconds: '5'
 
    alb.ingress.kubernetes.io/healthy-threshold-count: '3'
 
    alb.ingress.kubernetes.io/unhealthy-threshold-count: '2'
 
spec:
 
  ports:
 
- protocol: TCP
 
      port: 80
 
      targetPort: 8009
 
  selector:
 
    app: xxxxx
 
  type: ClusterIP

关于Service我们这里不再赘述，我们主要关心和AWS相关的配置，也就是annotation的信息，anntation这里我们主要配置的是目标组里的健康监测信息，当我们对上述资源文件进行应用以后，目标组上健康检测信息配置和上图将会对齐，如下图：

Ingress配置

这里我们将会定义我们的ingress资源文件，推荐按照环境进行区分（毕竟我们的Service以及Pod都是和环境对应的），一个ingress资源文件将会建立一个ALB负责均衡器，ingress资源文件示例如下:

kind: Ingress
 
apiVersion: networking.k8s.io/v1
 
metadata:
 
  name: alb-internal-ingress-backend
 
namespace: application-test
 
  annotations:
 
    alb.ingress.kubernetes.io/load-balancer-name: test-k8s-application
 
    alb.ingress.kubernetes.io/scheme: internal
 
    alb.ingress.kubernetes.io/security-groups: sg-xxxx
 
    alb.ingress.kubernetes.io/subnets: 'subnet-xxxx,subnet-xxxx'
 
    alb.ingress.kubernetes.io/target-type: ip
 
spec:
 
  ingressClassName: alb
 
  rules:
 
- host: test.xxx.example.com
 
      http:
 
        paths:
 
- path: /
 
            pathType: Prefix
 
            backend:
 
              service:
 
                name: test
 
                port:
 
                  number: 80

首先这里的ingressClassName必须指定为alb，这表明我们要使用alb controller对他进行控制，其次我们关注一下ingress的annotations：

alb.ingress.kubernetes.io/scheme：我们指定为internal，表明我们的alb不面向互联网，是面向内部网络的
alb.ingress.kubernetes.io/security-groups：指定alb上的安全组
alb.ingress.kubernetes.io/subnets：指定alb的子网
alb.ingress.kubernetes.io/target-type：指定目标组的注册类型，默认是实例，这里我们修改为IP，这个annotation也可以用在Service上

下面的rule表示域名test.xxx.example.com的请求将会被转发到test service的80端口，进而通过Service后端Pod负载上。

上述资源清单文件安装完成以后，会在AWS负载均衡器上创建一个alb，如下图：