Harbor边缘部署文档

$ openssl genrsa -out ca.key 4096

# 将yourdomain.com修改为需要配置的域名，本文当中使用cedhub.com
$ openssl req -x509 -new -nodes -sha512 -days 3650 \
 -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=yourdomain.com" \
 -key ca.key \
 -out ca.crt

$ openssl genrsa -out tls.key 4096

# 将yourdomain.com修改为需要配置的域名，本文当中使用cedhub.com
$ openssl req -sha512 -new \
    -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=yourdomain.com" \
    -key tls.key \
    -out tls.csr

# 将yourdomain.com修改为需要配置的域名，本文当中使用cedhub.com
$ cat > v3.ext <<-EOF
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names

[alt_names]
DNS.1=yourdomain.com
EOF

$ openssl x509 -req -sha512 -days 3650 \
    -extfile v3.ext \
    -CA ca.crt -CAkey ca.key -CAcreateserial \
    -in tls.csr \
    -out tls.crt

$ helm repo add harbor https://helm.goharbor.io
# 查看可以安装的chart（包含历史版本）
$ helm search repo harbor -l
# 安装1.8.1版本的Chart（2.4.1版本的Harbor）
$ helm fetch harbor/harbor --untar --version 1.8.1

# kubectl create namespace harbor

# 创建secret，将第一步生成的证书导入
kubectl create secret generic cedhub-tls --from-file=tls.crt --from-file=tls.key --from-file=ca.crt -n harbor
# 配置value.yaml
...
tls:
    # Enable the tls or not.
    # Delete the "ssl-redirect" annotations in "expose.ingress.annotations" when TLS is disabled and "expose.type" is "ingress"
    # Note: if the "expose.type" is "ingress" and the tls
    # is disabled, the port must be included in the command when pull/push
    # images. Refer to https://github.com/goharbor/harbor/issues/5291
    # for the detail.
    enabled: true
    # The source of the tls certificate. Set it as "auto", "secret"
    # or "none" and fill the information in the corresponding section
    # 1) auto: generate the tls certificate automatically
    # 2) secret: read the tls certificate from the specified secret.
    # The tls certificate can be generated manually or by cert manager
    # 3) none: configure no tls certificate for the ingress. If the default
    # tls certificate is configured in the ingress controller, choose this option
    certSource: secret
    auto:
      # The common name used to generate the certificate, it's necessary
      # when the type isn't "ingress"
      commonName: ""
    secret:
      # The name of secret which contains keys named:
      # "tls.crt" - the certificate
      # "tls.key" - the private key
      secretName: "cedhub-tls"
      # The name of secret which contains keys named:
      # "tls.crt" - the certificate
      # "tls.key" - the private key
      # Only needed when the "expose.type" is "ingress".
      notarySecretName: "cedhub-tls"
...

expose:
  # Set the way how to expose the service. Set the type as "ingress",
  # "clusterIP", "nodePort" or "loadBalancer" and fill the information
  # in the corresponding section
  type: clusterIP
  ...

# The external URL for Harbor core service. It is used to
# 1) populate the docker/helm commands showed on portal
# 2) populate the token service URL returned to docker/notary client
#
# Format: protocol://domain[:port]. Usually:
# 1) if "expose.type" is "ingress", the "domain" should be
# the value of "expose.ingress.hosts.core"
# 2) if "expose.type" is "clusterIP", the "domain" should be
# the value of "expose.clusterIP.name"
# 3) if "expose.type" is "nodePort", the "domain" should be
# the IP address of k8s node
#
# If Harbor is deployed behind the proxy, set it as the URL of proxy
externalURL: https://cedhub.com

mkdir -p /data/disks/{disk1-5G,disk2-5G,disk3-1G,disk4-1G,disk5-1G,disk6-5G}

apiVersion: v1
kind: PersistentVolume
metadata:
  name: disk1
spec:
  storageClassName: manual
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  hostPath:
    path: "/data/disks/disk1-5G"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - cvm-lrxdi7pm
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: disk2
spec:
  storageClassName: manual
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  hostPath:
    path: "/data/disks/disk2-5G"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - cvm-lrxdi7pm
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: disk3
spec:
  storageClassName: manual
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  hostPath:
    path: "/data/disks/disk3-1G"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - cvm-lrxdi7pm
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: disk4
spec:
  storageClassName: manual
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  hostPath:
    path: "/data/disks/disk4-1G"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - cvm-lrxdi7pm
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: disk5
spec:
  storageClassName: manual
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  hostPath:
    path: "/data/disks/disk5-1G"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - cvm-lrxdi7pm
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: disk6
spec:
  storageClassName: manual
  capacity:
    storage: 5Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Recycle
  hostPath:
    path: "/data/disks/disk6-5G"
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - cvm-lrxdi7pm

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: disk1-pv-claim
  namespace: harbor
spec:
  storageClassName: manual
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 5Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: disk2-pv-claim
  namespace: harbor
spec:
  storageClassName: manual
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 5Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: disk3-pv-claim
  namespace: harbor
spec:
  storageClassName: manual
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: disk4-pv-claim
  namespace: harbor
spec:
  storageClassName: manual
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: disk5-pv-claim
  namespace: harbor
spec:
  storageClassName: manual
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 1Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: disk6-pv-claim
  namespace: harbor
spec:
  storageClassName: manual
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 5Gi

···
# For storing images and charts, you can also use "azure", "gcs", "s3",
# "swift" or "oss". Set it in the "imageChartStorage" section
persistence:
  enabled: true
  # Setting it to "keep" to avoid removing PVCs during a helm delete
  # operation. Leaving it empty will delete PVCs after the chart deleted
  # (this does not apply for PVCs that are created for internal database
  # and redis components, i.e. they are never deleted automatically)
  resourcePolicy: "keep"
  persistentVolumeClaim:
    registry:
      # Use the existing PVC which must be created manually before bound,
      # and specify the "subPath" if the PVC is shared with other components
      existingClaim: "disk1-pv-claim"
      # Specify the "storageClass" used to provision the volume. Or the default
      # StorageClass will be used(the default).
      # Set it to "-" to disable dynamic provisioning
      storageClass: ""
      subPath: ""
      accessMode: ReadWriteOnce
      size: 5Gi
    chartmuseum:
      existingClaim: "disk2-pv-claim"
      storageClass: ""
      subPath: ""
      accessMode: ReadWriteOnce
      size: 5Gi
    jobservice:
      existingClaim: "disk3-pv-claim"
      storageClass: ""
      subPath: ""
      accessMode: ReadWriteOnce
      size: 1Gi
    # If external database is used, the following settings for database will
    # be ignored
    database:
      existingClaim: "disk4-pv-claim"
      storageClass: ""
      subPath: ""
      accessMode: ReadWriteOnce
      size: 1Gi
    # If external Redis is used, the following settings for Redis will
    # be ignored
    redis:
      existingClaim: "disk5-pv-claim"
      storageClass: ""
      subPath: ""
      accessMode: ReadWriteOnce
      size: 1Gi
    trivy:
      existingClaim: "disk6-pv-claim"
      storageClass: ""
      subPath: ""
      accessMode: ReadWriteOnce
      size: 5Gi
···

···
# For storing images and charts, you can also use "azure", "gcs", "s3",
# "swift" or "oss". Set it in the "imageChartStorage" section
persistence:
  enabled: true
  # Setting it to "keep" to avoid removing PVCs during a helm delete
  # operation. Leaving it empty will delete PVCs after the chart deleted
  # (this does not apply for PVCs that are created for internal database
  # and redis components, i.e. they are never deleted automatically)
  resourcePolicy: "keep"
  persistentVolumeClaim:
    registry:
      # Use the existing PVC which must be created manually before bound,
      # and specify the "subPath" if the PVC is shared with other components
      existingClaim: "
      # Specify the "storageClass" used to provision the volume. Or the default
      # StorageClass will be used(the default).
      # Set it to "-" to disable dynamic provisioning
      storageClass: "storageClassName"
      subPath: ""
      accessMode: ReadWriteOnce
      size: 5Gi
    chartmuseum:
      existingClaim: ""
      storageClass: "storageClassName"
      subPath: ""
      accessMode: ReadWriteOnce
      size: 5Gi
    jobservice:
      existingClaim: ""
      storageClass: "storageClassName"
      subPath: ""
      accessMode: ReadWriteOnce
      size: 1Gi
    # If external database is used, the following settings for database will
    # be ignored
    database:
      existingClaim: ""
      storageClass: "storageClassName"
      subPath: ""
      accessMode: ReadWriteOnce
      size: 1Gi
    # If external Redis is used, the following settings for Redis will
    # be ignored
    redis:
      existingClaim: ""
      storageClass: "storageClassName"
      subPath: ""
      accessMode: ReadWriteOnce
      size: 1Gi
    trivy:
      existingClaim: ""
      storageClass: "storageClassName"
      subPath: ""
      accessMode: ReadWriteOnce
      size: 5Gi
···

# 在harbor目录下
$ helm install --namespace harbor my-harbor .

$ helm uninstall my-harbor -n harbor

$ openssl x509 -inform PEM -in tls.crt -out tls.cert

# 将yourdomain.com修改为需要配置的域名，本文当中使用cedhub.com
# 如果将默认的443端口映射为其他端口，则需创建/etc/docker/certs.d/yourdomain.com:port或/etc/docker/certs.d/harbor_IP:port
$ mkdir -p /etc/docker/certs.d/yourdomain.com/

# 将yourdomain.com修改为需要配置的域名，本文当中使用cedhub.com
cp tls.cert /etc/docker/certs.d/yourdomain.com/
cp tls.key /etc/docker/certs.d/yourdomain.com/
cp ca.crt /etc/docker/certs.d/yourdomain.com/

# docker-server为service harbor的域名
kubectl create secret docker-registry registry-secret --namespace=default  \
--docker-server=cedhub.com \
--docker-username=admin \
--docker-password=Harbor12345

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
  labels: 
    appname: nginx
spec:
  selector:
    matchLabels:
     appname: nginx
  replicas: 3
  template:
    metadata:
      labels:
        appname: nginx
    spec:
      containers:
      - name: nginx
        image: cedhub.com/preheat/nginx:latest  # 边缘镜像地址
        ports:
        - containerPort: 80
      imagePullSecrets:  # 使用边缘Harbor的secret
      - name: registry-secret

apiVersion: v1
kind: Service
metadata:
  name: nginx-service
spec:
  selector:
    appname: nginx
  type: NodePort
  ports:
  - protocol: TCP
    port: 80
    targetPort: 80
    nodePort: 30080

# -k允许curl不验证证书
# -v详细输出，输出Headers等信息
# -i输出Response Body
# authorization: Basic 配置经base64编码的用户名密码，格式为：username:password
$ curl -k -v -i -X 'POST' 'https://cedhub.com/api/v2.0/registries/ping' -H 'accept: application/json' -H 'authorization: Basic YWRtaW46SGFyYm9yMTIzNDU=' -H 'Content-Type: application/json' -d '{
    "access_key": "AKIDNa7NM5WgZBWaNvEH3XliOoywd3D6i37v",
    "access_secret": "YpujHYYHkwrUqvjGWVDdngj6LL4WhJgm",
    "description": "",
    "insecure": false,
    "name": "test",
    "type": "tencent-tcr",
    "url": "https://preheat-test.tencentcloudcr.com"
}'

# -k允许curl不验证证书
# -v详细输出，输出Headers等信息
# -i输出Response Body
# authorization: Basic 配置经base64编码的用户名密码，格式为：username:password
$ curl -k -v -i -X 'POST' 'https://cedhub.com/api/v2.0/registries' -H 'accept: application/json' -H 'authorization: Basic YWRtaW46SGFyYm9yMTIzNDU='   -H 'Content-Type: application/json' -d '{
  "credential": {
    "access_key": "AKIDNa7NM5WgZBWaNvEH3XliOoywd3D6i37v",
    "access_secret": "YpujHYYHkwrUqvjGWVDdngj6LL4WhJgm",
    "type": "basic"
  },
  "description": "",
  "insecure": false,
  "name": "test",
  "type": "tencent-tcr",
  "url": "https://preheat-test.tencentcloudcr.com"
}'

# -k允许curl不验证证书
# -v详细输出，输出Headers等信息
# -i输出Response Body
# authorization: Basic 配置经base64编码的用户名密码，格式为：username:password
$ curl -k -v -i -X 'GET' 'https://cedhub.com/api/v2.0/registries?page=1&page_size=10' -H 'accept: application/json' -H 'authorization: Basic YWRtaW46SGFyYm9yMTIzNDU='

{
  "name": "sync_test",
  "description": "",
  "src_registry": {
    "creation_time": "2022-02-23T04:01:40.344Z",
    "credential": {
      "access_key": "AKIDNa7NM5WgZBWaNvEH3XliOoywd3D6i37v",
      "access_secret": "*****",
      "type": "basic"
    },
    "id": 1,
    "name": "test",
    "status": "healthy",
    "type": "tencent-tcr",
    "update_time": "2022-02-23T04:01:40.344Z",
    "url": "https://preheat-test.tencentcloudcr.com"
}
  "dest_registry": null,
  "dest_namespace": "library",
  "dest_namespace_replace_count": 1,
  "trigger": {
    "t15pe": "manual",
    "trigger_settings": {
      "cron": ""
    }
  },
  "enabled": true,
  "deletion": false,
  "override": false,
  "speed": -1,
  "filters": [
    {
      "type": "name",
      "value": "preheat/**"
    },
    {
      "type": "tag",
      "decoration": "matches",
      "value": "latest"
    },
    {
      "type": "resource",
      "value": "image"
    }
  ]
}

"src_registry": {
    "creation_time": "2022-02-23T03:49:16.652Z",
    "credential": {
      "access_key": "AKIDNa7NM5WgZBWaNvEH3XliOoywd3D6i37v",
      "access_secret": "*****",
      "type": "basic"
    },
    "id": 7,
    "name": "test",
    "status": "healthy",
    "type": "tencent-tcr",
    "update_time": "2022-02-23T03:49:16.652Z",
    "url": "https://preheat-test.tencentcloudcr.com"
  },
  

"filters": [
    {
      "type": "name",
      "value": "preheat/**"
    },
    {
      "type": "tag",
      "decoration": "matches",
      "value": "latest"
    },
    {
      "type": "resource",
      "value": "image"
    }
  ]

# -k允许curl不验证证书
# -v详细输出，输出Headers等信息
# -i输出Response Body
# authorization: Basic 配置经base64编码的用户名密码，格式为：username:password
# 需要从获取远端仓库列表接口获取src_registry信息
$ curl -k -v -i -X 'POST' \
  'https://cedhub.com/api/v2.0/replication/policies' \
  -H 'accept: application/json' \
  -H 'authorization: Basic YWRtaW46SGFyYm9yMTIzNDU=' \
  -H 'Content-Type: application/json' -d '
{
  "name": "test_sync",
  "description": "",
  "src_registry": {
    "creation_time": "2022-02-23T04:01:40.344Z",
    "credential": {
      "access_key": "AKIDNa7NM5WgZBWaNvEH3XliOoywd3D6i37v",
      "access_secret": "*****",
      "type": "basic"
    },
    "id": 1,
    "name": "test",
    "status": "healthy",
    "type": "tencent-tcr",
    "update_time": "2022-02-23T04:01:40.344Z",
    "url": "https://preheat-test.tencentcloudcr.com"
  },
  "dest_registry": null,
  "dest_namespace": "",
  "dest_namespace_replace_count": 1,
  "trigger": {
    "type": "manual",
    "trigger_settings": {
      "cron": ""
    }
  },
  "enabled": true,
  "deletion": false,
  "override": true,
  "speed": -1,
  "filters": [
    {
      "type": "name",
      "value": "preheat/**"
    }
  ]
} '

# -k允许curl不验证证书
# -v详细输出，输出Headers等信息
# -i输出Response Body
# authorization: Basic 配置经base64编码的用户名密码，格式为：username:password
$ curl -k -v -i -X 'GET' \
  'https://cedhub.com/api/v2.0/replication/policies?page=1&page_size=10' \
  -H 'accept: application/json' \
  -H 'authorization: Basic YWRtaW46SGFyYm9yMTIzNDU='

# -k允许curl不验证证书
# -v详细输出，输出Headers等信息
# -i输出Response Body
# authorization: Basic 配置经base64编码的用户名密码，格式为：username:password
# 需要从列出远程复制策略接口获取policy信息
$ curl -k -v -i -X 'POST' \
  'https://cedhub.com/api/v2.0/replication/executions' \
  -H 'accept: application/json' \
  -H 'Content-Type: application/json' \
  -H 'authorization: Basic YWRtaW46SGFyYm9yMTIzNDU=' \
  -d '{
  "policy_id": 1
}'

# -k允许curl不验证证书
# -v详细输出，输出Headers等信息
# -i输出Response Body
# authorization: Basic 配置经base64编码的用户名密码，格式为：username:password
# 需要从列出远程复制策略接口获取policy信息
$ curl -k -v -i  -X 'GET' \
  'https://cedhub.com/api/v2.0/replication/executions?page=1&page_size=10&policy_id=1' \
  -H 'accept: application/json'\
  -H 'authorization: Basic YWRtaW46SGFyYm9yMTIzNDU='