基于GitLab+Docker+K8S的持续集成和交付

sudo yum remove docker \
                  docker-client \
                  docker-client-latest \
                  docker-common \
                  docker-latest \
                  docker-latest-logrotate \
                  docker-logrotate \
                  docker-selinux \
                  docker-engine-selinux \
                  docker-engine

# Step 1: 安装必要的一些系统工具
sudo yum install -y yum-utils device-mapper-persistent-data lvm2

# Step 2: 添加软件源信息
sudo yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

# Step 3: 更新并安装 Docker-CE
sudo yum makecache fast
sudo yum -y install docker-ce

# Step 4：添加当前用户到docker组中允许普通用户运行docker命令，此步需要登出linux环境重新再登录方能生效
sudo usermod -aG docker $USER

# Step 5：添加开机启动
sudo systemctl enable docker.service

# Step 6：启动docker
sudo systemctl start docker.service

# Step 7：查看docker版本
docker version

# Step 8：验证docker环境是否能运行镜像
docker run hello-world

#运行效果如下所示：
Hello from Docker!
This message shows that your installation appears to be working correctly.

To generate this message, Docker took the following steps:
 1. The Docker client contacted the Docker daemon.
 2. The Docker daemon pulled the "hello-world" image from the Docker Hub.
    (amd64)
 3. The Docker daemon created a new container from that image which runs the
    executable that produces the output you are currently reading.
 4. The Docker daemon streamed that output to the Docker client, which sent it
    to your terminal.

To try something more ambitious, you can run an Ubuntu container with:
 $ docker run -it ubuntu bash

Share images, automate workflows, and more with a free Docker ID:
 https://hub.docker.com/

For more examples and ideas, visit:
 https://docs.docker.com/engine/userguide/

sudo docker run --detach \
    --hostname gitlab.example.com \
    --publish 443:443 --publish 80:80 --publish 22:22 \ #此处可以不配置22端口如果sshd已经占用该端口
    --name gitlab \
    --restart always \
    --volume /srv/gitlab/config:/etc/gitlab \
    --volume /srv/gitlab/logs:/var/log/gitlab \
    --volume /srv/gitlab/data:/var/opt/gitlab \
    gitlab/gitlab-ce:latest
    
#如果操作系统使用SELinux使用如下命令：
sudo docker run --detach \
    --hostname gitlab.example.com \
    --publish 443:443 --publish 80:80 --publish 22:22 \
    --name gitlab \
    --restart always \
    --volume /srv/gitlab/config:/etc/gitlab:Z \
    --volume /srv/gitlab/logs:/var/log/gitlab:Z \
    --volume /srv/gitlab/data:/var/opt/gitlab:Z \
    gitlab/gitlab-ce:latest

#进入gitlab容器
sudo docker exec -it gitlab /bin/bash
#进入container之后停止gitlab服务
gitlab-ctl stop
#退出container之后
sudo docker stop gitlab

#添加gitlab-runner库
curl -L https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.rpm.sh | sudo bash

# For RHEL/CentOS/Fedora
sudo yum install gitlab-runner

gitlab-runner register \
  --non-interactive \
  --url "http://10.30.13.2/" \ #该参数为gitlab服务器的位置
  --registration-token "F-CKeY1gKWRPxN5XL77N" \ #该以管理员身份从gitlab获取的registration token
  --executor "shell" \ #以shell方式执行任务，支持docker，docker machine，ssh 等等各种执行方式
  --docker-image alpine:3.7 \ #默认的任务镜像
  --description "gitlab-runner" \ #描述
  --tag-list "share-runner-tag" \ #标记在.gitlab-ci.yml会用到
  --run-untagged \ #可以运行在非tag代码上
  --locked="false" \ #是否锁定该执行器

## Configuration file of Harbor

#This attribute is for migrator to detect the version of the .cfg file, DO NOT MODIFY!
_version = 1.5.0
#The IP address or hostname to access admin UI and registry service.
#DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname = 10.30.13.1

sudo docker-compose start

1. 修改 docker-compose.yml

proxy:
    image: library/nginx:1.11.5
    restart: always
    volumes:
      - ./config/nginx:/etc/nginx
    ports:
      - 8888:80 #8888为最终要修改的端口，修改该位置即可
      - 443:443
    depends_on:
      - mysql
      - registry
      - ui
      - log
    logging:
      driver: "syslog"
      options:
        syslog-address: "tcp://127.0.0.1:1514"
        tag: "proxy"

2. 修改 harbor.cfg
hostname = 192.168.0.2:8888

#分别为两个VPC设置hostname，分别为server1，和server2
#第一台VPC执行
sudo hostnamectl set-hostname server1
#第二台VPC执行
sudo hostnamectl set-hostname server2

#分别在两台机器上执行如下命令
#安装依赖包
sudo yum install centos-release-gluster

#为vps添加硬盘(官方建议在新硬盘和分区上使用GFS，但也可以在根分区下使用)，并格式化
sudo mkfs.xfs -i size=512 /dev/sdb1

#创建挂载目录
sudo mkdir -p /bricks/brick1

sudo vi /etc/fstab
#为fstab文件自动挂载记录:
/dev/sdb1 /bricks/brick1 xfs defaults 1 2

#保存文件执行如下命令
sudo mount -a && mount

#安装集群
sudo  yum install glusterfs-server

#允许gfs开机启动
sudo systemctl enable glusterd

#启动gfs服务
sudo systemctl enable glusterd

#检查gfs运行状态
sudo systemctl status glusterd

#创建trusted pool
#在server1执行如下命令
sudo gluster peer probe server2
#在server2执行如下命令
sudo gluster peer probe server1

#在所有机器上分别执行如下命令
sudo mkdir -p /bricks/brick1/gv0
sudo gluster volume create gv0 replica 2 server1:/bricks/brick1/gv0 server2:/bricks/brick1/gv0 force
sudo gluster volume start gv0

#启动卷
sudo gluster volume start gv0

#查看卷状态信息
sudo gluster volume info

#查看卷的详细信息
sudo gluster volume status gv0 detail

#安装相关依赖包
sudo yum install -y centos-release-gluster
sudo yum install -y glusterfs-server
sudo yum install -y glusterfs

#挂载卷
mount -t glusterfs server1:/gv0 /mnt

#测试卷
for i in `seq -w 1 100`; do cp -rp /var/log/messages /mnt/copy-test-$i; done

#执行如下命令查看文件是否生成
ls -lA /mnt | wc -l

#使用fstab开机自动挂载卷
vi /etc/fstab
server1:/test-volume /mnt/glusterfs glusterfs defaults,_netdev 0 0

#各个VPC需要安装Docker，请参考Docker的安装策略

#为各VPC添加hostname，本环境分别为master，node1，node2，node3

#在各VPC机器下执行如下命令

#使用阿里源镜像安装k8s
#安装yum源库
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

#关闭SELinux
sudo setenforce 0
#修改/etc/selinux/config
SELINUX=disabled

#安装依赖库
sudo yum install -y kubelet kubeadm kubectl git conntrack
sudo systemctl enable kubelet

#允许ip转发
sudo sysctl net.bridge.bridge-nf-call-iptables=1
#修改/etc/sysctl.conf:
net.ipv4.ip_forward = 1

#开启
sudo cat <<EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

#查看转发设置是否成功
sudo sysctl --system

#查看cgroup驱动是否一致
docker info | grep -i cgroup
sudo cat /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
#如果不一致，修改如下文件
sed -i "s/cgroup-driver=systemd/cgroup-driver=cgroupfs/g" /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
#如修改了10-kubeadm.conf需要执行如下命令使其生效
sudo systemctl daemon-reload
sudo systemctl restart kubelet
sudo systemctl stop kubelet

##关闭交换分区
swapoff -a
#注释 /etc/fstab 有swap分区的一行

sudo kubeadm init --apiserver-advertise-address=10.30.5.1  --kubernetes-version=1.11.0 --pod-network-cidr=10.244.0.0/16  --token-ttl=0

Your Kubernetes master has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of machines by running the following on each node
as root:

  kubeadm join 10.30.5.1:6443 --token uzsv39.e2d48u4i1wgfw532 --discovery-token-ca-cert-hash sha256:73b8ef1a9ed85b489387abd8f1e6bffb7d7a4e0410f2511bb515b3be3d03b727

 mkdir -p $HOME/.kube
 sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
 sudo chown $(id -u):$(id -g) $HOME/.kube/config

export KUBECONFIG=/etc/kubernetes/admin.conf

#为k8s添加自动提示
echo "source <(kubectl completion bash)" >> ~/.bashrc

 kubectl apply -f kube-flannel.yml

kubeadm join 10.30.5.1:6443 --token uzsv39.e2d48u4i1wgfw532 --discovery-token-ca-cert-hash sha256:73b8ef1a9ed85b489387abd8f1e6bffb7d7a4e0410f2511bb515b3be3d03b727

[root@node1 ~]# kubeadm join 10.30.5.1:6443 --token uzsv39.e2d48u4i1wgfw532 --discovery-token-ca-cert-hash sha256:73b8ef1a9ed85b489387abd8f1e6bffb7d7a4e0410f2511bb515b3be3d03b727
[preflight] running pre-flight checks
I0802 13:55:41.353849   20648 kernel_validator.go:81] Validating kernel version
I0802 13:55:41.353994   20648 kernel_validator.go:96] Validating kernel config
	[WARNING SystemVerification]: docker version is greater than the most recently validated version. Docker version: 18.03.1-ce. Max validated version: 17.03
[discovery] Trying to connect to API Server "10.30.5.1:6443"
[discovery] Created cluster-info discovery client, requesting info from "https://10.30.5.1:6443"
[discovery] Requesting info from "https://10.30.5.1:6443" again to validate TLS against the pinned public key
[discovery] Cluster info signature and contents are valid and TLS certificate validates against pinned roots, will use API Server "10.30.5.1:6443"
[discovery] Successfully established connection with API Server "10.30.5.1:6443"
[kubelet] Downloading configuration for the kubelet from the "kubelet-config-1.11" ConfigMap in the kube-system namespace
[kubelet] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[preflight] Activating the kubelet service
[tlsbootstrap] Waiting for the kubelet to perform the TLS Bootstrap...
[patchnode] Uploading the CRI Socket information "/var/run/dockershim.sock" to the Node API object "node1" as an annotation

This node has joined the cluster:
* Certificate signing request was sent to master and a response
  was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the master to see this node join the cluster.

[root@master1 ~]# kubectl get nodes
NAME      STATUS    ROLES     AGE       VERSION
master1   Ready     master    37m       v1.11.0
node1     Ready     <none>    15m       v1.11.0
node2     Ready     <none>    33s       v1.11.0
node3     Ready     <none>    30s       v1.11.0

#在master节点停止相关服务
kubectl delete deploy 服务名称
#清理集群其他附属服务
kubectl drain <node name> --delete-local-data --force --ignore-daemonsets
kubectl delete node --all
kubeadm reset
#所有node节点执行如下命令
kubeadm reset

[root@master1 ~]# kubectl apply -f k8s-efk-log/
service/elasticsearch-logging created
serviceaccount/elasticsearch-logging created
clusterrole.rbac.authorization.k8s.io/elasticsearch-logging created
clusterrolebinding.rbac.authorization.k8s.io/elasticsearch-logging created
statefulset.apps/elasticsearch-logging created
configmap/fluentd-es-config-v0.1.4 created
serviceaccount/fluentd-es created
clusterrole.rbac.authorization.k8s.io/fluentd-es created
clusterrolebinding.rbac.authorization.k8s.io/fluentd-es created
daemonset.apps/fluentd-es-v2.2.0 created
deployment.apps/kibana-logging created
service/kibana-logging created
[root@master1 ~]#

sudo vi /etc/systemd/system/kubelet.service.d/10-kubeadm.conf

#为kubelet添加如下参数：
ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_KUBEADM_ARGS $KUBELET_EXTRA_ARGS --authentication-token-webhook --authorization-mode=Webhook --read-only-port=10255

#注意以下三个参数是追加的参数
--authentication-token-webhook --authorization-mode=Webhook  --read-only-port=10255

 kubectl apply -f kube-prometheus/manifests/