anchor.diff:  EOF
   ----
   
  +==== docker
  +
  +TIP: https://kubernetes.io/docs/setup/cri/#docker
  +
  +[source,bash]
  +----
  +yum install -y yum-utils \
  +  device-mapper-persistent-data \
  +  lvm2
  +
  +yum-config-manager \
  +    --add-repo \
  +    https://download.docker.com/linux/centos/docker-ce.repo
  +
  +yum update -y && yum install -y docker-ce-18.06.2.ce
  +
  +mkdir /etc/docker
  +
  +cat > /etc/docker/daemon.json <<EOF
  +{
  +  "exec-opts": ["native.cgroupdriver=systemd"],
  +  "log-driver": "json-file",
  +  "log-opts": {
  +    "max-size": "100m"
  +  },
  +  "storage-driver": "overlay2",
  +  "storage-opts": [
  +    "overlay2.override_kernel_check=true"
  +  ]
  +}
  +EOF
  +
  +mkdir -p /etc/systemd/system/docker.service.d
  +
  +# 添加代理
  +cat >>/etc/systemd/system/docker.service.d/http-proxy.conf <<EOF
  +[Service]
  +Environment="HTTP_PROXY=http://<代理host>:<代理端口>" "HTTPS_PROXY=http://<代理host>:<代理端口>" "NO_PROXY=localhost,127.0.0.1,dew.ms"
  +EOF
  +
  +systemctl daemon-reload
  +systemctl restart docker
  +systemctl enable docker.service
  +----
  +
  +==== kubernetes
  +
  +TIP: https://kubernetes.io/docs/setup/independent/install-kubeadm/
  +
  +[source,bash]
  +.安装
  +----
  +# 使用阿里云镜像
  +cat <<EOF > /etc/yum.repos.d/kubernetes.repo
  +[kubernetes]
  +name=Kubernetes
  +baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
  +enabled=1
  +gpgcheck=0
  +repo_gpgcheck=0
  +gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
  +        http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
  +EOF
  +
  +setenforce 0
  +sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config
  +
  +cat <<EOF >  /etc/sysctl.d/k8s.conf
  +net.bridge.bridge-nf-call-ip6tables = 1
  +net.bridge.bridge-nf-call-iptables = 1
  +EOF
  +
  +sysctl --system
  +
  +yum install -y kubelet kubeadm kubectl --disableexcludes=kubernetes
  +systemctl enable --now kubelet
  +----
  +
  +TIP: https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/
  +
  +[source,bash]
  +.Master配置
  +----
  +# 安装Git，后续会用到
  +yum install -y git
  +
  +# 初始化Kubernetes，二选一，使用代理方式
  +kubeadm init \
  +    --kubernetes-version v1.13.4 \
  +    --pod-network-cidr=10.244.0.0/16
  +
  +# 初始化Kubernetes，二选一，不使用代理方式，通过image-repository 及 --kubernetes-version 避免被墙
  +kubeadm init \
  +    --image-repository registry.aliyuncs.com/google_containers \
  +    --kubernetes-version v1.13.4 \
  +    --pod-network-cidr=10.244.0.0/16
  +
  +# 记录上述操作输出中的kubeadm join
  +# e.g. kubeadm join 10.200.131.18:6443 --token i3i7qw.2gst6kayu1e8ezlg --discovery-token-ca-cert-hash sha256:cabc90823a8e0bcf6e3bf719abc569a47c186f6cfd0e156ed5a3cd5a8d85fab0
  +
  +mkdir -p $HOME/.kube
  +cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  +chown $(id -u):$(id -g) $HOME/.kube/config
  +
  +# 查看集群状态
  +kubectl get cs
  +
  +# 安装flannel
  +kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/a70459be0084506e4ec919aa1c114638878db11b/Documentation/kube-flannel.yml
  +
  +# 都为Running后表示完成
  +kubectl get pods --all-namespaces
  +----
  +
  +[NOTE]
  +.Master做为Node
  +====
  +默认情况下 master 不会做为 node 节点，可通过此命令强制启用（不推荐）
  +``kubectl taint nodes --all node-role.kubernetes.io/master-``
  +====
  +
  +TIP: https://kubernetes.io/docs/setup/independent/create-cluster-kubeadm/
  +
  +[source,bash]
  +.Node配置
  +----
  +# 执行上一步输出的 kubeadm join ...
  +
  +# 完成后在master上执行情况如下（以1.13.4版本为例）
  +kubectl get no
  +NAME        STATUS     ROLES    AGE   VERSION
  +test1.k8s   Ready   master   11m   v1.13.4
  +test2.k8s   Ready   <none>   70s   v1.13.4
  +test3.k8s   Ready   <none>   52s   v1.13.4
  +test4.k8s   Ready   <none>   43s   v1.13.4
  +test5.k8s   Ready   <none>   34s   v1.13.4
  +----
  +
  +==== helm
  +
  +TIP: https://docs.helm.sh/using_helm/#installing-helm
  +
  +[source,bash]
  +----
  +
  +curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash
  +
  +cat <<EOF | kubectl apply -f -
  +apiVersion: v1
  +kind: ServiceAccount
  +metadata:
  +  name: tiller
  +  namespace: kube-system
  +---
  +apiVersion: rbac.authorization.k8s.io/v1
  +kind: ClusterRoleBinding
  +metadata:
  +  name: tiller
  +roleRef:
  +  apiGroup: rbac.authorization.k8s.io
  +  kind: ClusterRole
  +  name: cluster-admin
  +subjects:
  +  - kind: ServiceAccount
  +    name: tiller
  +    namespace: kube-system
  +EOF
  +
  +helm init --service-account tiller
  +
  +# 不使用代理方式，需要修改镜像
  +kubectl set image deployment/tiller-deploy tiller=registry.cn-hangzhou.aliyuncs.com/google_containers/tiller:v2.12.3 -n kube-system
  +
  +kubectl get pod -n kube-system -l app=helm
  +----
  +
  +==== dns
  +
  +[source,bash]
  +----
  +# 在middleware节点上执行安装
  +yum install -y dnsmasq
  +systemctl enable dnsmasq
  +systemctl start dnsmasq
  +
  +# 编辑各节点，加上middleware节点的IP
  +vi /etc/resolv.conf
  +-
  +nameserver 10.200.10.11
  +-
  +
  +# 编辑Kubernetes的DNS，加上dew.ms的代理
  +kubectl -n kube-system edit cm coredns
  +-
  +data:
  +  Corefile: |
  +    ...
  +    dew.ms:53 {
  +        errors
  +        cache 30
  +        proxy . 10.200.10.11
  +    }
  +-
  +----
  +
  +==== nfs
  +
  +[source,bash]
  +----
  +yum install -y nfs-utils
  +mkdir -p /data/nfs
  +chmod 755 /data/nfs
  +
  +mkdir -p /data/nfs/gitlab
  +
  +vi /etc/exports
  +
  +    /data/nfs/gitlab     *(rw,sync,no_root_squash,no_all_squash)
  +
  +systemctl enable rpcbind
  +systemctl enable nfs-server
  +systemctl start rpcbind
  +systemctl start nfs-server
  +
  +showmount -e localhost
  +----
  +
  +==== postgreSql
  +
  +[source,bash]
  +----
  +wget https://download.postgresql.org/pub/repos/yum/9.6/redhat/rhel-7-x86_64/pgdg-redhat96-9.6-3.noarch.rpm
  +
  +rpm -Uvh pgdg-redhat96-9.6-3.noarch.rpm
  +yum install -y postgresql96-server
  +
  +/usr/pgsql-9.6/bin/postgresql96-setup initdb
  +
  +vi /var/lib/pgsql/9.6/data/postgresql.conf
  +-
  +listen_addresses='*'
  +-
  +
  +vi /var/lib/pgsql/9.6/data/pg_hba.conf
  +-
  +host  all  all 0.0.0.0/0 md5
  +-
  +
  +systemctl enable postgresql-9.6.service
  +systemctl start postgresql-9.6.service
  +
  +su - postgres
  +psql -U postgres
  +-
  +ALTER USER postgres WITH PASSWORD 'Dew!123456';
  +-
  +----
  +
  +==== redis
  +
  +[source,bash]
  +----
  +yum install -y epel-release
  +yum -y install redis
  +vi /etc/redis.conf
  +-
  +# 注释
  +# bind 127.0.0.1
  +# 开启密码
  +requirepass Dew!123456
  +-
  +systemctl start redis
  +----
  +
  +
   [[podAssignment]]
   ==== pod的调度
   TIP: 此处简单说明pod的调度配置，方便后文中各组件的安装配置时的参考。
  
positive.code: # 监控APP
    1.首先需要将项目instrument
      参考文章：https://prometheus.io/docs/instrumenting/clientlibs/
    2.部署项目及创建进行监控的ServiceMonitor。 +
      注意ServiceMonitor的labels要含有Prometheus-operator创建的Prometheus的serviceMonitorSelector的label。 +
      详细文章：https://github.com/coreos/prometheus-operator/blob/master/Documentation/user-guides/getting-started.md#related-resources
  ----
  
  
negative.code:             values:
              - 3
  ----
  以上例子说明，pod将会被调度到打有 `devops=1` 和 `devops=2` 标签的节点上去， `devops=3` 的节点为备选节点。 +
  operator的值可以为：`In, NotIn, Exists, DoesNotExist, Gt, Lt.` +
  此外，可以通过给 `operator` 设值 `NotIn` 或 `DoesNotExist`，来实现node的反亲和性功能。
  
  ===== pod亲和性与反亲和性调度
  .Inter-pod affinity and anti-affinity
  