anchor.diff:  external_url 'http://gitlab.dew.ms'
   ...
   -
  -
   gitlab-ctl reconfigure
   
   # 浏览器访问并修改root密码
   
  -# 安装 gitlab runner，Helm方式，在k8s0节点上执行
  +# 安装 gitlab runner，Helm方式
   helm repo add gitlab https://charts.gitlab.io
   helm fetch --untar gitlab/gitlab-runner
   cd gitlab-runner
   
  -# 添加账号绑定关系
  -vi templates/role-binding.yaml
  --
  -    - kind: ServiceAccount
  -      name: default
  -      namespace: "{{ .Release.Namespace }}"
  --
  -
  -# 添加PVC，使用DooD方式
  -# 注意添加的位置在 “# Start the runner” 前
  -# DooD方式由于直接使用宿主机的Docker，存在一定的安全风险，但DinD模式下Kubernetes无法很好地处理镜像缓存，导致每次都要全量拉取
  -# 详见 https://docs.gitlab.com/runner/executors/kubernetes.html#using-docker-in-your-builds
  -vi templates/configmap.yaml
  --
  -    cat >>/home/gitlab-runner/.gitlab-runner/config.toml <<EOF
  -            [[runners.kubernetes.volumes.pvc]]
  -              name = "gitlab-runner-cache"
  -              mount_path = "{{ .Values.runners.cache.cachePath }}"
  -            [[runners.kubernetes.volumes.host_path]]
  -              name = "docker-socket"
  -              mount_path = "/var/run/docker.sock"
  -    EOF
  -    # Start the runner
  --
  -
  -# 创建PV
  -cat <<EOF | kubectl apply -f -
  -apiVersion: v1
  -kind: PersistentVolume
  -metadata:
  -  name: pv-gitlab
  -  labels:
  -    pv: pv-gitlab
  -spec:
  -  capacity:
  -    storage: 100Gi
  -  accessModes:
  -    - ReadWriteOnce
  -  persistentVolumeReclaimPolicy: Recycle
  -  nfs:
  -    path: /data/nfs/gitlab
  -    server: nfs.dew.ms
  -EOF
  -
  -# 创建PVC
  -cat <<EOF | kubectl apply -f -
  -kind: PersistentVolumeClaim
  -apiVersion: v1
  -metadata:
  -  name: gitlab-runner-cache
  -  namespace: devops
  -spec:
  -  accessModes:
  -    - ReadWriteOnce
  -  resources:
  -    requests:
  -      storage: 100Gi
  -  selector:
  -    matchLabels:
  -      pv: pv-gitlab
  -EOF
  -
  -# runnerRegistrationToken 需要从gitlab页面上获取
  -# 特别说明的是这里定义了cachePath，使用PV
  -helm install --name dew-gitlab-runner --namespace devops \
  +# 添加Cache secret
  +kubectl create secret generic minio-access -n devops \
  +    --from-literal=accesskey="dew" \
  +    --from-literal=secretkey="Dew123456"
  +
  +
  +# 为每个项目的每个集群环境添加Runner
  +helm install --name <projectName> --namespace devops \
       --set gitlabUrl=http://gitlab.dew.ms/ \
  -    --set runnerRegistrationToken=<...> \
  -    --set concurrent=20 \
  +    --set runnerRegistrationToken=<...> \ # 需要从gitlab页面上获取
       --set rbac.create=true \
       --set rbacWideAccess=true \
  -    --set runners.tags=general \
  -    --set runners.cache.cachePath=/opt/cache \
  -    --set runners.privileged=true \
  -    .
  +    --set runners.tags=<profile> \ # 当前集群环境名称
  +    --set runners.image=dewms/devops:latest \
  +    --set runners.cache.cacheType=s3 \
  +    --set runners.cache.cacheShared=true \
  +    --set runners.cache.s3ServerAddress=minio.dew.ms:9000 \
  +    --set runners.cache.s3BucketName=app-cache \
  +    --set runners.cache.s3CacheInsecure=true \
  +    --set runners.cache.secretName=minio-access \
  +    --set runners.env.MAVEN_OPTS=-Dmaven.repo.local=.m2 \
  +    --set runners.env.dew_devops_profile=test \
  +    --set runners.env.dew_devops_quiet=true \
  +    --set runners.env.dew_devops_docker_host=<...> \
  +    --set runners.env.dew_devops_docker_registry_url=https://harbor.dew.ms/v2 \
  +    --set runners.env.dew_devops_docker_registry_username=<...> \
  +    --set runners.env.dew_devops_docker_registry_password=<...> \
  +    --set runners.env.dew_devops_kube_config=<...>
  +   .
  +
   ----
   .gitlab pod的调度
   可通过helm安装时，设置以下各具体值来指定pod要调度的node。 +
  -具体可参考：<<podAssignment,pod的调度参考模板>> 或 <<prometheus-pod-assignment,prometheus pod的调度>>
  +具体可参考： <<prometheus-pod-assignment,prometheus pod的调度>>
   [source,bash]
   ----
      --set affinity: {}
      --set nodeSelector: {}
      --set tolerations: []
  +----
  +
  +[source,bash]
  +.Gitlab HA配置
  +----
  +TBD
  +----
  +
  +==== harbor（TBD）
  +
  +TBD Harbor独立部署以确保各环境使用同一套
  +
  +TIP: https://github.com/goharbor/harbor-helm
  +
  +[source,bash]
  +----
  +git clone https://github.com/goharbor/harbor-helm
  +cd harbor-helm
  +git checkout 1.0.0
  +
  +# 创建Postgres数据库
  +-
  +CREATE DATABASE  registry;
  +CREATE DATABASE  clair;
  +CREATE DATABASE  notary_server;
  +CREATE DATABASE  notary_signer;
  +-
  +
  +# 初始用户名/密码 admin/Harbor12345
  +
  +# 访问 https://harbor.dew.ms
  +
  +# 获取证书
  +kubectl -n devops get secrets/dew-harbor-harbor-ingress -o jsonpath="{.data.ca\.crt}" | base64 --decode
  +
  +# 以下操作在每台服务上执行
  +
  +mkdir -p /etc/docker/certs.d/harbor.dew.ms
  +cat <<EOF > /etc/docker/certs.d/harbor.dew.ms/ca.crt
  +<上一步获取的证书>
  +EOF
  +
  +systemctl daemon-reload
  +systemctl restart docker
  +
  +# 登录，用户名/密码 admin/Harbor12345
  +docker login harbor.dew.ms -u admin -p Harbor12345
  +
  +# 测试
  +docker pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.1
  +docker tag registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.1 harbor.dew.ms/library/pause:3.1
  +docker push harbor.dew.ms/library/pause:3.1
   ----
   
   ==== nginx Ingress Controller
  
positive.code: systemctl start redis
  ----
  
  
  [[podAssignment]]
  ==== pod的调度
  TIP: 此处简单说明pod的调度配置，方便后文中各组件的安装配置时的参考。 +
       更详细的配置细节请参考以下官方文档： +
       https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
       https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
  
  ===== node亲和性和调度
  .nodeName
  nodeName是最简单的pod节点选择的方法，直接根据节点的名字来选择pod要调度的节点：
  [source,yaml]
  ----
  apiVersion: v1
  kind: Pod
  metadata:
    name: nginx
  spec:
    containers:
    - name: nginx
      image: nginx
    nodeName: devops
  ----
  
  .nodeSelector
  使用此功能需要先给要被调度的node加上标签： +
  `kubectl label nodes <node-name> <label-key>=<label-value>` +
  e.g. +
  [source,bash]
  ----
  kubectl label nodes devops tag=devops
  kubectl label nodes k8s4 key=target-host-name
  kubectl label nodes k8s5 key=value
  ----
  
  查看node的标签： +
  `kubectl get nodes --show-labels`
  
  给pod添加nodeSelector（推荐在controller部分加，eg:Deployment,StatefulSet等）
  [source,yaml]
  ----
  spec:
    containers:
    - name: nginx
      image: nginx
      imagePullPolicy: IfNotPresent
    nodeSelector:
      tag: devops
  ----
  移除标签：
  `kubectl label nodes devops tag-`
  
  
  .Tolerations
  给node添加污点： +
  `kubectl taint nodes devops key=value:NoSchedule`
  给pod添加tolerations： +
  [source,yaml]
  ----
  tolerations:
  - key: "key"
    operator: "Equal"
    value: "value"
    effect: "NoSchedule"
  ----
  以上配置是指：当node满足pod的tolerations的条件 `key=value:NoSchedule` 时，pod才会被调度到该节点上。
  
  移除node的污点： +
  `kubectl taint nodes devops key:NoSchedule-`
  
  .node的亲和性和反亲和性
  e.g.
  [source,bash]
  ----
  kubectl label nodes k8s1 devops=1
  kubectl label nodes k8s2 devops=2
  kubectl label nodes k8s3 devops=3
  ----
  
  [source,yaml]
  ----
  spec:
    affinity:
      nodeAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          nodeSelectorTerms:
          - matchExpressions:
            - key: devops
              operator: In
              values:
              - 1
              - 2
        preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 1
          preference:
            matchExpressions:
            - key: devops
              operator: In
              values:
              - 3
  ----
  以上例子说明，pod将会被调度到打有 `devops=1` 和 `devops=2` 标签的节点上去， `devops=3` 的节点为备选节点。 +
  operator的值可以为：`In, NotIn, Exists, DoesNotExist, Gt, Lt.` +
  此外，可以通过给 `operator` 设值 `NotIn` 或 `DoesNotExist`，来实现node的反亲和性功能。
  
  ===== pod亲和性与反亲和性调度
  .Inter-pod affinity and anti-affinity
  [source,yaml]
  ----
  spec:
    affinity:
      podAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
        - labelSelector:
            matchExpressions:
            - key: security
              operator: In
              values:
              - S1
          topologyKey: failure-domain.beta.kubernetes.io/zone
      podAntiAffinity:
        preferredDuringSchedulingIgnoredDuringExecution:
        - weight: 100
          podAffinityTerm:
            labelSelector:
              matchExpressions:
              - key: security
                operator: In
                values:
                - S2
            topologyKey: kubernetes.io/hostname
  ----
  
  pod的亲和性调度，主要是根据pod及node的标签来进行调度的。 +
  当一个节点上已有至少一个正在运行的，并带有满足该pod的podAffinity条件的标签的pod1、且该节点有 `failure-domain.beta.kubernetes.io/zone` 的标签，则该pod会被调度到pod1的节点上去。 +
  反亲和性与之相反，当某一节点上至少有一个正在运行的、具有满足podAntiAffinity条件的pod2，则该pod不会被调度到pod2的节点上去。
  
  ==== gitlab
  
  TIP: https://docs.gitlab.com/omnibus/README.html#installation-and-configuration-using-omnibus-package
  
negative.code: test3.k8s   Ready   <none>   52s   v1.14.0
  test4.k8s   Ready   <none>   43s   v1.14.0
  test5.k8s   Ready   <none>   34s   v1.14.0
  ----
  
  [source,bash]
  .Master HA配置
  ----
  TBD
  ----
  
  ==== helm
  