anchor.diff:  
   [source,bash]
   ----
  -# TBD 使用PVC
  -
  -# TBD 安全认证
  -
  -helm install stable/kibana --name dew-kibana --namespace devops \
  +
  +# 在NFS服务器上加上文件路径,并创建需要的目录
  +
  +vi /etc/exports
  +
  +/data/nfs/kibana    *(rw,sync,no_root_squash,no_all_squash)
  +# 重启NFS
  +systemctl restart nfs-server
  +# 查看结果
  +showmount -e localhost
  +
  +使用PVC
  +apps=("kibana")
  +size=10Gi
  +
  +for i in ${apps[@]};do
  +cat <<EOF | kubectl -n devops apply -f -
  +apiVersion: v1
  +kind: PersistentVolumeClaim
  +metadata:
  +  labels:
  +    app: ${i}
  +  name: dew-${i}
  +spec:
  +  accessModes:
  +    - ReadWriteOnce
  +  resources:
  +    requests:
  +      storage: ${size}
  +  selector:
  +    matchLabels:
  +      app: ${i}
  +---
  +apiVersion: v1
  +kind: PersistentVolume
  +metadata:
  +  labels:
  +    app: ${i}
  +  name: dew-${i}
  +spec:
  +  capacity:
  +    storage: ${size}
  +  accessModes:
  +    - ReadWriteOnce
  +  persistentVolumeReclaimPolicy: Recycle
  +  nfs:
  +    path: /data/nfs/${apps}
  +    server: 10.200.131.182
  +EOF
  +done
  +
  +helm install --name dew-kibana  stable/kibana  --namespace devops \
       --set image.tag="6.6.1" \
       --set env."ELASTICSEARCH_URL"="http://dew-elasticsearch-client:9200" \
       --set service.internalPort=5601 \
  -    --set ingress.enabled=true \
  -    --set ingress.hosts={kibana.dew.ms} \
  +    --set ingress.enabled=true,ingress.hosts={kibana.dew.ms} \
  +    --set-string ingress.annotations."kubernetes\.io/ingress\.class"=nginx \
       --set-string ingress.annotations."kubernetes\.io/tls-acme"="true" \
  -    --set ingress.tls[0].hosts={kibana.dew.ms} \
  -    --set ngress.tls[0].secretName=kibana-certs \
  +    --set ingress.tls[0].hosts={kibana.dew.ms},ingress.tls[0].secretName=kibana-certs \
  +    --set dashboardImport.enabled=true \
  +    --set dashboardImport.dashboards."k8s"="https://raw.githubusercontent.com/monotek/kibana-dashboards/master/k8s-fluentd-elasticsearch.json" \
  +    --set serviceAccount.create=true,serviceAccountName=kibana \
  +    --set plugins.enabled=true \
  +    --set persistentVolumeClaim.enabled=true \
  +    --set persistentVolumeClaim.existingClaim=true \
  +    --set securityContext.enabled=true \
  +    --set securityContext.allowPrivilegeEscalation=true \
  +    --set securityContext.runAsUser=0 \
  +    --set securityContext.fsGroup=0
  +
  +    # pod调度相关配置，请根据实际情况设值
  +    --set nodeSelector."tag"="devops" \
  +    --set tolerations[0].key="key" \
  +    --set tolerations[0].operator="Equal" \
  +    --set tolerations[0].value="value" \
  +    --set tolerations[0].effect="NoSchedule" \
  +    --set affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key="key" \
  +    --set affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator=In \
  +    --set affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[0]=target-host-name \
  +
  +    # xpack security相关参数：
  +    --set image.repository=docker.elastic.co/kibana/kibana \
  +    --set env."XPACK_SECURITY_ENABLED"="true" \
  +    --set env."ELASTICSEARCH_USERNAME"="kibana" \
  +    --set env."ELASTICSEARCH_PASSWORD"="dew123456" \
       --set dashboardImport.xpackauth.enabled=true \
  -    --set dashboardImport.xpackauth.username=admin \
  -    --set dashboardImport.xpackauth.password=Dew\!123456 \
  -    --set dashboardImport.dashboards."k8s"="https://raw.githubusercontent.com/monotek/kibana-dashboards/master/k8s-fluentd-elasticsearch.json" \
  -    --set serviceAccount.create=true \
  -    --set serviceAccountName=kibana \
  -    --set plugins.enabled=true \
  -    --set plugins.reset=true \
  -    --set persistentVolumeClaim.enabled=true \
  -    --set persistentVolumeClaim.size=10Gi \
  -    --set securityContext.enabled=true \
  -    --set securityContext.allowPrivilegeEscalation=true
  +    --set dashboardImport.xpackauth.username=kibana\
  +    --set dashboardImport.xpackauth.password=dew123456
  +
   ----
   
   ==== jaeger
  
positive.code: 
  ==== elasticsearch
  
  TIP: https://github.com/helm/charts/tree/master/stable/elasticsearch
  
  [source,bash]
  ----
  # TBD 使用PVC
  
  helm install --name dew-elasticsearch stable/elasticsearch --namespace devops \
      --set image.tag="6.6.1" \
      --set cluster.name=dew-elasticsearch \
      --set client.ingress.enabled=true \
      --set client.ingress.user=elasticsearch \
      --set client.ingress.password=Dew\!123456 \
      --set client.ingress.hosts={es.dew.ms} \
      --set master.exposeHttp=true \
      --set master.persistence.enabled=true \
      --set master.persistence.size=200Gi \
      --set data.exposeHttp=true \
      --set data.persistence.enabled=true \
      --set data.persistence.size=200Gi
  ----
  
  ==== kibana
  
  
negative.code:             path: /
  EOF
  
  ----
  
  .pod的调度
  目前jaeger-operator暂不支持直接设置，请关注该项目的更新情况。
  可以自行给需要调度的pod的deployment添加限制条件。可参考： <<podAssignment>>
  
  
  ==== prometheus-operator 和 grafana
  TIP: https://github.com/helm/charts/tree/master/stable/prometheus-operator
  
    prometheus-operator结构：
      |--- prometheus-operator
      |--- prometheus
      |--- alertmanager
      |--- node-exporter
      |--- kube-state-metrics
      |--- service monitors to scrape internal kubernetes components
      |     |---kube-apiserver
      |     |---kube-scheduler
      |     |---kube-controller-manager
      |     |---etcd
      |     |---kube-dns/coredns
      |
      |--- grafana
  
  ===== prometheus-operator 使用PV
  
  [source,bash]
  ----
  # 在NFS服务器上加上文件路径,并创建需要的目录
  
  vi /etc/exports
  
  /data/nfs/prometheus-operator    *(rw,sync,no_root_squash,no_all_squash)
  
  # 重启NFS
  systemctl restart nfs-server
  # 查看结果
  showmount -e localhost
  
  
  # 创建PV,注意label的对应
  
  app=prometheus-operator
  components=("alertmanager" "prometheus")
  size=100Gi
  
  for i in ${components[@]};do
  cat <<EOF | kubectl -n devops apply -f -
  apiVersion: v1
  kind: PersistentVolume
  metadata:
    labels:
      component: ${i}
    name: dew-${app}-${i}
  spec:
    capacity:
      storage: ${size}
    accessModes:
      - ReadWriteOnce
    persistentVolumeReclaimPolicy: Recycle
    nfs:
      path: /data/nfs/${app}/${i}
      server: 10.200.131.182
  EOF
  done
  
  ----
  
  ===== 创建grafana的PVC和PV
  [source,bash]
  ----
  apps=("grafana")
  size=50Gi
  
  for i in ${apps[@]};do
  cat <<EOF | kubectl -n devops apply -f -
  apiVersion: v1
  kind: PersistentVolumeClaim
  metadata:
    labels:
      app: ${i}
    name: dew-${i}
  spec:
    accessModes:
      - ReadWriteOnce
    resources:
      requests:
        storage: ${size}
    selector:
      matchLabels:
        app: ${i}
  ---
  apiVersion: v1
  kind: PersistentVolume
  metadata:
    labels:
      app: ${i}
    name: dew-${i}
  spec:
    capacity:
      storage: ${size}
    accessModes:
      - ReadWriteOnce
    persistentVolumeReclaimPolicy: Recycle
    nfs:
      path: /data/nfs/prometheus-operator/${i}
      server: 10.200.131.182
  EOF
  done
  
  ----
  
  
  ===== 使用helm 安装
  
  注意安装前先更新chart仓库
  `helm repo update`
  
  [source,yaml]
  ----
  # 若需要对etcd进行监控，则需要先创建secret
  kubectl -n devops create secret generic dew-prometheus-operator-etcd  --from-file=/etc/kubernetes/pki/etcd/ca.crt  --from-file=/etc/kubernetes/pki/etcd/peer.crt  --from-file=/etc/kubernetes/pki/etcd/peer.key
  
  helm install stable/prometheus-operator --name dew-prometheus-operator --namespace devops \
      --set kubelet.serviceMonitor.https=true \
      --set prometheus.ingress.enabled=true \
      --set prometheus.ingress.hosts={prometheus.dew.ms} \
      --set alertmanager.ingress.enabled=true \
      --set alertmanager.ingress.hosts={prometheus.alertmanager.ms} \
      --set prometheusOperator.securityContext.runAsNonRoot=false \
      --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage=100Gi \
      --set alertmanager.alertmanagerSpec.storage.volumeClaimTemplate.spec.resources.requests.storage=100Gi \
      --set alertmanager.alertmanagerSpec.storage.volumeClaimTemplate.spec.selector.matchLabels."component"="alertmanager" \
      --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.selector.matchLabels."component"="prometheus"
      # 对etcd监测相关参数
      --set prometheus.prometheusSpec.secrets[0]=dew-prometheus-operator-etcd
      --set kubeEtcd.serviceMonitor.scheme=https \
      --set kubeEtcd.serviceMonitor.insecureSkipVerify=true \
      --set kubeEtcd.serviceMonitor.caFile="/etc/prometheus/secrets/dew-prometheus-operator-etcd/ca.crt" \
      --set kubeEtcd.serviceMonitor.certFile="/etc/prometheus/secrets/dew-prometheus-operator-etcd/peer.crt" \
      --set kubeEtcd.serviceMonitor.keyFile="/etc/prometheus/secrets/dew-prometheus-operator-etcd/peer.key"
      # 直接使用prometheus-operator的grafana，添加以下设置
      --set grafana.enabled=true \
      --set grafana.adminPassword=Dew123456 \
      --set grafana.defaultDashboardsEnabled=true \
      --set grafana.ingress.enabled=true \
      --set grafana.ingress.hosts={grafana.dew.ms} \
      --set grafana.ingress.tls[0].host={grafana.dew.ms},ingress.tls[0].secretName=dew-grafana \
      --set grafana.sidecar.dashboards.enabled=true \
      --set grafana.sidecar.dashboards.searchNamespace="devops"\
      --set grafana.sidecar.dashboards.label=grafana_dashboard \
      --set grafana.sidecar.datasources.enabled=true \
      --set grafana.sidecar.datasources.searchNamespace="devops" \
      --set grafana.sidecar.datasources.label=grafana_datasource \
      --set grafana.'grafana\.ini'.smtp.enabled="true" \
      --set grafana.'grafana\.ini'.smtp.host="smtp.163.com:25" \
      --set grafana.'grafana\.ini'.smtp.user=syl_test@163.com \
      --set grafana.'grafana\.ini'.smtp.password=test123456 \
      --set grafana.'grafana\.ini'.smtp.from_address="syl_test@163.com" \
      --set grafana.'grafana\.ini'.smtp.skip_verify=true \
      --set grafana.persistence.enabled=true \
      --set grafana.persistence.existingClaim=dew-grafana
      # 可选设置
      --set grafana.'grafana\.ini'.server.root_url="https://grafana.dew.ms"
  
      如不使用代理,更换以下镜像仓库
      --set kube-state-metrics.image.repository=registry.cn-hangzhou.aliyuncs.com/google_containers/kube-state-metrics
  
   TIP: grafana默认用户名：admin,
   查看密码：
   kubectl get secret --namespace devops dew-prometheus-operator-grafana -o jsonpath="{.data.admin-password}" | base64 --decode ; echo
  
   grafana重置密码：进入grafana的容器内部后
   grafana-cli admin reset-admin-password passwordvalue
  
  INFO: 若有pod启动失败,报文件权限拒绝相关问题，很可能和PV的文件目录的权限有关，检查下权限是否一致，设置对应的securityContext进行排查。
  例：
  kubectl edit statefulset prometheus-dew-prometheus-operator-prometheus -n devops
  设置securityContext为以下内容
        securityContext:
          fsGroup: 0
          runAsNonRoot: false
          runAsUser: 0
  
  INFO: 若验证prometheus的target，kube-scheduler、kube-controller处于down状态，是因为它们只能在宿主机上通过127.0.0.1访问，可使用以下操作：
      . 如果使用kubeadm启动的集群，初始化时的config.yml里可以加入如下参数
          controllerManagerExtraArgs:
            address: 0.0.0.0
          schedulerExtraArgs:
            address: 0.0.0.0
      . 已经启动后的使用下面命令更改就会滚动更新
          sed -e "s/- --address=127.0.0.1/- --address=0.0.0.0/" -i /etc/kubernetes/manifests/kube-controller-manager.yaml
          sed -e "s/- --address=127.0.0.1/- --address=0.0.0.0/" -i /etc/kubernetes/manifests/kube-scheduler.yaml
        或者全部替换：
          sed -ri '/--address/s#=.+#=0.0.0.0#' /etc/kubernetes/manifests/kube-*
      参考文章： http://www.servicemesher.com/blog/prometheus-operator-manual/
                https://github.com/coreos/prometheus-operator/blob/master/Documentation/troubleshooting.md
  
  # 监控APP
    1.首先需要将项目instrument
      参考文章：https://prometheus.io/docs/instrumenting/clientlibs/
    2.部署项目及创建进行监控的ServiceMonitor。 +
      注意ServiceMonitor的labels要含有Prometheus-operator创建的Prometheus的serviceMonitorSelector的label。 +
      详细文章：https://github.com/coreos/prometheus-operator/blob/master/Documentation/user-guides/getting-started.md#related-resources
  ----
  
  ===== pod的调度
  pod调度相关配置，以下配置仅供参考
  [source,yaml]
  ----
      --set alertmanager.alertmanagerSpec.nodeSelector."tag"="devops" \
      --set alertmanager.alertmanagerSpec.tolerations[0].key="key" \
      --set alertmanager.alertmanagerSpec.tolerations[0].operator="Equal" \
      --set alertmanager.alertmanagerSpec.tolerations[0].value="value" \
      --set alertmanager.alertmanagerSpec.tolerations[0].effect="NoSchedule"
      # podAntiAffinity的值可以 hard 或 soft
      --set alertmanager.alertmanagerSpec.podAntiAffinity="hard" \
      --set alertmanager.alertmanagerSpec.podAntiAffinityTopologyKey="kubernetes\.io/hostname" \
  
      --set prometheusOperator.nodeSelector."tag"="devops" \
      --set prometheusOperator.tolerations[0].key="key" \
      --set prometheusOperator.tolerations[0].operator="Equal" \
      --set prometheusOperator.tolerations[0].value="value" \
      --set prometheusOperator.tolerations[0].effect="NoSchedule"
      --set prometheusOperator.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key="key" \
      --set prometheusOperator.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator=In \
      --set prometheusOperator.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[0]=target-host-name \
  
      --set prometheus.prometheusSpec.nodeSelector."tag"="devops" \
      --set prometheus.prometheusSpec.tolerations[0].key="key" \
      --set prometheus.prometheusSpec.tolerations[0].operator="Equal" \
      --set prometheus.prometheusSpec.tolerations[0].value="value" \
      --set prometheus.prometheusSpec.tolerations[0].effect="NoSchedule" \
      # podAntiAffinity的值可以 hard 或 soft
      --set prometheus.prometheusSpec.podAntiAffinity=hard \
      --set prometheus.prometheusSpec.podAntiAffinityTopologyKey="kubernetes\.io/hostname" \
  
      --set kube-state-metrics.nodeSelector."tag"="devops" \
      --set kube-state-metrics.tolerations[0].key="key" \
      --set kube-state-metrics.tolerations[0].operator="Equal" \
      --set kube-state-metrics.tolerations[0].value="value" \
      --set kube-state-metrics.tolerations[0].effect="NoSchedule"
  
      --set nodeExporter.nodeSelector."tag"="devops" \
      --set nodeExporter.tolerations[0].key="key" \
      --set nodeExporter.tolerations[0].operator="Equal" \
      --set nodeExporter.tolerations[0].value="value" \
      --set nodeExporter.tolerations[0].effect="NoSchedule" \
      --set nodeExporter.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key="key" \
      --set nodeExporter.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator=In \
      --set nodeExporter.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[0]=target-host-name \
  
      --set grafana.nodeSelector."tag"="devops" \
      --set grafana.tolerations[0].key="key" \
      --set grafana.tolerations[0].operator="Equal" \
      --set grafana.tolerations[0].value="value" \
      --set grafana.tolerations[0].effect="NoSchedule"
      --set grafana.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].key="key" \
      --set grafana.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].operator=In \
      --set grafana.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[0].matchExpressions[0].values[0]=target-host-name \
  ----
  
  ===== 卸载
  [source,yaml]
  ----
  helm del --purge dew-prometheus-operator
  
  kubectl delete crd prometheuses.monitoring.coreos.com prometheusrules.monitoring.coreos.com servicemonitors.monitoring.coreos.com alertmanagers.monitoring.coreos.com
  
  kubectl delete pvc -n devops prometheus-dew-prometheus-operator-prometheus-db-prometheus-dew-prometheus-operator-prometheus-0 alertmanager-dew-prometheus-operator-alertmanager-db-alertmanager-dew-prometheus-operator-alertmanager-0
  
  最后注意删除自己创建的PV、PVC
  ----
  