title: Kubernetes mysql statefulset with root password
body: <p>I followed the example in here <a href="https://kubernetes.io/docs/tasks/run-application/run-replicated-stateful-application/" rel="noreferrer">https://kubernetes.io/docs/tasks/run-application/run-replicated-stateful-application/</a> to create a 3 replica mysql statefulset in kubernetes and everything works fine.</p>
  
  <p>However, after I added root password, and modified the deploy yaml to include the password, mysql-1 always fail.</p>
  
  <p>Here is how I add the password:</p>
  
  <ol>
  <li>I create a secret <code>mysql-root-pass</code></li>
  <li>I remove the <code>MYSQL_ALLOW_EMPTY_PASSWORD</code> and replace it with <code>MYSQL_ROOT_PASSWORD</code> read from the secret</li>
  <li>Add <code>-p$MYSQL_ROOT_PASSWORD</code> to everywhere mysql is called in the yaml file.</li>
  <li>Still, mysql-0 is created and runs, but mysql-1 crashed:</li>
  </ol>
  
  <p><code>mysql-1                            0/2       Init:CrashLoopBackOff</code></p>
  
  <p>Anyone has a working example on how to add password?</p>
  
  <h2>Edit: Add yaml file</h2>
  
  <pre><code>apiVersion: apps/v1
  kind: StatefulSet
  metadata:
    name: mysql
    labels:
      app: mysql
  spec:
    selector:
      matchLabels:
        app: mysql
    serviceName: mysql
    replicas: 3
    template:
      metadata:
        labels:
          app: mysql
      spec:
        initContainers:
        - name: init-mysql
          image: mysql:5.7
          command:
          # initContainer is used to setup config according to config-map
          - bash
          - "-c"
          - |
            set -ex
            # Generate mysql server-id from pod ordinal index.
            [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
            ordinal=${BASH_REMATCH[1]}
            echo [mysqld] &gt; /mnt/conf.d/server-id.cnf
            # Add an offset to avoid reserved server-id=0 value.
            echo server-id=$((100 + $ordinal)) &gt;&gt; /mnt/conf.d/server-id.cnf
            # Copy appropriate conf.d files from config-map to emptyDir.
            if [[ $ordinal -eq 0 ]]; then
              cp /mnt/config-map/master.cnf /mnt/conf.d/
            else
              cp /mnt/config-map/slave.cnf /mnt/conf.d/
            fi
          volumeMounts:
          - name: conf
            mountPath: /mnt/conf.d
          - name: config-map
            mountPath: /mnt/config-map
        - name: clone-mysql
          image: gcr.io/google-samples/xtrabackup:1.0
          command:
          - bash
          - "-c"
          - |
            set -ex
            # Skip the clone if data already exists.
            [[ -d /var/lib/mysql/mysql ]] &amp;&amp; exit 0
            # Skip the clone on master (ordinal index 0).
            [[ `hostname` =~ -([0-9]+)$ ]] || exit 1
            ordinal=${BASH_REMATCH[1]}
            [[ $ordinal -eq 0 ]] &amp;&amp; exit 0
            # Clone data from previous peer.
            ncat --recv-only mysql-$(($ordinal-1)).mysql 3307 | xbstream -x -C /var/lib/mysql
            # Prepare the backup.
            xtrabackup --prepare --user=root --password=$MYSQL_ROOT_PASSWORD --target-dir=/var/lib/mysql
          volumeMounts:
          - name: data
            mountPath: /var/lib/mysql
            subPath: mysql
          - name: conf
            mountPath: /etc/mysql/conf.d
        containers:
        - name: mysql
          image: mysql:5.7
          env:
          - name: MYSQL_ROOT_PASSWORD
            valueFrom:
              secretKeyRef:
                name: mysql-root-pass
                key: password
          ports:
          - name: mysql
            containerPort: 3306
          volumeMounts:
          - name: data
            mountPath: /var/lib/mysql
            subPath: mysql
          - name: conf
            mountPath: /etc/mysql/conf.d
          resources:
            requests:
              cpu: 500m
              memory: 1Gi
          livenessProbe:
            exec:
              command:
              - bash
              - "-c"
              - |
                set -ex
                mysqladmin -p$MYSQL_ROOT_PASSWORD ping &amp;&gt; /dev/null
            initialDelaySeconds: 30
            periodSeconds: 10
            timeoutSeconds: 5
          readinessProbe:
            exec:
              # Check we can execute queries over TCP (skip-networking is off).
              command:
              - bash
              - "-c"
              - |
                set -ex
                mysql -h 127.0.0.1 -p$MYSQL_ROOT_PASSWORD -e "SELECT 1" &amp;&gt; /dev/null
            initialDelaySeconds: 5
            periodSeconds: 2
            timeoutSeconds: 1
        - name: xtrabackup
          image: gcr.io/google-samples/xtrabackup:1.0
          ports:
          - name: xtrabackup
            containerPort: 3307
          command:
          - bash
          - "-c"
          - |
            set -ex
            cd /var/lib/mysql
  
            # Determine binlog position of cloned data, if any.
            if [[ -f xtrabackup_slave_info ]]; then
              # XtraBackup already generated a partial "CHANGE MASTER TO" query
              # because we're cloning from an existing slave.
              mv xtrabackup_slave_info change_master_to.sql.in
              # Ignore xtrabackup_binlog_info in this case (it's useless).
              rm -f xtrabackup_binlog_info
            elif [[ -f xtrabackup_binlog_info ]]; then
              # We're cloning directly from master. Parse binlog position.
              [[ `cat xtrabackup_binlog_info` =~ ^(.*?)[[:space:]]+(.*?)$ ]] || exit 1
              rm xtrabackup_binlog_info
              echo "CHANGE MASTER TO MASTER_LOG_FILE='${BASH_REMATCH[1]}',\
                    MASTER_LOG_POS=${BASH_REMATCH[2]}" &gt; change_master_to.sql.in
            fi
  
            # Check if we need to complete a clone by starting replication.
            if [[ -f change_master_to.sql.in ]]; then
              echo "Waiting for mysqld to be ready (accepting connections)"
              until mysql -h 127.0.0.1 -p$MYSQL_ROOT_PASSWORD -e "SELECT 1"; do sleep 1; done
  
              echo "Initializing replication from clone position"
              # In case of container restart, attempt this at-most-once.
              mv change_master_to.sql.in change_master_to.sql.orig
              mysql -h 127.0.0.1 -p$MYSQL_ROOT_PASSWORD &lt;&lt;EOF
            $(&lt;change_master_to.sql.orig),
              MASTER_HOST='mysql-0.mysql',
              MASTER_USER='root',
              MASTER_PASSWORD='',
              MASTER_CONNECT_RETRY=10;
            START SLAVE;
            EOF
            fi
  
            # Start a server to send backups when requested by peers.
            exec ncat --listen --keep-open --send-only --max-conns=1 3307 -c \
              "xtrabackup --backup --slave-info --stream=xbstream --host=127.0.0.1 --user=root"
          volumeMounts:
          - name: data
            mountPath: /var/lib/mysql
            subPath: mysql
          - name: conf
            mountPath: /etc/mysql/conf.d
          resources:
            requests:
              cpu: 100m
              memory: 100Mi
  
        volumes:
        - name: conf
          emptyDir: {}
        - name: config-map
          configMap:
            name: mysql
    volumeClaimTemplates:
    - metadata:
        name: data
      spec:
        accessModes: ["ReadWriteOnce"]
        storageClassName: dynamic-nfs
        resources:
          requests:
            storage: 10Gi
  </code></pre>