'Ensure RPC retry on broken pipe error.
When the cluster closes the TCP socket, try reconnecting
and retrying the command before returing error for the operation.'

def test_rpc_client_recovery_on_broken_pipe(self):

    mock_socket = self.mock_object(socket, 'socket')
    mock_make_call = self.mock_object(coho.Client, '_make_call')
    socket_error = socket.error('[Errno    32]    Broken    pipe')
    socket_error.errno = errno.EPIPE
    mock_make_call.side_effect = socket_error
    rpc_client = coho.CohoRPCClient(ADDR, RPC_PORT)
    with self.assertRaisesRegex(exception.CohoException, 'Failed    to    establish.*'):
        rpc_client.create_snapshot('src', 'dest', 0)
    self.assertEqual(coho.COHO_MAX_RETRIES, mock_make_call.call_count)
    self.assertEqual((coho.COHO_MAX_RETRIES + 1), mock_socket.call_count)
    socket_error.errno = errno.EINVAL
    mock_make_call.side_effect = socket_error
    with self.assertRaisesRegex(exception.CohoException, 'Unable    to    send    request.*'):
        rpc_client.delete_snapshot('src')
    self.assertEqual((coho.COHO_MAX_RETRIES + 1), mock_make_call.call_count)
    self.assertEqual((coho.COHO_MAX_RETRIES + 1), mock_socket.call_count)
