'Create a DistributedMixtureOfExperts.
If `secondary_gating_hp` is `None`, then this is a flat MoE with
`primary_gating_hp.num_experts` experts. Otherwise, this is a hierarchical
MoE with `primary_gating_hp.num_experts` groups of
`secondary_gating_hp.num_experts` experts.
The assignemnt of experts (or groups of experts) to devices is by
round-robin.   So to make equal use of all the devices, one should set
`primary_gating_hp.num_experts` to the number of devices or a multiple
thereof.
Args:
primary_gating_hp: hyperparameters for the primary gating network.
e.g. NoisyTopKGatingParams().
secondary_gating_hp: hyperparameters for the secondary gating network.
e.g. NoisyTopKGatingParams().  None indicates a flat MoE.
expert_hp: hyperparameters for the expert networks.
e.g. FeedForwardExpertParams()
input_size: an integer.
output_size: an integer.
expert_devices: a list of device strings.  The devices to be used for
the experts.
name: a string.'

def __init__(self, primary_gating_hp, secondary_gating_hp, expert_hp, input_size, output_size, expert_devices, name):

    self._name = name
    _SetInputOutputSizes(primary_gating_hp, input_size, None)
    _SetInputOutputSizes(expert_hp, input_size, output_size)
    self._is_hierarchical = (secondary_gating_hp is not None)
    self._primary_gating_hp = primary_gating_hp
    self._primary_gating = primary_gating_hp.gating_class(primary_gating_hp, (name + '_primary_gating'))
    n1 = self._primary_gating_hp.num_experts
    expert_devices = [expert_devices[(i % len(expert_devices))] for i in xrange(n1)]
    self._expert_devices = expert_devices
    self._all_vars = []
    self._all_vars.extend(self._primary_gating.vars)
    if self._is_hierarchical:
        self._secondary_moe = []
        for i in xrange(n1):
            with tf.device(expert_devices[i]):
                secondary_moe = LocalMixtureOfExperts(secondary_gating_hp, expert_hp, input_size, output_size, ('%s_secondary_%d' % (name, i)))
                self._secondary_moe.append(secondary_moe)
                self._all_vars.extend(secondary_moe.vars)
    else:
        self._experts = []
        for i in xrange(n1):
            with tf.device(expert_devices[i]):
                expert = expert_hp.expert_class(expert_hp, (name + ('_%d' % i)))
                self._experts.append(expert)
                self._all_vars.extend(expert.vars)
