setting color_set of armature to CUSTOM via python
import bpy
from mathutils import Color

obj = bpy.context.active_object
bone_group = obj.pose.bone_groups['Group']
bone_group.color_set = "CUSTOM"
bone_group.colors.normal = Color((1.0, 0.0, 0.0))
bone_group.colors.select = Color((0.0, 1.0, 0.0))
bone_group.colors.active = Color((0.0, 0.0, 1.0))