CollectionProperty avoid duplicate names
Here is my idea: Overwrite the attribute setting function of the class. The following code has been created mostly be the OP, based on that idea. I added some minor formatting for better readability:

import bpy
import re
from bpy.props import *

unique_names = ['name']

class RenderGroups(bpy.types.PropertyGroup):
    image = StringProperty(
            name="Image", 
            description="The image-texture of this group")

    def __setattr__(self, name, value): 

        def collection_from_element(self):
            # this gets the collection that the element is in
            path = self.path_from_id()
            match = re.match('(.*)\[\d*\]', path)
            parent = self.id_data
            try:
                coll_path = match.group(1)
            except AttributeError:
                raise TypeError("Propery not element in a collection.") 
            else:
                return parent.path_resolve(coll_path)

        def new_val(stem, nbr):
            # simply for formatting
            return '{st}.{nbr:03d}'.format(st=stem, nbr=nbr)

        # =====================================================

        if name not in unique_names:
            # don't want to handle
            self[name] = value
            return
        if value == getattr(self, name):
            # check for assignement of current value
            return

        coll = collection_from_element(self)
        if value not in coll:
            # if value is not in the collection, just assign
            self[name] = value
            return

        # see if value is already in a format like 'name.012'
        match = re.match('(.*)\.(\d{3,})', value)
        if match is None:
            stem, nbr = value, 1
        else:
            stem, nbr = match.groups()

        # check for each value if in collection
        new_value = new_val(stem, nbr)
        while new_value in coll:
            nbr += 1
            new_value = new_val(stem, nbr)
        self[name] = new_value

class MHFCMeshProps(bpy.types.PropertyGroup):
    render_groups = CollectionProperty(
            type=RenderGroups,
            name="Render Groups")

if __name__ == "__main__":
    bpy.utils.register_class(RenderGroups)
    bpy.utils.register_class(MHFCMeshProps)
    bpy.types.Object.testprop = PointerProperty(type=MHFCMeshProps)


Now you can do this in the blender python console:

obj=bpy.context.object
g=obj.testprop .render_groups.add()
g.name, g.whatever="testme", "keepme"
g.name, g['whatever']
('testme', 'keepme')

g=obj.testprop .render_groups.add()
g.name, g.whatever="testme", "keepme"
g.name, g['whatever']
('testme.001', 'keepme')


I think the code example above is now pretty much complete. Although there is still one caveat here. This keeps track of reassigning the current value and always assigns the lowest nbr possible.