Custom Views Relationship with Custom Entity
I fixed it by copying some of the code from the entityreference. 
I have to use a different views handler for this relationship  and add other attributes to that relationship

function clinical_trial_views_data_alter(&amp;$data) {

  $data['clinical_trial_research']['clinical_trial_research_sites'] = array(
    'title' =&gt; t('Clinical Trial Research Sites'),
    'help' =&gt; t('Field Reference Appear in Clinical Trial Research Sites.'),
    'relationship' =&gt; array(
      'handler' =&gt; 'views_handler_relationship',
      'label' =&gt; t('Clinical Trial Research Sites'),
      'base' =&gt; 'clinical_trial_research_sites',
      'base field' =&gt; 'study_num',
      'relationship field' =&gt; 'study_num',
    ),
  );
  // The reference field in the node entity is not entity reference , its just a basic text field
  $data['clinical_trial_research_sites']['site_comp_code']['relationship'] = array(
    'handler' =&gt; 'views_handler_relationship_entity_reverse',
    'field_name' =&gt; 'field_site_comp_code',
    'field table' =&gt; 'field_data_field_site_comp_code',
    'field field' =&gt; "field_site_comp_code_value",
    'base' =&gt; 'node',
    'base field' =&gt; 'nid',
    'title' =&gt; t('Referencing entity'),
    'help' =&gt; t('Field Reference Appear in Location Content type.'),
    'title' =&gt; t('Location'),
  );
}