Minimizing the time taken for entity_load while using for apachesolr indexing
You could very well form an array of the nodes to be indexed after the query execute and do a single entity_load instead of doing a entity_load for each and every node separately and I hope that will reduce the time and below is the patch that could help you



Index: apachesolr/apachesolr.index.inc

--- apachesolr/apachesolr.index.inc     (revision 9348)
+++ apachesolr/apachesolr.index.inc     (working copy)
@@ -636,9 +636,21 @@
     $query->condition('aie.entity_type', $entity_type);
   }
   $query->range(0, $limit);
-  $records = $query->execute();
-  $status_callbacks = apachesolr_entity_get_callback($entity_type, 'status callback');
+  $records = $query->execute()->fetchAll();
+  foreach ($records as $node_record) {
+    $entities[] = $node_record->entity_id;
+  }
+  $arr_entity = entity_load($entity_type, $entities);
+  foreach ($records as $record) {
+    $record->status = $record->status &amp;&amp; $arr_entity[$record->entity_id]->status;
+    $rows[] = $record;
+  }
+  // Commenting as we have already used the entity_load
+  /*$status_callbacks = apachesolr_entity_get_callback($entity_type, 'status callback');
   foreach ($records as $record) {
     // Check status and status callbacks before sending to the index
     if (is_array($status_callbacks)) {
@@ -651,7 +663,7 @@
       }
     }
     $rows[] = $record;
-  }
+  }*/
   return $rows;
 }


As you have said most of your fields are taxonomy term it is idle that you check the term_load method and also whether the term_data table is properly indexed.