text: import csv
  from collections import Counter
  from io import TextIOWrapper
  from itertools import chain
  import json
  import ijson
  from django.contrib.auth.models import User
  
  from django.db.utils import IntegrityError
  from django.shortcuts import get_object_or_404, get_list_or_404
  from django_filters.rest_framework import DjangoFilterBackend
  from rest_framework import viewsets, generics, filters, mixins
  from rest_framework.decorators import action
  from rest_framework.pagination import PageNumberPagination
  from rest_framework.permissions import IsAuthenticated, IsAdminUser
  from rest_framework.response import Response
  from rest_framework.views import APIView
  from rest_framework import status
  from rest_framework import exceptions
  from rest_framework import parsers
  from django.http import HttpResponse, HttpResponseRedirect, HttpResponseForbidden
  
  from .models import Project, Label, Document, Setting, NamedEntityAnnotationHistory, Annotation, \
      TriggerExplanation, NaturalLanguageExplanation, RelationExtractionAnnotationHistory, Task, NamedEntityAnnotation, \
      ModelMetaData, SentimentAnalysisAnnotation, RelationExtractionAnnotation
  from .permissions import IsAdminUserAndWriteOnly, IsProjectUser, IsOwnAnnotation
  from .serializers import ProjectSerializer, LabelSerializer, DocumentSerializer, SettingSerializer, \
  NamedEntityAnnotationHistorySerializer, CreateBaseAnnotationSerializer, \
  NamedEntityAnnotationSerializer, SingleUserAnnotatedDocumentSerializer, \
  RelationExtractionAnnotationSerializer, TriggerExplanationSerializer, \
  NaturalLanguageExplanationSerializer, SingleUserAnnotatedDocumentExplanationsSerializer, \
  SentimentAnalysisAnnotationSerializer, RelationExtractionAnnotationHistorySerializer, UserSerializer, TaskSerializer, \
  ModelMetaDataSerializer
  from .utils import SPACY_WRAPPER
  from .constants import TRAINING_UPDATE_FOLDER, TRAINING_KEY, METADATA_KEY, MODEL_META_FILE, EXPLANATION_CHOICES, RELATION_EXTRACTION_VALUE, NAMED_ENTITY_RECOGNITION_VALUE,SENTIMENT_ANALYSIS_VALUE
  import time
  from django.db import transaction
  import pickle
  from django.utils import timezone
  import random
  from os import listdir
  from os.path import isfile, join
  import os
  import requests
  from .constants import EXPLANATION_CHOICES
  from os import path
  import logging
  logger = logging.getLogger(__name__)
  
  class ImportFileError(Exception):
      def __init__(self, message):
          self.message = message
  
  
  class MethodSerializerView(object):
      '''
          Utility class for get different serializer class by method.
          For example:
          method_serializer_classes = {
              ('GET', ): MyModelListViewSerializer,
              ('PUT', 'PATCH'): MyModelCreateUpdateSerializer
          }
      '''
      method_serializer_options = None
  
      def get_serializer_class(self):
          assert self.method_serializer_options is not None, (
                  'Expected view %s should contain method_serializer_options '
                  'to get right serializer class.' %
                  (self.__class__.__name__,)
          )
          for methods, serializer_options in self.method_serializer_options.items():
              if self.request.method in methods:
                  for key in serializer_options:
                      if key in self.request.path:
                          return serializer_options[key]
  
          raise exceptions.MethodNotAllowed(self.request.method)
  
  
  class MethodQuerysetView(object):
      '''
          Utility class for get different queryset class by method.
          For example:
          method_serializer_classes = {
              ('GET', ): Object_One.all(),
              ('PUT', 'PATCH'): Object_Two.all()
          }
      '''
      method_queryset_options = None
  
      def get_queryset(self):
          assert self.method_queryset_options is not None, (
                  'Expected view %s should contain method_queryset_options '
                  'to get right queryset.' %
                  (self.__class__.__name__,)
          )
          for methods, queryset_options in self.method_queryset_options.items():
              if self.request.method in methods:
                  for key in queryset_options:
                      if key in self.request.path:
                          return queryset_options[key]
  
          raise exceptions.MethodNotAllowed(self.request.method)
  
  
  class ProjectViewSet(viewsets.ModelViewSet):
      queryset = Project.objects.all()
      serializer_class = ProjectSerializer
      pagination_class = None
      permission_classes = (IsAuthenticated, IsAdminUserAndWriteOnly)
  
      def get_queryset(self):
          return self.request.user.projects
  
      @action(methods=['get'], detail=True)
      def progress(self, request, pk=None):
          project = self.get_object()
          return Response(project.get_progress())
  
      def create(self, request, *args, **kwargs):
          serializer = self.get_serializer(data=request.data)
          serializer.is_valid(raise_exception=True)
          self.perform_create(serializer)
          headers = self.get_success_headers(serializer.data)
          return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers)
  
      def perform_create(self, serializer):
          serializer.save(creator=self.request.user)
  
      def destory(self, request, pk=None):
          return Response(status=200)
  
  
  class ProjectRetrieveView(generics.RetrieveAPIView):
  	queryset = Project.objects.all()
  	serializer_class = ProjectSerializer
  	lookup_url_kwarg = 'project_id'
  
  
  class LabelList(generics.ListCreateAPIView):
  	queryset = Label.objects.all()
  	serializer_class = LabelSerializer
  	pagination_class = None
  	permission_classes = (IsAuthenticated, IsProjectUser, IsAdminUserAndWriteOnly)
  
  	def get_queryset(self):
  		queryset = self.queryset.filter(project=self.kwargs['project_id'], user_provided=False)
  		return queryset
  
  	def perform_create(self, serializer):
  		project = get_object_or_404(Project, pk=self.kwargs['project_id'])
  		serializer.save(project=project)
  
  
  class ProjectStatsAPI(APIView):
  	pagination_class = None
  	permission_classes = (IsAuthenticated, IsProjectUser, IsAdminUserAndWriteOnly)
  
  	def _get_is_annotated(self, project):
  		response = project.get_progress()
  		if response["annotated_doc_count"] > 0:
  			return True
  		return False
  
  	def get(self, request, *args, **kwargs):
  		p = get_object_or_404(Project, pk=self.kwargs['project_id'])
  
  		if "heading_status" in self.request.path:
  			is_annotated = self._get_is_annotated(p)
  			return Response({
  				"is_annotated": is_annotated,
  				"has_documents": len(p.get_documents()) > 0
  			})
  
  		labels = [label.text for label in p.labels.all()]
  		users = [user.username for user in p.users.all()]
  		docs = Document.objects.get_project_docs_with_labels(self.kwargs['project_id'])
  
  		nested_labels = [[a.label.text for a in doc.annotations.all()] for doc in docs]
  		nested_users = [[a.user.username for a in doc.annotations.all()] for doc in docs]
  
  		label_count = Counter(chain(*nested_labels))
  		label_data = [label_count[name] for name in labels]
  
  		user_count = Counter(chain(*nested_users))
  		user_data = [user_count[name] for name in users]
  
  		response = {'label': {'labels': labels, 'data': label_data},
  		            'user': {'users': users, 'data': user_data}}
  
  		return Response(response)
  
  class LabelDetail(generics.RetrieveUpdateDestroyAPIView):
  	queryset = Label.objects.all()
  	serializer_class = LabelSerializer
  	permission_classes = (IsAuthenticated, IsProjectUser, IsAdminUser)
  
  	def get_queryset(self):
  		queryset = self.queryset.filter(project=self.kwargs['project_id'])
  
  		return queryset
  
  	def get_object(self):
  		queryset = self.filter_queryset(self.get_queryset())
  		obj = get_object_or_404(queryset, pk=self.kwargs['label_id'])
  		self.check_object_permissions(self.request, obj)
  
  		return obj
  
  class LargeResultsSetPagination(PageNumberPagination):
      page_size = 5
      page_size_query_param = 'page_size'
      max_page_size = 10
  
  
  class DocumentList(generics.ListCreateAPIView):
      queryset = Document.objects.all().order_by("id")
      filter_backends = (DjangoFilterBackend, filters.SearchFilter, filters.OrderingFilter)
      search_fields = ('text',)
      permission_classes = (IsAuthenticated, IsProjectUser, IsAdminUserAndWriteOnly)
      serializer_class = SingleUserAnnotatedDocumentSerializer
      pagination_class = LargeResultsSetPagination
  
      def list(self, request, *args, **kwargs):
          queryset = self.get_queryset()
          serializer = self.get_serializer(queryset, many= True)
          annotatedCount = 0
          for q in queryset:
              if q.annotated:
                  annotatedCount += 1
  
          page = self.paginate_queryset(queryset)
          serializer = self.get_serializer(page, many=True)
          return self.get_paginated_response({'annotatedCount': annotatedCount, "results": serializer.data})
  
      def get_queryset(self):
          queryset = self.queryset.filter(project=self.kwargs['project_id'])
          project = get_object_or_404(Project, pk=self.kwargs['project_id'])
          # TODO fix this, you know what
          explanation = project.explanation_type
  
          if explanation != 1:
              self.serializer_class = SingleUserAnnotatedDocumentExplanationsSerializer
  
          # TODO Remove all this logic, annotation server should be
          # able to send back correct doc ids
          if not self.request.query_params.get('active_indices'):
              return queryset
  
          active_indices = self.request.query_params.get('active_indices')
          active_indices = list(map(int, active_indices.split(",")))
  
          queryset = project.get_index_documents(active_indices).order_by("id")
          return queryset
  
  class DocumentDetail(generics.RetrieveUpdateDestroyAPIView):
  	queryset = Document.objects.all()
  	serializer_class = DocumentSerializer
  	permission_classes = (IsAuthenticated, IsProjectUser, IsAdminUser)
  
  	def get_queryset(self):
  		queryset = self.queryset.filter(project=self.kwargs['project_id'])
  		return queryset
  
  	def get_object(self):
  		queryset = self.filter_queryset(self.get_queryset())
  		obj = get_object_or_404(queryset, pk=self.kwargs['doc_id'])
  		self.check_object_permissions(self.request, obj)
  		return obj
  
  	def patch(self, request, *args, **kwargs):
  		return self.partial_update(request, *args, **kwargs)
  
  
  class BaseAnnotationCreateAndDestroyView(generics.GenericAPIView, mixins.CreateModelMixin, mixins.DestroyModelMixin):
  	permission_classes = (IsAuthenticated, IsProjectUser, IsOwnAnnotation)
  
  	queryset = Annotation.objects.all()
  	serializer_class = CreateBaseAnnotationSerializer
  
  	def perform_create(self, serializer):
  		doc = get_object_or_404(Document, pk=self.kwargs['doc_id'])
  		task = doc.project.task
  		serializer.save(document=doc, user=self.request.user, task=task)
  
  	def post(self, request, *args, **kwargs):
  		return self.create(request, *args, **kwargs)
  
  	def delete(self, request, *args, **kwargs):
  		return self.destroy(request, *args, **kwargs)
  
  
  class AnnotationDecoratorCreateView(MethodSerializerView, generics.GenericAPIView, mixins.CreateModelMixin):
  	permission_classes = (IsAuthenticated, IsProjectUser, IsOwnAnnotation)
  	queryset = None
  
  	method_serializer_options = {
  		('POST'): {
  			'ner': NamedEntityAnnotationSerializer,
  			're': RelationExtractionAnnotationSerializer,
  			'trigger': TriggerExplanationSerializer,
  			'nl': NaturalLanguageExplanationSerializer,
  			'sa': SentimentAnalysisAnnotationSerializer
  		}
  	}
  
  	def perform_create(self, serializer):
  		annotation_id = self.kwargs['annotation_id']
  		serializer.save(annotation_id=annotation_id)
  
  	def post(self, request, *args, **kwargs):
  		return self.create(request, *args, **kwargs)
  
  
  class ExplanationDestroyView(MethodQuerysetView, generics.GenericAPIView, mixins.DestroyModelMixin):
  	method_queryset_options = {
  		('DELETE'): {
  			'trigger': TriggerExplanation.objects.all(),
  			'nl': NaturalLanguageExplanation.objects.all()
  		}
  	}
  	serializer_class = None
  	def delete(self, request, *args, **kwargs):
  		return self.destroy(request, *args, **kwargs)
  
  
  class SettingList(generics.GenericAPIView, mixins.CreateModelMixin, mixins.UpdateModelMixin):
  	queryset = Setting.objects.all()
  	serializer_class = SettingSerializer
  	pagination_class = None
  	permission_classes = (IsAuthenticated, IsProjectUser)
  
  	def get_queryset(self):
  		project = get_object_or_404(Project, pk=self.kwargs['project_id'])
  
  		self.queryset = self.queryset.filter(project=project).filter(user__in=[project.creator, self.request.user])
  
  		return self.queryset
  
  	# There is where I provide defaults via project creator's settings
  	def get_object(self):
  		objects = self.filter_queryset(self.get_queryset()).select_related("user").all()
  		obj = None
  		if len(objects) == 0:
  			return {}
  		elif len(objects) > 1:
  			for settings in objects:
  				if settings.user == self.request.user:
  					obj = settings
  		else:
  			obj = objects.get()
  
  		self.check_object_permissions(self.request, obj)
  		return obj
  
  	def perform_create(self, serializer):
  		project = get_object_or_404(Project, pk=self.kwargs['project_id'])
  		serializer.save(project=project, user=self.request.user)
  
  	def get(self, request, *args, **kwargs):
  		return Response(self.serializer_class(self.get_object()).data)
  
  	def put(self, request, *args, **kwargs):
  		project = get_object_or_404(Project, pk=self.kwargs['project_id'])
  		_, created = Setting.objects.get_or_create(project=project, user=self.request.user,
  		                                           defaults=self.request.data)
  		if not created:
  			return self.update(request, *args, **kwargs)
  		return Response(created)
  
  
  class HistoryListView(MethodSerializerView, MethodQuerysetView, generics.ListCreateAPIView):
  	permission_classes = (IsAuthenticated, IsProjectUser)
  
  	method_queryset_options = {
  		('GET'): {
  			'ner': NamedEntityAnnotationHistory.objects.all(),
  			're': RelationExtractionAnnotationHistory.objects.all()
  		}
  	}
  
  	method_serializer_options = {
  		('POST', 'GET'): {
  			'ner': NamedEntityAnnotationHistorySerializer,
  			're': RelationExtractionAnnotationHistorySerializer,
  		}
  	}
  
  	def perform_create(self, serializer):
  		try:
  			project = get_object_or_404(Project, pk=self.kwargs['project_id'])
  			serializer.save(project=project, user=self.request.user)
  		except IntegrityError:
  			print("The word with that label is already exist in history.")
  
  	def get_queryset(self):
  		queryset = MethodQuerysetView.get_queryset(self)
  		return queryset.filter(project=self.kwargs['project_id'])
  
  class HistoryDestroyView(MethodQuerysetView, generics.GenericAPIView, mixins.DestroyModelMixin):
  	method_queryset_options = {
  		('DELETE'): {
  			'ner': NamedEntityAnnotationHistory.objects.all(),
  			're': RelationExtractionAnnotationHistory.objects.all()
  		}
  	}
  	serializer_class = None
  
  	def delete(self, request, *args, **kwargs):
  		return self.destroy(request, *args, **kwargs)
  
  
  class AnnotationHistoryFileUpload(APIView):
      def _process_ner(self, data, all_labels, user, project, max_batch=500):
          history = []
          for entry in data:
              label_info = entry["label"].lower()
              if label_info in all_labels:
                  obj = NamedEntityAnnotationHistory(user=user, project=project, \
                                                     word=entry["word"], label=all_labels[label_info], \
                                                     user_provided=True)
                  history.append(obj)
              if len(history) == max_batch:
                  NamedEntityAnnotationHistory.objects.bulk_create(history, ignore_conflicts=True)
                  history = []
  
          if len(history):
              NamedEntityAnnotationHistory.objects.bulk_create(history, ignore_conflicts=True)
          return True
  
      def _process_re(self, data, all_labels, user, project, max_batch=500):
          history = []
          for entry in data:
              label_info = entry["label"].lower()
              if label_info in all_labels:
                  obj = RelationExtractionAnnotationHistory(user=user, project=project, \
                                                            word_1=entry["word_1"], word_2=entry["word_2"], \
                                                            label=all_labels[label_info], user_provided=True)
                  history.append(obj)
              if len(history) == max_batch:
                  RelationExtractionAnnotationHistory.objects.bulk_create(history, ignore_conflicts=True)
                  history = []
  
          if len(history):
              RelationExtractionAnnotationHistory.objects.bulk_create(history, ignore_conflicts=True)
          return True
  
      def post(self, request, *args, **kwargs):
          action = request.POST['action']
          task = request.POST['task']
          user = self.request.user
          project_id = self.kwargs["project_id"]
          project = get_object_or_404(Project, pk=project_id)
          all_labels = {}
          for label in Label.objects.all().filter(project=project_id):
              label_text = label.text.lower()
              all_labels[label_text] = label
          try:
              data_file = request.FILES['history']
              try:
                  if data_file.multiple_chunks():
                      data = ijson.items(data_file, "data.item")
                  else:
                      data = json.load(data_file)
                      data = data["data"]
                  with transaction.atomic(): # use this always
                      if action == "replace":
                          if task == "2":
                              ids = NamedEntityAnnotationHistory.objects.all() \
                                  .filter(project=project_id) \
                                  .filter(user_provided=True) \
                                  .delete()
                          else:
                              ids = RelationExtractionAnnotationHistory.objects.all() \
                                  .filter(project=project_id) \
                                  .filter(user_provided=True) \
                                  .delete()
                      if task == "2":
                          self._process_ner(data, all_labels, user, project)
                      else:
                          self._process_re(data, all_labels, user, project)
  
                      return Response(status=204)
              except:
                  return Response(status=500)
          except:
              raise ImportFileError("No file was uploaded")
  
  
  class RecommendationList(APIView):
  	pagination_class = None
  	permission_classes = (IsAuthenticated, IsProjectUser)
  
  	def create_re_recommendations(self, doc, entity_pairs):
  		entity_positions = {}
  		doc_id = doc.id
  		for annotation in doc.annotations.all():
  			if annotation.user_provided:
  				ner_position = annotation.named_entity_annotation
  				key = str(ner_position.start_offset) + ":" + str(ner_position.end_offset)
  				entity_positions[key] = 1
  
  		recs = {}
  		for pair in entity_pairs:
  			if pair["sbj_offset"] in entity_positions and pair["obj_offset"] in entity_positions:
  				sbj_offsets = pair["sbj_offset"].split(":")
  				obj_offsets = pair["obj_offset"].split(":")
  				key = pair["sbj_offset"] + "---" + pair["obj_offset"]
  				rec = {
  					"doc_id": doc_id,
  					"key": key,
  					"label": pair["label"]
  				}
  				if key in recs:
  					recs[key].append(rec)
  				else:
  					recs[key] = [rec]
  
  		return recs
  
  	def create_ner_recommendations(self, doc_id, entities):
  		recommendations = {}
  		for entity in entities:
  			rec = {
  				"document": doc_id,
  				"start_offset": entity["start_offset"],
  				"end_offset": entity["end_offset"]
  			}
  
  			if "label" in entity:
  				rec["label"] = entity["label"]
  			else:
  				rec["label"] = None
  
  			key = range(rec["start_offset"], rec["end_offset"])
  			recommendations[key] = rec
  
  		return recommendations
  
  	def _resolve_ner_key(self, key, current_keys):
  		key_as_set = set(key)
  		keys_to_delete = []
  		for cur_key in current_keys:
  			if len(key_as_set.intersection(cur_key)):
  				keys_to_delete.append(cur_key)
  
  		return keys_to_delete
  
  	def resolve_ner_keys(self, keys, current_keys):
  		keys_to_delete = {}
  		for key in keys:
  			delete_list = self._resolve_ner_key(key, current_keys)
  			for d_key in delete_list:
  				keys_to_delete[d_key] = 1
  
  		return keys_to_delete.keys()
  
  	def resolve_recs(self, keys_to_delete, cur_recs, new_recs):
  		for key in keys_to_delete:
  			del cur_recs[key]
  
  		for key in new_recs:
  			cur_recs[key] = new_recs[key]
  
  		return cur_recs
  
  	def get(self, request, *args, **kwargs):
  		# project, document
  		doc_id = self.kwargs['doc_id']
  		project = get_object_or_404(Project, pk=self.kwargs['project_id'])
  		task_id = self.kwargs['task_id']
  		document = project.documents.get(id=doc_id)
  
  		# settings
  		setting_queryset = Setting.objects.all()
  		serializer_class = SettingSerializer
  		setting_queryset = setting_queryset.filter(project=project, user=self.request.user)
  		setting_obj = get_object_or_404(setting_queryset)
  		setting_data = serializer_class(setting_obj).data
  		opt_n = setting_data['noun_chunk']
  		opt_m = setting_data['model_backed_recs']
  		opt_h = setting_data['history']
  		recommendations = {}
  		cur_recommendation_keys = []
  
  		# note there is a hierarchy in place here, the earlier recommendation stragies are overwitten by later ones
  		if opt_n:
  			noun_phrases = SPACY_WRAPPER.get_noun_phrases(document.text)
  			recs = self.create_ner_recommendations(doc_id, noun_phrases)
  			for key in recs:
  				recommendations[key] = recs[key]
  
  			cur_recommendation_keys = recommendations.keys()
  
  		if opt_h:
  			if task_id == 2: # ner
  				history_queryset = NamedEntityAnnotationHistory.objects.all()
  				serializer_class = NamedEntityAnnotationHistorySerializer
  				history_queryset = history_queryset.filter(project=project)
  				if history_queryset.count() > 0:
  					history_obj = get_list_or_404(history_queryset)
  					h_list = serializer_class(history_obj, many=True).data
  					formatted_h_list = []
  					for entry in h_list:
  						offsets = SPACY_WRAPPER.get_char_offsets_of_noun_phrase(entry["word"], document.text)
  						for offset_pair in offsets:
  							h_dict = {
  								"label": entry["label"],
  								"start_offset": offset_pair["start_offset"],
  								"end_offset": offset_pair["end_offset"]
  							}
  							formatted_h_list.append(h_dict)
  
  					recs = self.create_ner_recommendations(doc_id, formatted_h_list)
  
  					keys_to_delete = self.resolve_ner_keys(recs.keys(), cur_recommendation_keys)
  					recommendations = self.resolve_recs(keys_to_delete, recommendations, recs)
  			elif task_id == 3: # RE
  				history_queryset = RelationExtractionAnnotationHistory.objects.all()
  				serializer_class = RelationExtractionAnnotationHistorySerializer
  				history_queryset = history_queryset.filter(project=project)
  				if history_queryset.count() > 0:
  					history_obj = get_list_or_404(history_queryset)
  					h_list = serializer_class(history_obj, many=True).data
  					formatted_h_list = []
  					for entry in h_list:
  						sbj_offsets = SPACY_WRAPPER.get_char_offsets_of_noun_phrase(entry["word_1"], document.text)
  						obj_offsets = SPACY_WRAPPER.get_char_offsets_of_noun_phrase(entry["word_2"], document.text)
  						for i in range(len(sbj_offsets)):
  							sbj_offset = sbj_offsets[i]
  							for j in range(len(obj_offsets)):
  								obj_offset = obj_offsets[j]
  								h_dict = {
  									"label": entry["label"],
  									"sbj_offset": str(sbj_offset["start_offset"]) + ":" + str(sbj_offset["end_offset"]),
  									"obj_offset": str(obj_offset["start_offset"]) + ":" + str(obj_offset["end_offset"])
  								}
  
  								formatted_h_list.append(h_dict)
  
  					recommendations = self.create_re_recommendations(document, formatted_h_list)
  
  		if len(recommendations):
  			labels = Label.objects.all().filter(project=self.kwargs['project_id'])
  			label_dict = {}
  			for label in labels:
  				label_dict[label.id] = LabelSerializer(label).data
  
  			for key in recommendations:
  				rec = recommendations[key]
  
  				if task_id == 3:
  					for r in rec:
  						if r["label"] != None:
  							r["label"] = label_dict[r["label"]]
  				# TODO: Make NER also handle multi recs per annotation
  				else:
  					if rec["label"] != None:
  						rec["label"] = label_dict[rec["label"]]
  
  		return Response({"recommendation": list(recommendations.values())})
  
  
  class ModelAPIView(APIView):
  	def get(self, request):
  		results = []
  		qs = ModelMetaData.objects.all().filter(project__users=self.request.user).prefetch_related('project')
  
  		for model in qs:
  			if model.is_errored:
  				training_status = "Failed"
  			else:
  				training_status = "Finished" if model.is_trained else "Training"
  			matching = "N/A"  # Standard
  			if not model.is_standard_trained:
  				soft_match = model.training_meta_data.get('soft_match', None)
  				if soft_match is not None:
  					matching = "Soft" if soft_match else "Strict"
  			cur_model_json = {
  				'project_id': model.project.id,
  				'project_name': model.project.name,
  				'project_task': model.project.get_task_name(),
  				'model': model.name,
  				'training_status': training_status,
  				"time_spent":  model.time_spent,
  				"time_left":  model.time_left,
  				"stage": model.stage,
  				'best_train_loss': model.best_train_loss,
  				"file_path": model.save_path,
  				'file_size': model.file_size,
  				'is_errored': model.is_errored,
  				'error_message': model.error_message,
  				'matching': matching,
  				'training_meta_data': model.training_meta_data,
  			}
  			results.append(cur_model_json)
  
  		return Response(results)
  
  
  def assemble_file_name(my_dict):
  	return "models/uid"+str(my_dict['uid'])+"-"+my_dict['model_name']+"-"+timezone.now().strftime("%d_%m_%Y_%H.%M.%S") + ".p"
  
  
  def generate_random_models_dicts():
  	for i in range(0,  20):
  		uid = random.randint(0, 4)
  		model_name = "test_model_name_" + str(random.randint(0, 100))
  		model_path = "/a/b/c/d.p"
  
  		my_dict = {'uid': uid, "model_name": model_name, "model_file_path": model_path}
  		with open(assemble_file_name(my_dict), 'wb') as f:
  			pickle.dump(my_dict, f)
  
  
  class TrainModelAPIView(APIView):
  	permission_classes = (IsAuthenticated, IsProjectUser)
  	queryset = Document.objects.all()
  
  	def post(self, request, project_id):
  		model_name = request.data['experiment_name']
  		# call model_settings, params <- if annoying, don't worry
  		# model_name -> experiment_name
  		model_settings = request.data['params']
  		model_settings["experiment_name"] = model_name
  		model_settings["dataset_name"] = request.data['dataset_name']
  		include_documents = request.data['include_documents']
  
  		project = get_object_or_404(Project, pk=project_id)
  
  		try:
  			model_meta: ModelMetaData = ModelMetaData.objects.get(project=project, name=model_name)
  			model_meta.delete()
  		except:
  			pass
  		model_meta = ModelMetaData(project=project, name=model_name, training_meta_data=model_settings)
  		model_meta.save()
  
  		if project.get_task_name() == NAMED_ENTITY_RECOGNITION_VALUE:  # NER
  			train_standard = True if project.explanation_type != 3 else False
  			if train_standard:  # standard
  				req_url = "http://localhost:9000/training/standard/ner/lean-life/"
  			else:
  				req_url = "http://localhost:9000/training/trigger/lean-life/"
  		else:  # RE or SA
  			train_standard = True if project.explanation_type != 2 else False
  			if train_standard:  # standard
  				req_url = "http://localhost:9000/training/standard/lean-life/"
  			else:
  				req_url = "http://localhost:9000/training/next/lean-life/"
  
  		if train_standard:
  			json_object = self.generate_json_for_standard_training_api(project, model_settings, include_documents)
  		else:
  			json_object = self.generate_json_for_model_training_api(project, model_settings, include_documents)
  		response = requests.post(req_url, json=json_object)
  		if response.status_code == 201:
  			return Response(status=201)
  		else:
  			return Response(data=response.text, status=500)
  
  	def generate_json_for_model_training_api(self, project: Project, model_settings, include_documents):
  		data_info = {"label_space": [], "annotated": [], "unlabeled": []}
  
  		model_settings["dataset_size"] = project.documents.filter(annotated=False).count()
  		model_settings["project_id"] = project.pk
  		model_settings["project_type"] = project.get_task_name()
  		labels = get_list_or_404(Label, project=project)
  
  		for label in labels:
  			data_info["label_space"].append({"id": label.id, "text": label.text, "user_provided": label.user_provided})
  
  		if not include_documents:
  			result = {
  				"lean_life_data": data_info,
  				"params": model_settings
  			}
  			return result
  
  		for doc in project.documents.all():
  			annotated_row = {"text": doc.text}
  
  			if not doc.annotated:
  				if project.get_task_name() != RELATION_EXTRACTION_VALUE:
  					data_info['unlabeled'].append(annotated_row)
  					continue
  
  			annotated_row['annotations'] = []
  			annotated_row["explanations"] = []
  
  			for ann in doc.annotations.all():
  				cur_ann = {
  					"label_text": ann.label.text,
  					"id" : ann.id,
  					"user_provided" : ann.user_provided
  				}
  				ext_ann = ann.get_extended_annotation()
  
  				for key_ext_ann in ext_ann.__dict__.keys():
  					if key_ext_ann in ["_state", "annotation_id", "id"]:
  						continue
  					else:
  						cur_ann[key_ext_ann]= ext_ann.__dict__[key_ext_ann]
  				annotated_row['annotations'].append(cur_ann)
  
  				if ann.get_explanations() is not None:
  					for exp in ann.get_explanations():
  						if hasattr(exp, 'trigger_id'):
  							annotated_row['explanations'].append({
  								"annotation_id": exp.annotation.id,
  								"trigger_id": exp.trigger_id,
  								"start_offset": exp.start_offset,
  								"end_offset": exp.end_offset,
  							})
  						else:
  							annotated_row['explanations'].append({
  								"annotation_id": exp.annotation.id,
  								"text": exp.text
  							})
  
  			if not doc.annotated:
  				del annotated_row["explanations"]
  				data_info['unlabeled'].append(annotated_row)
  				continue
  
  			data_info['annotated'].append(annotated_row)
  
  		result = {
  			"lean_life_data": data_info,
  			"params": model_settings
  		}
  		return result
  
  	def generate_json_for_standard_training_api(self, project: Project, model_settings, include_documents):
  		data_info = {"label_space": [], "labeled": []}
  
  		model_settings["project_id"] = project.pk
  		model_settings["project_type"] = project.get_task_name()
  		labels = get_list_or_404(Label, project=project)
  
  		for label in labels:
  			data_info["label_space"].append({"id": label.id, "text": label.text, "user_provided": label.user_provided})
  
  		if not include_documents:
  			result = {
  				"lean_life_data": data_info,
  				"params": model_settings
  			}
  			return result
  
  		for doc in project.documents.all():
  			if not doc.annotated:
  				continue
  			annotated_row = {
  				"text": doc.text,
  				'annotations': []
  			}
  
  			for ann in doc.annotations.all():
  				cur_ann = {
  					"label_text": ann.label.text,
  					"id": ann.id,
  					"user_provided": ann.user_provided
  				}
  				ext_ann = ann.get_extended_annotation()
  
  				for key_ext_ann in ext_ann.__dict__.keys():
  					if key_ext_ann in ["_state", "annotation_id", "id"]:
  						continue
  					else:
  						cur_ann[key_ext_ann] = ext_ann.__dict__[key_ext_ann]
  				annotated_row['annotations'].append(cur_ann)
  
  			data_info['labeled'].append(annotated_row)
  
  		result = {
  			"lean_life_data": data_info,
  			"params": model_settings
  		}
  		return result
  	# def get_is_annotated(self, doc_id):
  	# 	docs = Document.objects.all()
  	# 	for doc in docs:
  	# 		if doc.id == doc_id:
  	# 			return doc.annotated
  
  
  class DownloadModelFile(APIView):
  	permission_classes = ()
  
  	def get(self, request):
  		file_path = request.GET.get("file_path")
  		model_file_json = json.loads(requests.get("http://localhost:9000/download/%s" % file_path).content)
  
  		response = HttpResponse(content_type='application/octet-stream')
  		response['Content-Disposition'] = 'attachment; filename=aaa.p'
  
  		response.write(pickle.dumps(model_file_json))
  		return response
  
  
  class ModelUpdateAPI(generics.UpdateAPIView):
  	permission_classes = ()
  	queryset = ModelMetaData
  	serializer_class = ModelMetaDataSerializer
  	multiple_lookup_fields = ('project_id', 'name')
  
  	def get_object(self):
  		queryset = self.get_queryset()
  		filter = {}
  		for field in self.multiple_lookup_fields:
  			filter[field] = self.request.query_params[field]
  
  		obj = get_object_or_404(queryset, **filter)
  		self.check_object_permissions(self.request, obj)
  		return obj
  
  
  class UserRetrieveAPIView(generics.ListAPIView):
      queryset = User.objects.all()
      serializer_class = UserSerializer
  
  
  class TaskRetrieveAPIView(generics.ListAPIView):
      queryset = Task.objects.all()
      serializer_class = TaskSerializer
  
  
  class ExplanationAPIView(APIView):
      def get(self, request, format=None):
          return Response(EXPLANATION_CHOICES)
  
  class FileUploadAPIView(APIView):
      class ImportFileError(Exception):
          def __init__(self, message):
              self.message = message
  
      def extract_metadata_csv(self, row, text_col, header_without_text):
          vals_without_text = [val for i, val in enumerate(row) if i != text_col]
          return json.dumps(dict(zip(header_without_text, vals_without_text)))
  
      def csv_to_documents(self, project, data_file, text_key='text', max_batch=500):
          decoded_data_file = TextIOWrapper(data_file, encoding='utf-8')
          reader = csv.reader(decoded_data_file)
          maybe_header = next(reader)
          if maybe_header:
              header = False
              docs = []
              if text_key in maybe_header:
                  text_col = maybe_header.index(text_key)
                  header_without_text = [title for i, title in enumerate(maybe_header) if i != text_col]
                  header = True
              elif len(maybe_header) == 1:
                  text_col = 0
                  text = maybe_header[text_col]
                  docs.append(Document(text=text, project=project))
              else:
                  raise FileUploadAPIView.ImportFileError(
                      "CSV file must either have a column named '{}' or only have one column.".format(text_key))
  
              with transaction.atomic():
                  for row in reader:
                      text = row[text_col]
                      metadata = self.extract_metadata_csv(row, text_col, header_without_text) if header else '\{\}'
                      docs.append(Document(text=text, metadata=metadata, project=project))
                      if len(docs) == max_batch:
                          Document.objects.bulk_create(docs)
                          docs = []
                  Document.objects.bulk_create(docs)
          else:
              raise FileUploadAPIView.ImportFileError("CSV file is empty")
  
      def extract_metadata_json(self, entry):
          if "metadata" in entry:
              return json.dumps(entry["metadata"])
          else:
              temp = {}
              ignore_keys = set(["text", "annotations", "doc_id", "user", "ner_annotations"])
              for key in entry:
                  if key not in ignore_keys:
                      temp[key] = entry[key]
              return json.dumps(temp)
  
      def create_annotations_and_explanations(self, annotations, all_annotation_labels, project, user, current_doc):
          for annotation in annotations:
              # check label
              if annotation['label'] not in all_annotation_labels:
                  new_label = Label(text=annotation["label"], project=project)
                  new_label.save()
                  all_annotation_labels[annotation["label"]] = new_label
  
              # create base annotation
              cur_annotation = Annotation(task=project.task, user=user, document=current_doc,
                                          label=all_annotation_labels[annotation["label"]])
              cur_annotation.save()
  
              # Task extended annotation
              if project.task.name == SENTIMENT_ANALYSIS_VALUE:
                  ex_annotation = SentimentAnalysisAnnotation(annotation=cur_annotation)
                  ex_annotation.save()
              if project.task.name == NAMED_ENTITY_RECOGNITION_VALUE:
                  ex_annotation = NamedEntityAnnotation(annotation=cur_annotation,
                                                        start_offset=annotation["start_offset"],
                                                        end_offset=annotation["end_offset"])
                  ex_annotation.save()
              if project.task.name == RELATION_EXTRACTION_VALUE:
                  ex_annotation = RelationExtractionAnnotation(annotation=cur_annotation,
                                                               sbj_start_offset=annotation["sbj_start_offset"],
                                                               sbj_end_offset=annotation["sbj_end_offset"],
                                                               obj_start_offset=annotation["obj_start_offset"],
                                                               obj_end_offset=annotation["obj_end_offset"])
                  ex_annotation.save()
  
              # Explanation
              # No explanation
              if project.explanation_type == 1:
                  pass
              # Natural Language explanation
              if project.explanation_type == 2:
                  for explanation in annotation.get("explanations", []):
                      cur_explanation = NaturalLanguageExplanation(annotation=cur_annotation, text=explanation)
                      cur_explanation.save()
              # Trigger explanation
              if project.explanation_type == 3:
                  for index, explanation_array in enumerate(annotation.get("explanations", [])):
                      for explanation in explanation_array:
                          cur_explanation = TriggerExplanation(annotation=cur_annotation, trigger_id=index,
                                                               start_offset=explanation["start_offset"],
                                                               end_offset=explanation["end_offset"])
                          cur_explanation.save()
  
      def create_plain_docs_from_json(self, project, data_file, user, text_key="text"):
          # load previous
          all_annotation_labels = {}
          for label in Label.objects.filter(project=project, user_provided=False):
              all_annotation_labels[label.text] = label
  
          try:
              if data_file.multiple_chunks():
                  data = ijson.items(data_file, "data.item")
              else:
                  data = json.load(data_file)
                  data = data["data"]
          except:
              raise FileUploadAPIView.ImportFileError("Document dictionaries do not have the 'data' key")
  
          with transaction.atomic():
              for entry in data:
                  try:
                      text = entry[text_key]
                  except:
                      raise FileUploadAPIView.ImportFileError("Document dictionaries do not have the '{}' key".format(text_key))
  
                  metadata = self.extract_metadata_json(entry)
                  annotations = entry.get("annotations", [])
                  cur_doc = Document(text=text, metadata=metadata, project=project, annotated=len(annotations) > 0)
                  cur_doc.save()
  
                  self.create_annotations_and_explanations(
                      annotations,
                      all_annotation_labels,
                      project,
                      user,
                      cur_doc
                  )
  
  
      def create_ner_docs_from_json(self, project, data_file, user, text_key="text"):
          all_ner_labels = {}
          # load previous
          for label in Label.objects.filter(project=project, user_provided=True):
              all_ner_labels[label.text] = label
  
          all_annotation_labels = {}
          for label in Label.objects.filter(project=project, user_provided=False):
              all_annotation_labels[label.text] = label
  
          try:
              if data_file.multiple_chunks():
                  data = ijson.items(data_file, "data.item")
              else:
                  data = json.load(data_file)
                  data = data["data"]
          except:
              raise FileUploadAPIView.ImportFileError("Document dictionaries do not have the 'data' key")
  
          with transaction.atomic():
              for entry in data:
                  try:
                      text = entry[text_key]
                  except:
                      raise FileUploadAPIView.ImportFileError("Document dictionaries do not have the '{}' key".format(text_key))
  
                  metadata = self.extract_metadata_json(entry)
  
                  current_doc = Document(text=text, metadata=metadata, project=project)
                  current_doc.save()
  
                  for annotation in entry["ner_annotations"]:
                      if annotation['label'] not in all_ner_labels:
                          new_label = Label(text=annotation["label"], project=project, user_provided=True)
                          new_label.save()
                          all_ner_labels[annotation["label"]] = new_label
  
                      cur_annotation = Annotation(user_provided=True, task=project.task, user=user, document=current_doc,
                                                  label=all_ner_labels[annotation["label"]])
                      cur_annotation.save()
                      ner_annotation = NamedEntityAnnotation(annotation=cur_annotation,
                                                             start_offset=annotation["start_offset"],
                                                             end_offset=annotation["end_offset"])
                      ner_annotation.save()
  
                  self.create_annotations_and_explanations(
                      entry.get("annotations", []),
                      all_annotation_labels,
                      project,
                      user,
                      current_doc
                  )
  
      def post(self, request, *args, **kwargs):
          project = get_object_or_404(Project, pk=kwargs.get('project_id'))
          import_format = request.POST['format']
          upload_type = request.POST['upload_type']
          user = self.request.user
          try:
              data_file = request.FILES['dataset']
              if import_format == 'csv':
                  self.csv_to_documents(project, data_file)
              elif import_format == 'json':
                  if upload_type == "ner":
                      self.create_ner_docs_from_json(project, data_file, user)
                  else:
                      self.create_plain_docs_from_json(project, data_file, user)
              return Response({"success": "your file has been received"}, status=status.HTTP_202_ACCEPTED, )
          except FileUploadAPIView.ImportFileError as e:
              return Response(exception=e)
          except Exception as e:
              return Response(exception=e)
  
  
  class DownloadAnnotationAPI(APIView):
      # permission_classes = ()
  
      def get(self, request, project_id):
          user_id = self.request.user.id
          project = get_object_or_404(Project, pk=project_id)
          explanation_int = project.explanation_type
          export_format = request.GET.get('downloadFormat')
          task_name=project.get_task_name()
          user_annotations= Annotation.objects.get_annotations_for_export(user_id, task_name, explanation_int)
          # TODO below step will cause exception in NER
          dataset = Document.objects.export_project_user_documents(task_name, project_id, user_id, export_format, user_annotations, explanation_int)
          filename = "_".join(project.name.lower().split())
          try:
              if export_format == 'csv':
                  response = self.get_csv(filename, dataset)
              elif export_format == 'json':
                  response = self.get_json(filename, dataset)
              return response
          except Exception as e:
              logger.exception(e)
              return Response(status=404, data=e)
          return Response(status=200, data={"ok"})
  
      def get_csv(self, filename, dataset):
          response = HttpResponse(content_type='text/csv')
          response['Content-Disposition'] = 'attachment; filename="{}.csv"'.format(filename)
          writer = csv.writer(response)
          writer.writerows(dataset)
          return response
  
      def get_json(self, filename, dataset):
          response = HttpResponse(content_type='text/json')
          response['Content-Disposition'] = 'attachment; filename="{}.json"'.format(filename)
          response.write(json.dumps(dataset, ensure_ascii=False, indent=1))
          return response