from flair.data import Sentence

"""
Explanation: Python Flair Basics
(C) 2018-2020 by Damir Cavar
Version: 0.3, February 2020
Download: This and various other Jupyter notebooks are available from my GitHub repo.
License: Creative Commons Attribution-ShareAlike 4.0 International License (CA BY-SA 4.0)
This material was used in my Advanced Topics in AI class, introduction to Deep Learning environments in Spring 2019 at Indiana University.
Two types of central objects:

Sentence
Token

A Sentence holds a textual sentence and is essentially a list of Token objects.
For creating a Sentence object we first import the Sentence class from the flair.data module:
End of explanation
"""


sentence = Sentence('The grass is green .')
print(sentence)

"""
Explanation: We can now define a sentence:
End of explanation
"""


print(sentence.get_token(4))
print(sentence[3])

"""
Explanation: We can access the tokens of a sentence via their token id or with their index:
End of explanation
"""


for token in sentence:
    print(token)

"""
Explanation: We can also iterate over all tokens in a sentence:
End of explanation
"""


sentence = Sentence('The grass is green.', use_tokenizer=True)
print(sentence)

"""
Explanation: Tokenization
There is a simple tokenizer included in the library using the lightweight segtok library to tokenize your text for such a Sentence definition. In the Sentence constructor use the flag use_tokenize to tokenize the input string before instantiating a Sentence object:
End of explanation
"""


sentence[3].add_tag('ner', 'color')
print(sentence.to_tagged_string())

"""
Explanation: Tags on Tokens
A Token has fields for linguistic annotation:

lemma
part-of-speech tag
named entity tag

We can add a tag by specifying the tag type and the tag value.
We will adding an NER tag of type 'color' to the word 'green'. This means that we've tagged this word as an entity of type color:
End of explanation
"""


from flair.data import Label

tag: Label = sentence[3].get_tag('ner')
print(f'"{sentence[3]}" is tagged as "{tag.value}" with confidence score "{tag.score}"')

"""
Explanation: Each tag is of class Label. An associated score indicates confidence:
End of explanation
"""


sentence = Sentence('France is the current world cup winner.')
sentence.add_label('sports')
sentence.add_labels(['sports', 'world cup'])
sentence = Sentence('France is the current world cup winner.', labels=['sports', 'world cup'])

"""
Explanation: The manually added color tag has a score of 1.0. A tag predicted by a sequence labeler will have a score value that indicates the classifier confidence.
A Sentence can have one or multiple labels that can for example be used in text classification tasks. For instance, the example below shows how we add the label 'sports' to a sentence, thereby labeling it as belonging to the sports category.
End of explanation
"""


sentence = Sentence('France is the current world cup winner.', labels=['sports', 'world cup'])

print(sentence)
for label in sentence.labels:
    print(label)

"""
Explanation: Labels are also of the Label class. So, you can print a sentence's labels like this:
End of explanation
"""


from flair.models import SequenceTagger

tagger = SequenceTagger.load('ner')

"""
Explanation: Tagging Text
Using Pre-Trained Sequence Tagging Models
Flair has numerous pre-trained models. For the named entity recognition (NER) task there is a model that was trained on the English CoNLL-03 task and can recognize 4 different entity types. Import it using:
End of explanation
"""


sentence = Sentence('Washington went to New York .')

tagger.predict(sentence)

print(sentence.to_tagged_string())

"""
Explanation: We use the predict() method of the tagger on a sentence to add predicted tags to the tokens in the sentence:
End of explanation
"""


for entity in sentence.get_spans('ner'):
    print(entity)

"""
Explanation: Getting annotated spans for multi-word expressions can be achieved using the following command:
End of explanation
"""


print(sentence.to_dict(tag_type='ner'))

print(sentence.to_dict())

"""
Explanation: Which indicates that "George Washington" is a person (PER) and "Washington" is a location (LOC). Each such Span has a text, a tag value, its position in the sentence and "score" that indicates how confident the tagger is that the prediction is correct. You can also get additional information, such as the position offsets of each entity in the sentence by calling:
End of explanation
"""


tagger = SequenceTagger.load('de-ner')

sentence = Sentence('George Washington ging nach Washington .')

tagger.predict(sentence)

print(sentence.to_tagged_string())

"""
Explanation: Flair contains various sequence tagger models. You choose which pre-trained model you load by passing the appropriate string to the load() method of the SequenceTagger class. Currently, the following pre-trained models are provided:
As indicated in the list above, we also provide pre-trained models for languages other than English. Currently, we support German, French, and Dutch other languages are forthcoming. To tag a German sentence, just load the appropriate model:
End of explanation
"""


tagger = SequenceTagger.load('pos-multi')

sentence = Sentence('George Washington lebte in Washington . Dort kaufte er a horse .')

tagger.predict(sentence)

print(sentence.to_tagged_string())

"""
Explanation: Flair offers access to multi-lingual models for multi-lingual text.
End of explanation
"""


tagger = SequenceTagger.load('frame')

sentence_1 = Sentence('George returned to Berlin to return his hat .')
sentence_2 = Sentence('He had a look at different hats .')

tagger.predict(sentence_1)
tagger.predict(sentence_2)

print(sentence_1.to_tagged_string())
print(sentence_2.to_tagged_string())

"""
Explanation: Semantic Frames
For English, Flair provides a pre-trained model that detects semantic frames in text, trained using Propbank 3.0 frames. This provides some sort of word sense disambiguation for frame evoking words.
End of explanation
"""


text = "This is a sentence. John read a book. This is another sentence. I love Berlin."

from segtok.segmenter import split_single

sentences = [Sentence(sent, use_tokenizer=True) for sent in split_single(text)]

tagger: SequenceTagger = SequenceTagger.load('ner')
tagger.predict(sentences)
for i in sentences:
    print(i.to_tagged_string())

"""
Explanation: The frame detector makes a distinction in sentence 1 between two different meanings of the word 'return'. 'return.01' means returning to a location, while 'return.02' means giving something back.
Similarly, in sentence 2 the frame detector finds a light verb construction in which 'have' is the light verb and 'look' is a frame evoking word.
Sentence Tagging
To tag an entire text corpus, one needs to split the corpus into sentences and pass a list of Sentence objects to the .predict() method.
End of explanation
"""


from flair.models import TextClassifier

classifier = TextClassifier.load('en-sentiment')

"""
Explanation: Using the mini_batch_size parameter of the .predict() method, you can set the size of mini batches passed to the tagger. Depending on your resources, you might want to play around with this parameter to optimize speed.
Pre-Trained Text Classification Models
Flair provides a pre-trained model for detecting positive or negative comments. It was trained on the IMDB dataset and it can recognize positive and negative sentiment in English text. The IMDB data set can be downloaded from the linked site.
End of explanation
"""


sentence = Sentence('This film hurts. It is so bad that I am confused.')

classifier.predict(sentence)

print(sentence.labels)

sentence = Sentence('This film is fantastic. I love it.')

classifier.predict(sentence)

print(sentence.labels)

"""
Explanation: We call the predict() method of the classifier on a sentence. This will add the predicted label to the sentence:
End of explanation
"""


from flair.embeddings import WordEmbeddings

glove_embedding = WordEmbeddings('glove')

"""
Explanation: Flair has a pre-trained German and English model.
Using Word Embeddings
Flair provides a set of classes with which we can embed the words in sentences in various ways.
All word embedding classes inherit from the TokenEmbeddings class and implement the embed() method which we need to call to embed our text. This means that for most users of Flair, the complexity of different embeddings remains hidden behind this interface. Simply instantiate the embedding class we require and call embed() to embed our text.
All embeddings produced with Flair's methods are Pytorch vectors, so they can be immediately used for training and fine-tuning.
Classic word embeddings are static and word-level, meaning that each distinct word gets exactly one pre-computed embedding. Most embeddings fall under this class, including the popular GloVe or Komninos embeddings.
We instantiate the WordEmbeddings class and pass a string identifier of the embedding we wish to load. If we want to use GloVe embeddings, we pass the string 'glove' to the constructor:
End of explanation
"""


sentence = Sentence('The grass is green .')

glove_embedding.embed(sentence)

for token in sentence:
    print(token)
    print(token.embedding)

"""
Explanation: We create an example sentence and call the embedding's embed() method. We can also pass a list of sentences to this method since some embedding types make use of batching to increase speed.
End of explanation
"""


german_embedding = WordEmbeddings('de')

"""
Explanation: GloVe embeddings are Pytorch vectors of dimensionality 100.
We choose which pre-trained embeddings we want to load by passing the appropriate id string to the constructor of the WordEmbeddings class. We would use the two-letter language code to init an embedding, so 'en' for English and 'de' for German and so on. By default, this will initialize FastText embeddings trained over Wikipedia. We can also always use FastText embeddings over Web crawls, by instantiating with '-crawl'. The 'de-crawl' option would use embeddings trained over German web crawls.
For English, Flair provides a few more options. We can choose between instantiating 'en-glove', 'en-extvec' and so on.
If we want to load German FastText embeddings, instantiate as follows:
End of explanation
"""


german_embedding = WordEmbeddings('de-crawl')

"""
Explanation: If we want to load German FastText embeddings trained over crawls, we instantiate as follows:
End of explanation
"""


custom_embedding = WordEmbeddings('path/to/your/custom/embeddings.gensim')

"""
Explanation: If the models are not locally available, Flair will automatically download them and install them into the local user cache.
It is recommended to use the FastText embeddings, or GloVe if we want a smaller model.
If we want to use any other embeddings (not listed in the list above), we can load those by calling:
End of explanation
"""


import gensim

word_vectors = gensim.models.KeyedVectors.load_word2vec_format('/path/to/fasttext/embeddings.txt', binary=False)
word_vectors.save('/path/to/converted')

"""
Explanation: If we want to load custom embeddings, we need to make sure that the custom embeddings are correctly formatted to gensim.
We can, for example, convert FastText embeddings to gensim using the following code snippet:
End of explanation
"""


from flair.embeddings import CharacterEmbeddings

embedding = CharacterEmbeddings()

sentence = Sentence('The grass is green .')

embedding.embed(sentence)

"""
Explanation: Character Embeddings
Some embeddings - such as character-features - are not pre-trained but rather trained on the downstream task. Normally this requires an implementation of a hierarchical embedding architecture.
With Flair, we don't need to worry about such things. Just choose the appropriate embedding class and the character features will then automatically train during downstream task training.
End of explanation
"""


from flair.embeddings import BytePairEmbeddings

embedding = BytePairEmbeddings('en')

sentence = Sentence('The grass is green .')

embedding.embed(sentence)

"""
Explanation: Sub-Word Embeddings
Flair now also includes the byte pair embeddings calulated by @bheinzerling that segment words into subsequences. This can dramatically reduce the model size vis-a-vis using normal word embeddings at nearly the same accuracy. So, if we want to train small models try out the new BytePairEmbeddings class.
We initialize with a language code (275 languages supported), a number of 'syllables', and a number of dimensions (one of 50, 100, 200 or 300). The following initializes and uses byte pair embeddings for English:
End of explanation
"""


from flair.embeddings import WordEmbeddings, CharacterEmbeddings

glove_embedding = WordEmbeddings('glove')

character_embeddings = CharacterEmbeddings()

"""
Explanation: Sub-word embeddings are interesting, since 
BPEmb is a collection of pre-trained subword embeddings in 275 languages, based on Byte-Pair Encoding (BPE) and trained on Wikipedia. Its intended use is as input for neural models in natural language processing.

subwords allow guessing the meaning of unknown / out-of-vocabulary words. E.g., the suffix -shire in Melfordshire indicates a location.
Byte-Pair Encoding gives a subword segmentation that is often good enough, without requiring tokenization or morphological analysis. In this case the BPE segmentation might be something like melf ord shire.
Pre-trained byte-pair embeddings work surprisingly well, while requiring no tokenization and being much smaller than alternatives: an 11 MB BPEmb English model matches the results of the 6 GB FastText model in our evaluation.

If you are using word embeddings like word2vec or GloVe, you have probably encountered out-of-vocabulary words, i.e., words for which no embedding exists. A makeshift solution is to replace such words with an UNK token and train a generic embedding representing such unknown words.
Subword approaches try to solve the unknown word problem differently, by assuming that you can reconstruct a word's meaning from its parts. For example, the suffix -shire lets you guess that Melfordshire is probably a location, or the suffix -osis that Myxomatosis might be a sickness.
There are many ways of splitting a word into subwords. A simple method is to split into characters and then learn to transform this character sequence into a vector representation by feeding it to a convolutional neural network (CNN) or a recurrent neural network (RNN), usually a long-short term memory (LSTM). This vector representation can then be used like a word embedding.
Another, more linguistically motivated way is a morphological analysis, but this requires tools and training data which might not be available for your language and domain of interest.
Enter Byte-Pair Encoding (BPE) [Sennrich et al, 2016], an unsupervised subword segmentation method. BPE starts with a sequence of symbols, for example characters, and iteratively merges the most frequent symbol pair into a new symbol.
For example, applying BPE to English might first merge the characters h and e into a new symbol he, then t and h into th, then t and he into the, and so on.
Learning these merge operations from a large corpus (e.g. all Wikipedia articles in a given language) often yields reasonable subword segementations. For example, a BPE model trained on English Wikipedia splits melfordshire into mel, ford, and shire.
Stacked Embeddings
Stacked embeddings are one of the most important concepts of Flair. We can use them to combine different embeddings together, for instance if we want to use both traditional embeddings together with contextual string embeddings (see below). Stacked embeddings allow us to mix and match. We find that a combination of embeddings often gives best results.
All we need to do is use the StackedEmbeddings class and instantiate it by passing a list of embeddings that we wish to combine. For instance, lets combine classic GloVe embeddings with character embeddings. This is effectively the architecture proposed in (Lample et al., 2016).
End of explanation
"""


from flair.embeddings import StackedEmbeddings

stacked_embeddings = StackedEmbeddings(
    embeddings=[glove_embedding, character_embeddings])

"""
Explanation: Now instantiate the StackedEmbeddings class and pass it a list containing these two embeddings.
End of explanation
"""


sentence = Sentence('The grass is green .')

stacked_embeddings.embed(sentence)

for token in sentence:
    print(token)
    print(token.embedding)

"""
Explanation: We use this embedding like all the other embeddings, i.e. call the embed() method over our sentences.
End of explanation
"""


from flair.embeddings import FlairEmbeddings

flair_embedding_forward = FlairEmbeddings('news-forward')

sentence = Sentence('The grass is green .')

flair_embedding_forward.embed(sentence)

"""
Explanation: Words are now embedded using a concatenation of two different embeddings. This means that the resulting embedding vector is still a single PyTorch vector.
Other Embeddings: BERT, ELMo, Flair
Next to standard WordEmbeddings and CharacterEmbeddings, Flair also provides classes for BERT, ELMo and Flair embeddings. These embeddings enable us to train truly state-of-the-art NLP models.
All word embedding classes inherit from the TokenEmbeddings class and implement the embed() method which we need to call to embed our text. This means that for most users of Flair, the complexity of different embeddings remains hidden behind this interface. We instantiate the embedding class we require and call embed() to embed our text.
All embeddings produced with Flair's methods are Pytorch vectors, so they can be immediately used for training and fine-tuning.
Flair Embeddings
Contextual string embeddings are powerful embeddings that capture latent syntactic-semantic information that goes beyond standard word embeddings. Key differences are: (1) they are trained without any explicit notion of words and thus fundamentally model words as sequences of characters. And (2) they are contextualized by their surrounding text, meaning that the same word will have different embeddings depending on its contextual use.
Recent advances in language modeling using recurrent neural networks have made it viable to model language as distributions over characters. By learning to predict the next character on the basis of previous characters, such models have been shown to automatically internalize linguistic concepts such as words, sentences, subclauses and even sentiment. In Flair the internal states of a trained character language model is leveraged to produce a novel type of word embedding which the authors refer to as contextual string embeddings. The proposed embeddings have the distinct properties that they (a) are trained without any explicit notion of words and thus fundamentally model words as sequences of characters, and (b) are contextualized by their surrounding text, meaning that the same word will have different embeddings depending on its contextual use. The authors conduct a comparative evaluation against previous embeddings and find that their embeddings are highly useful for downstream tasks: across four classic sequence labeling tasks they consistently outperform the previous state-of-the-art. In particular, they significantly outperform previous work on English and German named entity recognition (NER), allowing them to report new state-of-the-art F1-scores on the CONLL03 shared task.
With Flair, we can use these embeddings simply by instantiating the appropriate embedding class, same as standard word embeddings:
End of explanation
"""


from flair.embeddings import WordEmbeddings, FlairEmbeddings, StackedEmbeddings

stacked_embeddings = StackedEmbeddings([
                                        WordEmbeddings('glove'), 
                                        FlairEmbeddings('news-forward'), 
                                        FlairEmbeddings('news-backward'),
                                       ])

"""
Explanation: We can choose which embeddings we load by passing the appropriate string to the constructor of the FlairEmbeddings class. Currently, there are numerous contextual string embeddings provided in models (more coming). See list.
The recommendation is to combine both forward and backward Flair embeddings. Depending on the task, it is also recommended to add standard word embeddings into the mix. So, the recommendation is to use StackedEmbedding for most English tasks:
End of explanation
"""


from flair.embeddings import BertEmbeddings

embedding = BertEmbeddings()

sentence = Sentence('The grass is green .')

embedding.embed(sentence)

for i in sentence:
    print(i, i.embedding)

"""
Explanation: We would use this embedding like all the other embeddings, i.e. call the embed() method over our sentences.
BERT Embeddings
BERT embeddings were developed by Devlin et al. (2018) and are a different kind of powerful word embedding based on a bidirectional transformer architecture. Flair is using the implementation of huggingface. The embeddings are wrapped into our simple embedding interface, so that they can be used like any other embedding.
End of explanation
"""


from flair.embeddings import ELMoEmbeddings

embedding = ELMoEmbeddings(model="small")

sentence = Sentence('The grass is green .')

embedding.embed(sentence)

"""
Explanation: We can load any of the pre-trained BERT models by providing the model string during initialization:

'bert-base-uncased': English; 12-layer, 768-hidden, 12-heads, 110M parameters
'bert-large-uncased': English; 24-layer, 1024-hidden, 16-heads, 340M parameters
'bert-base-cased': English; 12-layer, 768-hidden, 12-heads , 110M parameters
'bert-large-cased': English; 24-layer, 1024-hidden, 16-heads, 340M parameters
'bert-base-multilingual-cased': 104 languages; 12-layer, 768-hidden, 12-heads, 110M parameters
'bert-base-chinese': Chinese Simplified and Traditional; 12-layer, 768-hidden, 12-heads, 110M parameters

ELMo Embeddings
ELMo embeddings were presented by Peters et al. in 2018. They are using a bidirectional recurrent neural network to predict the next word in a text. Flair is using the implementation of AllenNLP. As this implementation comes with a lot of sub-dependencies, which Flair authors don't want to include in Flair, you need to first install the library via pip install allennlp before we can use it in Flair. Using the embeddings is as simple as using any other embedding type. To specify the correct model, pick one of small, medium, portugese, original:
End of explanation
"""


from flair.embeddings import FlairEmbeddings, BertEmbeddings

flair_forward_embedding = FlairEmbeddings('multi-forward')
flair_backward_embedding = FlairEmbeddings('multi-backward')

bert_embedding = BertEmbeddings('bert-base-multilingual-cased')

"""
Explanation: AllenNLP provides the following pre-trained models. To use any of the following models inside Flair simple specify the embedding id when initializing the ELMoEmbeddings.

'small': English; 1024-hidden, 1 layer, 14.6M parameters
'medium': English; 2048-hidden, 1 layer, 28.0M parameters
'original': English; 4096-hidden, 2 layers, 93.6M parameters
'pt': Portuguese
'pubmed': English biomedical data; more information

BERT and Flair Combined
We can very easily mix and match Flair, ELMo, BERT and classic word embeddings. We instantiate each embedding we wish to combine and use them in a StackedEmbedding.
For instance, let's say we want to combine the multilingual Flair and BERT embeddings to train a hyper-powerful multilingual downstream task model.
First, instantiate the embeddings we wish to combine:
End of explanation
"""


from flair.embeddings import StackedEmbeddings

stacked_embeddings = StackedEmbeddings(
    embeddings=[flair_forward_embedding, flair_backward_embedding, bert_embedding])

"""
Explanation: Now we instantiate the StackedEmbeddings class and pass it a list containing these three embeddings:
End of explanation
"""


sentence = Sentence('The grass is green .')

stacked_embeddings.embed(sentence)

for token in sentence:
    print(token)
    print(token.embedding)

"""
Explanation: We use this embedding like all the other embeddings, i.e. call the embed() method over our sentences.
End of explanation
"""


