code/Chapter_6.py
def main(model=None, output_dir=None, n_iter=100):
    """Load the model, set up the pipeline and train the entity recognizer."""
    if model is not None:
        nlp = spacy.load(model)  # load existing spaCy model
        print("Loaded model '%s'" % model)
    else:
        nlp = spacy.blank('en')  # create blank Language class
        print("Created blank 'en' model")

    # create the built-in pipeline components and add them to the pipeline
    # nlp.create_pipe works for built-ins that are registered with spaCy
    if 'ner' not in nlp.pipe_names:
        ner = nlp.create_pipe('ner')
        nlp.add_pipe(ner, last=True)
    # otherwise, get it so we can add labels
    else:
        ner = nlp.get_pipe('ner')

    # add labels
    for _, annotations in TRAIN_DATA:
        for ent in annotations.get('entities'):
            ner.add_label(ent[2])

    # get names of other pipes to disable them during training
    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']
    with nlp.disable_pipes(*other_pipes):  # only train NER
        optimizer = nlp.begin_training()
        for itn in range(n_iter):
            random.shuffle(TRAIN_DATA)
            losses = {}
            for text, annotations in TRAIN_DATA:
                nlp.update(
                    [text],  # batch of texts
                    [annotations],  # batch of annotations
                    drop=0.5,  # dropout - make it harder to memorise data
                    sgd=optimizer,  # callable to update weights
                    losses=losses)
            print(losses)

    # test the trained model
    for text, _ in TRAIN_DATA:
        doc = nlp(text)
        print('Entities', [(ent.text, ent.label_) for ent in doc.ents])
        print('Tokens', [(t.text, t.ent_type_, t.ent_iob) for t in doc])

    # save model to output directory
    if output_dir is not None:
        output_dir = Path(output_dir)
        if not output_dir.exists():
            output_dir.mkdir()
        nlp.to_disk(output_dir)
        print("Saved model to", output_dir)

        # test the saved model
        print("Loading from", output_dir)
        nlp2 = spacy.load(output_dir)
        for text, _ in TRAIN_DATA:
            doc = nlp2(text)
            print('Entities', [(ent.text, ent.label_) for ent in doc.ents])
            print('Tokens', [(t.text, t.ent_type_, t.ent_iob) for t in doc])
Load the model, set up the pipeline and train the entity recognizer.
Load the model, set up the pipeline and train the entity recognizer.
Load the model set up the pipeline and train the entity recognizer .
def main(model=None, output_dir=None, n_iter=100):
    if model is not None:
        nlp = spacy.load(model)  
        print("Loaded model '%s'" % model)
    else:
        nlp = spacy.blank('en')  
        print("Created blank 'en' model")
    if 'ner' not in nlp.pipe_names:
        ner = nlp.create_pipe('ner')
        nlp.add_pipe(ner, last=True)
    else:
        ner = nlp.get_pipe('ner')
    for _, annotations in TRAIN_DATA:
        for ent in annotations.get('entities'):
            ner.add_label(ent[2])
    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']
    with nlp.disable_pipes(*other_pipes):  
        optimizer = nlp.begin_training()
        for itn in range(n_iter):
            random.shuffle(TRAIN_DATA)
            losses = {}
            for text, annotations in TRAIN_DATA:
                nlp.update(
                    [text],  
                    [annotations],  
                    drop=0.5,  
                    sgd=optimizer,  
                    losses=losses)
            print(losses)
    for text, _ in TRAIN_DATA:
        doc = nlp(text)
        print('Entities', [(ent.text, ent.label_) for ent in doc.ents])
        print('Tokens', [(t.text, t.ent_type_, t.ent_iob) for t in doc])
    if output_dir is not None:
        output_dir = Path(output_dir)
        if not output_dir.exists():
            output_dir.mkdir()
        nlp.to_disk(output_dir)
        print("Saved model to", output_dir)
        print("Loading from", output_dir)
        nlp2 = spacy.load(output_dir)
        for text, _ in TRAIN_DATA:
            doc = nlp2(text)
            print('Entities', [(ent.text, ent.label_) for ent in doc.ents])
            print('Tokens', [(t.text, t.ent_type_, t.ent_iob) for t in doc])
def main ( model = None , output_dir = None , n_iter = 100 ) : if model is not None : nlp = spacy . load ( model ) print ( "Loaded model '%s'" % model ) else : nlp = spacy . blank ( 'en' ) print ( "Created blank 'en' model" ) if 'ner' not in nlp . pipe_names : ner = nlp . create_pipe ( 'ner' ) nlp . add_pipe ( ner , last = True ) else : ner = nlp . get_pipe ( 'ner' ) for _ , annotations in TRAIN_DATA : for ent in annotations . get ( 'entities' ) : ner . add_label ( ent [ 2 ] ) other_pipes = [ pipe for pipe in nlp . pipe_names if pipe != 'ner' ] with nlp . disable_pipes ( * other_pipes ) : optimizer = nlp . begin_training ( ) for itn in range ( n_iter ) : random . shuffle ( TRAIN_DATA ) losses = { } for text , annotations in TRAIN_DATA : nlp . update ( [ text ] , [ annotations ] , drop = 0.5 , sgd = optimizer , losses = losses ) print ( losses ) for text , _ in TRAIN_DATA : doc = nlp ( text ) print ( 'Entities' , [ ( ent . text , ent . label_ ) for ent in doc . ents ] ) print ( 'Tokens' , [ ( t . text , t . ent_type_ , t . ent_iob ) for t in doc ] ) if output_dir is not None : output_dir = Path ( output_dir ) if not output_dir . exists ( ) : output_dir . mkdir ( ) nlp . to_disk ( output_dir ) print ( "Saved model to" , output_dir ) print ( "Loading from" , output_dir ) nlp2 = spacy . load ( output_dir ) for text , _ in TRAIN_DATA : doc = nlp2 ( text ) print ( 'Entities' , [ ( ent . text , ent . label_ ) for ent in doc . ents ] ) print ( 'Tokens' , [ ( t . text , t . ent_type_ , t . ent_iob ) for t in doc ] )
Load the model, set up the pipeline and train the entity recognizer.
Load the model set up the pipeline and train the entity recognizer .
"""Load the model, set up the pipeline and train the entity recognizer.""" # load existing spaCy model # create blank Language class # create the built-in pipeline components and add them to the pipeline # nlp.create_pipe works for built-ins that are registered with spaCy # otherwise, get it so we can add labels # add labels # get names of other pipes to disable them during training # only train NER # batch of texts # batch of annotations # dropout - make it harder to memorise data # callable to update weights # test the trained model # save model to output directory # test the saved model
model
output_dir
n_iter