code/Chapter_6.py
def main(model=None, new_model_name='animal', output_dir=None, n_iter=20):
    """Set up the pipeline and entity recognizer, and train the new entity."""
    if model is not None:
        nlp = spacy.load(model)  # load existing spaCy model
        print("Loaded model '%s'" % model)
    else:
        nlp = spacy.blank('en')  # create blank Language class
        print("Created blank 'en' model")
    # Add entity recognizer to model if it's not in the pipeline
    # nlp.create_pipe works for built-ins that are registered with spaCy
    if 'ner' not in nlp.pipe_names:
        ner = nlp.create_pipe('ner')
        nlp.add_pipe(ner)
    # otherwise, get it, so we can add labels to it
    else:
        ner = nlp.get_pipe('ner')

    ner.add_label(LABEL)   # add new entity label to entity recognizer
    if model is None:
        optimizer = nlp.begin_training()
    else:
        # Note that 'begin_training' initializes the models, so it'll zero out
        # existing entity types.
        optimizer = nlp.entity.create_optimizer()



    # get names of other pipes to disable them during training
    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']
    with nlp.disable_pipes(*other_pipes):  # only train NER
        for itn in range(n_iter):
            random.shuffle(TRAIN_DATA)
            losses = {}
            for text, annotations in TRAIN_DATA:
                nlp.update([text], [annotations], sgd=optimizer, drop=0.35,
                           losses=losses)
            print(losses)

    # test the trained model
    test_text = 'Do you like horses?'
    doc = nlp(test_text)
    print("Entities in '%s'" % test_text)
    for ent in doc.ents:
        print(ent.label_, ent.text)

    # save model to output directory
    if output_dir is not None:
        output_dir = Path(output_dir)
        if not output_dir.exists():
            output_dir.mkdir()
        nlp.meta['name'] = new_model_name  # rename model
        nlp.to_disk(output_dir)
        print("Saved model to", output_dir)

        # test the saved model
        print("Loading from", output_dir)
        nlp2 = spacy.load(output_dir)
        doc2 = nlp2(test_text)
        for ent in doc2.ents:
            print(ent.label_, ent.text)
Set up the pipeline and entity recognizer, and train the new entity.
Set up the pipeline and entity recognizer, and train the new entity.
Set up the pipeline and entity recognizer and train the new entity .
def main(model=None, new_model_name='animal', output_dir=None, n_iter=20):
    if model is not None:
        nlp = spacy.load(model)  
        print("Loaded model '%s'" % model)
    else:
        nlp = spacy.blank('en')  
        print("Created blank 'en' model")
    if 'ner' not in nlp.pipe_names:
        ner = nlp.create_pipe('ner')
        nlp.add_pipe(ner)
    else:
        ner = nlp.get_pipe('ner')
    ner.add_label(LABEL)   
    if model is None:
        optimizer = nlp.begin_training()
    else:
        optimizer = nlp.entity.create_optimizer()
    other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']
    with nlp.disable_pipes(*other_pipes):  
        for itn in range(n_iter):
            random.shuffle(TRAIN_DATA)
            losses = {}
            for text, annotations in TRAIN_DATA:
                nlp.update([text], [annotations], sgd=optimizer, drop=0.35,
                           losses=losses)
            print(losses)
    test_text = 'Do you like horses?'
    doc = nlp(test_text)
    print("Entities in '%s'" % test_text)
    for ent in doc.ents:
        print(ent.label_, ent.text)
    if output_dir is not None:
        output_dir = Path(output_dir)
        if not output_dir.exists():
            output_dir.mkdir()
        nlp.meta['name'] = new_model_name  
        nlp.to_disk(output_dir)
        print("Saved model to", output_dir)
        print("Loading from", output_dir)
        nlp2 = spacy.load(output_dir)
        doc2 = nlp2(test_text)
        for ent in doc2.ents:
            print(ent.label_, ent.text)
def main ( model = None , new_model_name = 'animal' , output_dir = None , n_iter = 20 ) : if model is not None : nlp = spacy . load ( model ) print ( "Loaded model '%s'" % model ) else : nlp = spacy . blank ( 'en' ) print ( "Created blank 'en' model" ) if 'ner' not in nlp . pipe_names : ner = nlp . create_pipe ( 'ner' ) nlp . add_pipe ( ner ) else : ner = nlp . get_pipe ( 'ner' ) ner . add_label ( LABEL ) if model is None : optimizer = nlp . begin_training ( ) else : optimizer = nlp . entity . create_optimizer ( ) other_pipes = [ pipe for pipe in nlp . pipe_names if pipe != 'ner' ] with nlp . disable_pipes ( * other_pipes ) : for itn in range ( n_iter ) : random . shuffle ( TRAIN_DATA ) losses = { } for text , annotations in TRAIN_DATA : nlp . update ( [ text ] , [ annotations ] , sgd = optimizer , drop = 0.35 , losses = losses ) print ( losses ) test_text = 'Do you like horses?' doc = nlp ( test_text ) print ( "Entities in '%s'" % test_text ) for ent in doc . ents : print ( ent . label_ , ent . text ) if output_dir is not None : output_dir = Path ( output_dir ) if not output_dir . exists ( ) : output_dir . mkdir ( ) nlp . meta [ 'name' ] = new_model_name nlp . to_disk ( output_dir ) print ( "Saved model to" , output_dir ) print ( "Loading from" , output_dir ) nlp2 = spacy . load ( output_dir ) doc2 = nlp2 ( test_text ) for ent in doc2 . ents : print ( ent . label_ , ent . text )
Set up the pipeline and entity recognizer, and train the new entity.
Set up the pipeline and entity recognizer and train the new entity .
"""Set up the pipeline and entity recognizer, and train the new entity.""" # load existing spaCy model # create blank Language class # Add entity recognizer to model if it's not in the pipeline # nlp.create_pipe works for built-ins that are registered with spaCy # otherwise, get it, so we can add labels to it # add new entity label to entity recognizer # Note that 'begin_training' initializes the models, so it'll zero out # existing entity types. # get names of other pipes to disable them during training # only train NER # test the trained model # save model to output directory # rename model # test the saved model
model
new_model_name
output_dir
n_iter