# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import numpy as np

samples = ['我 毕业 于 北京理工大学', '我 就职 于 中国科学院技术研究所']

token_index = {}
for sample in samples:
    for w in sample.split():
        if w not in token_index:
            token_index[w] = len(token_index) + 1

print(len(token_index))

print(token_index)

result = np.zeros(shape=(len(samples), len(token_index) + 1, max(token_index.values()) + 1))

result.shape

for i, sample in enumerate(samples):
    for j, w in list(enumerate(sample.split())):
        index = token_index.get(w)
        print(j, index, w)
        result[i, j, index] = 1

result

results = np.zeros(shape=(len(samples), max(token_index.values()) + 1))

for i, sample in enumerate(samples):
    for _, w in list(enumerate(sample.split())):
        index = token_index.get(w)
        results[i, index] = 1

results



StarcoderdataJupyterScriptsDedupFiltered