Spaces:
Sleeping
Sleeping
File size: 4,488 Bytes
462fea8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 |
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from tokenize import tokenize\n",
"from io import BytesIO\n",
"\n",
"code = \"\"\"import nltk\n",
" from nltk.stem import PorterStemmer\n",
" porter_stemmer=PorterStemmer()\n",
" words=[\"connect\",\"connected\",\"connection\",\"connections\",\"connects\"]\n",
" stemmed_words=[porter_stemmer.stem(word) for word in words]\n",
" stemmed_words\"\"\"\n",
" \n",
"for tok in tokenize(BytesIO(code.encode('utf-8')).readline):\n",
" print(f\"Type: {tok.type}\\nString: {tok.string}\\nStart: {tok.start}\\nEnd: {tok.end}\\nLine: {tok.line.strip()}\\n======\\n\")\n"
]
},
{
"cell_type": "code",
"execution_count": 40,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['Create a function to summarize the data.', 'For each column in the dataframe, create a correlation matrix.', '3']\n"
]
}
],
"source": [
"import re\n",
"my_summary = '\\n1. Create a function to summarize the code.\\n2. At first, we will start by importing the pandas and numpy modules.'.strip()\n",
"my_summary = 'Create a function summarize and load the dataset.\\n1. To Load the dataset\\n2. To display the basic information\\n3.'.strip()\n",
"my_summary = '\\n1. Create a function to summarize the data.\\n2. For each column in the dataframe, create a correlation matrix.\\n3'\n",
"my_symmary = \"\\n1. Create a function to summarize the code.\\n2. At first, we will start by importing the pandas and numpy modules.\"\n",
"sentences = my_summary.split('\\n')[1:]\n",
"#remove the trailing list enumeration\n",
"new_sentences = []\n",
"for sentence in sentences:\n",
" new_sentences.append(re.sub(\"[0-9]+\\.\\s\", \"\", sentence))\n",
"print(new_sentences)"
]
},
{
"cell_type": "code",
"execution_count": 42,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\n",
"1. Create a function to summarize the data.\n",
"2.\n",
"the sentence is valid? True\n",
"\n",
" False SPACE\n",
"1 False X\n",
". False PUNCT\n",
"Create True VERB\n",
"a True DET\n",
"function True NOUN\n",
"to True PART\n",
"summarize True VERB\n",
"the True DET\n",
"data True NOUN\n",
". False PUNCT\n",
"\n",
" False SPACE\n",
"2 False X\n",
". False PUNCT\n",
"For each column in the dataframe, create a correlation matrix.\n",
"\n",
"the sentence is valid? True\n",
"For True ADP\n",
"each True DET\n",
"column True NOUN\n",
"in True ADP\n",
"the True DET\n",
"dataframe True NOUN\n",
", False PUNCT\n",
"create True VERB\n",
"a True DET\n",
"correlation True NOUN\n",
"matrix True NOUN\n",
". False PUNCT\n",
"\n",
" False SPACE\n",
"3\n",
"the sentence is valid? False\n",
"3 False NUM\n"
]
}
],
"source": [
"import spacy\n",
"nlp = spacy.load(\"en_core_web_sm\")\n",
"\n",
"\n",
"def is_valid(words: list[str]):\n",
" has_noun = False\n",
" has_verb = False\n",
" for word in words: \n",
" if word.pos_ in ['NOUN', 'PROPN', 'PRON']:\n",
" has_noun = True\n",
" if word.pos_ == 'VERB':\n",
" has_verb = True\n",
" return has_noun and has_verb\n",
"\n",
"doc = nlp(my_summary)\n",
"sentences = list(doc.sents)\n",
"\n",
"for sentence in sentences:\n",
" print(sentence)\n",
" print(\"the sentence is valid?\", is_valid(sentence))\n",
" for word in sentence:\n",
" print(word, word.is_alpha, word.pos_)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.2"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
|