# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Introduction to Python  
#
# ## Regular Expressions Tutorial
#
# Inspired in these sources: [1](https://github.com/KennyMiyasato/regex_blog_post/blob/master/regex_blog_post.ipynb), [2](https://luca-pessina.medium.com/python-regular-expressions-in-5-minutes-ecc8b6624308), [3](https://developers.google.com/edu/python/regular-expressions)  
# Tables from: [regexr.com](https://regexr.com/)

# #### Regular expressions are a powerful language for matching text patterns. This page gives a basic introduction to regular expressions themselves sufficient for our Python exercises and shows how regular expressions work in Python. The Python "re" module provides regular expression support.
#
# #### In Python a regular expression search is typically written as:
#
# > match = re.search(pattern, str)
#

# ### Importing modules

import re
import pandas as pd

# ### Defining sample strings 

# +
lowercase_alphabet = "abcdefghijklmnopqrstuvwxyz"
uppercase_alphabet = lowercase_alphabet.upper()
numbers = "1234567890"
sentence = "The Quick Brown Fox Jumps Over The Lazy Dog"
website = "https://webmail2016.univie.ac.at/"
phone_numbers = """123-456-7890
                    987.654.321
                    234-567-8901
                    654.321.987
                    345-678-9012
                    321.654.978
                    456-789-0123
                """
special_characters = "[\^$.|?*+()"

text_with_email = 'xyz <EMAIL> purple monkey'

long_text = '''
Nicht alles, was Gold ist, funkelt
Nicht alles, was Gold ist, funkelt,
Nicht jeder, der wandert, verlorn,
Das Alte wird nicht verdunkelt,
Noch Wurzeln der Tiefe erfroren.
Aus Asche wird Feuer geschlagen,
Aus Schatten geht Licht hervor;
Heil wird geborstenes Schwert,
Und König, der die Krone verlor.
'''
# -

# ### Match explicit character(s)
# In order to match characters explicitly, all you need to do is type what you'd like to find.

re.findall("abc", lowercase_alphabet)

re.findall("abde", lowercase_alphabet)

re.findall("cht", long_text)

re.findall("ABC", uppercase_alphabet)

re.findall("abc", uppercase_alphabet)

re.findall("abc", uppercase_alphabet, flags=re.IGNORECASE)

# ### Match with special character(s)
# In order to match any *special characters `[\^$.|?*+()`* you have first introduce a backslash `\` followed by the character you'd like to select.

re.findall("webmail2016\.univie\.ac\.at", website)

re.findall("we.ma.l2016.uni.ie..c.at", website)

re.findall("\$", special_characters)

re.findall("$", special_characters)

re.findall("\|", special_characters)

re.findall("|", special_characters)

# ### Match by pattern
# There are a lot of ways we can match a pattern. Regex has its own syntax so we could pick and choose how we want our patterns to look like.

# ### Character Classes
# | Class | Explanation |
# |---|---|
# | . | any character except newline |
# | \w \d \s | word, digit, whitespace |
# | \W \D \S | not word, not digit, not whitespace |
# | [abc] | any of a, b, or c |
# | [^abc] | not a, b, or c |
# | [a-g] | characters between a & g |
#
# [0-9] is not always equivalent to \d. 
# In python3, [0-9] matches only 0123456789 characters, while \d matches [0-9] and other digit characters, for example Eastern Arabic numerals ٠١٢٣٤٥٦٧٨٩.

# ### Quantifiers & Alternation
# | Class | Explanation |
# |---|---|
# | a* a+ a? | 0 or more, 1 or more, 0 or 1 |
# | a{5} a{2,} | exactly five, two or more |
# | a{1,3} | between one & three |
# | a+? a{2,}? | match as few as possible |
# | ab\|cd | match ab or cd |
#

re.findall("\w{1,}", sentence)

re.findall("\W+", sentence)

re.findall("[The]+", sentence)

re.findall("[^TLB]{1,}", sentence)

re.findall("[a-rA-R]+", sentence)

re.findall("l+", long_text)

re.findall("l+?", long_text)

# ### Anchors
# | Class | Explanation |
# |---|---|
# | ^abc$ | start / end of the string |
# | \b \B | character between a & g |

re.findall("^[TheQBFJOLD]", sentence)

re.findall("^[TheQBFJOLD]+", sentence)

re.findall("[a-z,\.]{1,}$", long_text)

# ### Escaped Characters
# | Class | Explanation |
# |---|---|
# | \\. \\* \\\ | escaped special characters |
# | \\t \\n \\r | tab, linefeed, carriage return |

re.findall("\d{3}\-\d{3}\-\d{4}", phone_numbers)

re.findall("\d{3}[\.\-]{1}\d{3}[\.\-]{1}\d{3,4}", phone_numbers)

print(long_text)

re.findall("\n.{5}", long_text)

# ### Groups & Lookaround
# | Class | Explanation |
# |---|---|
# | (abc) | capture group |
# | \1 | backreference to group #1 |
# | (?:abc) | non-capturing group |
# | (?=abc) | positive lookahead |
# | (?!abc) | negative lookahead |

match = re.search('([\w.-]+)@([\w.-]+)', text_with_email)
if match:
    print(match.group())   ## '<EMAIL>' (the whole match)
    print(match.group(1))  ## 'alice-b' (the username, group 1)
    print(match.group(2))  ## 'google.com' (the host, group 2)

my_string = 'purple <EMAIL>, blah monkey <EMAIL> blah dishwasher'
tuples = re.findall(r'([\w\.-]+)@([\w\.-]+)', my_string)
print(tuples)  ## [('alice', 'google.com'), ('bob', 'abc.com')]
for tuple in tuples:
    print(tuple[0])  ## username
    print(tuple[1])  ## host

# ### Substituting characters

string = 'This%&$is@an#example'
string = re.sub('[%&$@#]+', ' ', string)
print(string)

# ### Regular Expressions with Pandas

# +
fragebogen = pd.read_csv('../Data/CSV/fragebogen.csv', names=["id",
                                                              "nummer", 
                                                              "titel", 
                                                              "schlagwoerter", 
                                                              "erscheinungsjahr",
                                                              "autoren", 
                                                              "originaldaten",
                                                              "anmerkung",
                                                              "freigabe",
                                                              "checked",
                                                              "wordleiste",
                                                              "druck",
                                                              "online",
                                                              "publiziert",
                                                              "fragebogen_typ_id",])
fragebogen.drop(["id","schlagwoerter","erscheinungsjahr","autoren","originaldaten","anmerkung","freigabe",
                 "checked","wordleiste","druck","online","publiziert","fragebogen_typ_id",], inplace=True, axis=1)

fragebogen.set_index("nummer", drop=True, inplace=True)
fragebogen = fragebogen[fragebogen.titel.str.startswith('Fragebogen')]
# -

fragebogen.info()

fragebogen.head(20)

# #### Spliting the string in groups

# +
regex1 = r'([Fragebon]+)\s{1}([0-9]+)[:]{1}([,A-ZÄÖÜa-zäöüß0-9.\s]+)[,\s]*([=\-\(\)\sA-ZÄÖÜa-zäöüß0-9]*)'

fragebogen.titel.str.extract(regex1).head()
# -

fragebogen.loc[:,'fragebogen_num'] = fragebogen.titel.str.extract(regex1)[1].str.strip()
fragebogen.loc[:,'fragebogen_headwords'] = fragebogen.titel.str.extract(regex1)[2].str.strip()
fragebogen.loc[:,'fragebogen_series'] = fragebogen.titel.str.extract(regex1)[3].str.strip()

fragebogen.head(20)

# ## Exercises

# ### Twitter:
#     
# #### In twitter a username is validated if:
#
# + Is a text with a ‘@’ at the beginning so in regex ‘^@’
# + Contains only alphanumerical characters (letters A-Z, numbers 0–9). 
#     + In this case we can use either ‘[A-Za-z0–9]’ or ‘[\w\d]’. 
#     + We use the brackets because we want to apply another quantifiers to the set.
# + It must be between 4 and 15 characters, so {4,15}.
#
# You can use the re.match() or re.search() methods to find a match for your string, in the following code is reported the selection throught the regex statement.

# #### Let's find some twitter databases:
#
# [First GOP debate](https://www.kaggle.com/crowdflower/first-gop-debate-twitter-sentiment)  
# [US Elections 2020](https://www.kaggle.com/manchunhui/us-election-2020-tweets)

import sqlite3
conn = sqlite3.connect('../Data/tweet_database.sqlite')
cur = conn.cursor()

# #### Checking the tables in database

q1 = "SELECT name FROM sqlite_master WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%'"
cur.execute(q1)
cur.fetchall()

# #### Checking columns in table "sentiments" and querying the first row from the "sentiments" table

q2 = "SELECT * from Sentiment limit 1"
cur.execute(q2)
#print([f[0] for f in cur.description])
#cur.fetchall()

for idx, row in enumerate(cur.fetchall()[0]):
    print(f"{cur.description[idx][0]}:\t{row}")
    print()

# #### Retrieving first 50 rows from field sentiment.text:

q3 = "SELECT text from Sentiment limit 50"
cur.execute(q3)
cur.fetchmany(3)

tweets = cur.execute(q3).fetchall()
tweets = [t[0] for t in tweets]
tweets

# ### Or you may choose usingmentions Pandas for querying

q4 = "SELECT * from Sentiment limit 50"
df = pd.read_sql_query(q4, conn)
df.head()

# ### Now it is you time!
#
# #### Extract from tweets:
# + Mentions
# + URLs
# + Hashtags

for tweet in tweets:
    match = re.findall('@\w+', tweet)
    if match:
        print(match)

for tweet in tweets:
    match = re.findall('http+s?:/{2}\w{1,}\.\w{1,}/.\w{1,}', tweet)
    if match:
        print(match)

for tweet in tweets:
    match = re.findall('\#\w+', tweet)
    if match:
        print(match)



StarcoderdataJupyterScriptsDedupFiltered