dissertation/_preprocess.py at main · HelloYuiYui/dissertation · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
import pandas as pd
import nltk

from nltk.corpus import wordnet
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords

import numpy as np

from gensim.parsing.preprocessing import remove_stopwords

import preprocessor as p

import ssl, re, string

# Define a regex pattern to match URLs
url_pattern = re.compile(r'https?://\S+')

try:
    _create_unverified_https_context = ssl._create_unverified_context
except AttributeError:
    pass
else:
    ssl._create_default_https_context = _create_unverified_https_context


nltk.download("wordnet")
nltk.download("stopwords")
nltk.download("punkt")
nltk.download("averaged_perceptron_tagger")

def preprocess(t):

    p.set_options(p.OPT.URL, p.OPT.HASHTAG, p.OPT.MENTION, p.OPT.SMILEY, p.OPT.EMOJI)
    t = p.clean(t)

    t = lowercase(t)

    # t = url_removal(t)

    # t = numberSub(t)

    t = punctuation_removal(t)

    t = stopword_removal(t)

    t = lemmatizer(t)

    t = t.strip()

    return t

def lowercase(t):
    return t.lower()

def stopword_removal(t):
    # define set of English stopwords
    stop_words = set(stopwords.words('english'))

    # remove stopwords from tokens in dataset
    return " ".join([word for word in t.split(" ") if word not in stop_words])

def url_removal(t):
    # Define a function to remove URLs from text
    return url_pattern.sub('', t)

def punctuation_removal(text):
    text = text.lower()
    text = text.strip()
    text = re.sub('[,.?!()_:;]', '', text)
    # text = re.compile('<.*?>').sub('', text)
    # text = re.compile('[%s]' % re.escape(string.punctuation)).sub(' ', text)
    # text = re.sub('\s+', ' ', text)
    # text = re.sub('\[[0-9]*\]','NUM',text)
    # text = re.sub('[^\w\s]', '', str(text).lower().strip())
    # text = re.sub('\d',' ',text)
    # text = re.sub('\s+',' ',text)
    return text


def punctuation_removal_2(t):
    # t = re.compile('<.*?>').sub('.*?', t)
    # t = re.compile('[%s]' % re.escape(string.punctuation)).sub('', t)
    # t = re.sub('\s+', 'S', t)
    # t = re.sub(r'\[[0-9]*\]','D',t)
    # t = re.sub('[^a-zA-Z0-9-_\ ]+/gm', '', t)
    # t = re.sub(r'\d','N',t)
    # t = re.sub(r'\s+','S?',t)
    return t

def numberSub(t):
    # Doesn't do the thing I want it to do aAAAA
    t = re.sub('(?:^|\s)(\d+)(?=\s|$)', 'NUM', t)
    return t

#LEMMATIZATION
# Initialize the lemmatizer
wl = WordNetLemmatizer()

# This is a helper function to map NTLK position tags
def get_wordnet_pos(tag):
    if tag.startswith('J'):
        return wordnet.ADJ
    elif tag.startswith('V'):
        return wordnet.VERB
    elif tag.startswith('N'):
        return wordnet.NOUN
    elif tag.startswith('R'):
        return wordnet.ADV
    else:
        return wordnet.NOUN

# Tokenize the sentence
def lemmatizer(string):
    word_pos_tags = nltk.pos_tag(word_tokenize(string)) # Get position tags
    a=[wl.lemmatize(tag[0], get_wordnet_pos(tag[1])) for idx, tag in enumerate(word_pos_tags)] # Map the position tag and lemmatize the word/token
    return " ".join(a)

# Mainly taken from https://medium.com/analytics-vidhya/nlp-tutorial-for-text-classification-in-python-8f19cd17b49e