Simple-Author-Profiling-Task-with-python/preprocessing.py at master · NiloofarSafi/Simple-Author-Profiling-Task-with-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
import pandas as pd

import string
import re
from nltk.corpus import stopwords
import enchant
from preprocess import ark_tweet_tokenizer

EN_DICT = enchant.Dict("en_US")

def check_english(string):
    english_words = []
    tokens = ark_tweet_tokenizer(string)
    for token in tokens:
        if EN_DICT.check(token):
            english_words.append(token)
    return " ".join(english_words)


X, Y, doc_id = [], [], []
dict = enchant.Dict("en_US")
df = pd.read_csv('/home/niloofar/PycharmProjects/AdvancedNLP/Assign1/blog_test_data.csv', encoding='utf-8')
for index, row in df.iterrows():
    lower_case = row['Document'].lstrip().rstrip().lower()
    for c in string.punctuation:
        lower_case = lower_case.replace(c,'')
    normal = re.sub(r'[0-9]',"",lower_case)
    pattern = re.compile(r'\b(' + r'|'.join(stopwords.words('english')) + r')\b\s*')
    normal = pattern.sub('', normal)
    normal = check_english(normal)
    X.append(normal)
    # Y.append(row['Label'])
    doc_id.append(row['File_name'])
    print(index)

# train_data = {}
# train_data["File_name"] = doc_id
# train_data["Document"] = X
# train_data["Label"] = Y
# df = pd.DataFrame(train_data, columns=['File_name', 'Document', 'Label'])
# df.to_csv('train_data_preprocessed.csv', index=False, encoding='utf-8')


test_data = {}
test_data["File_name"] = doc_id
test_data["Document"] = X
df = pd.DataFrame(test_data, columns=['File_name', 'Document'])
df.to_csv('blog_test_data_preprocessed.csv', index=False, encoding='utf-8')
#


X, Y, doc_id = [], [], []
dict = enchant.Dict("en_US")
df = pd.read_csv('/home/niloofar/PycharmProjects/AdvancedNLP/Assign1/twitter_test_data.csv', encoding='utf-8')
for index, row in df.iterrows():
    lower_case = row['Document'].lstrip().rstrip().lower()
    for c in string.punctuation:
        lower_case = lower_case.replace(c,'')
    normal = re.sub(r'[0-9]',"",lower_case)
    pattern = re.compile(r'\b(' + r'|'.join(stopwords.words('english')) + r')\b\s*')
    normal = pattern.sub('', normal)
    normal = check_english(normal)
    X.append(normal)
    # Y.append(row['Label'])
    doc_id.append(row['File_name'])
    print(index)


twitter_test_data = {}
twitter_test_data["File_name"] = doc_id
twitter_test_data["Document"] = X
df = pd.DataFrame(twitter_test_data, columns=['File_name', 'Document'])
df.to_csv('twitter_test_data_preprocessed.csv', index=False, encoding='utf-8')