GenderClassification

from __future__ import division
from collections import defaultdict
import os
import argparse
import io
import importlib
import codecs
import traceback
import logging
import nltk
from nltk.corpus import stopwords
from nltk.tokenize import sent_tokenize, word_tokenize
import textblob
from textblob.classifiers import NaiveBayesClassifier
from textblob import formats
from xml.etree import ElementTree
import pickle
import csv
import json
import sys
import random
import re
import math
import operator

dataset = [],[]
data = []



def split_data(data,prob,count):
    #split data as per the split ratio
    results = [],[]
    trainSize = int(count * prob)
    print ("trainSize", trainSize)
    #while len(result) < trainSize:
    i=0
    for row in data:
        i += 1
        results[1 if i > trainSize else 0].append(row)
        #print("row", row)
    #print ("results", results)
    return results


def tokenize(message):
    '''preprocess the message (string) to produce a bag or words.'''
    message = message.lower()
    all_words = re.findall("[a-z0-9]+",message)
    #print("all words", all_words)
    return set(all_words)


def count_words(training_set):
    ''' training set consists of pairs (message,is_male),divide all words and their occurances in each status '''
    counts = defaultdict(lambda:[0,0])
    for message,is_male in training_set:
        for word in tokenize(message):
            counts[word][0 if is_male=="0" else 1] += 1
        #print("count" , counts)
    return counts


def class_probabiltiies(total_males,total_females):
    ''' Calculate priors '''
    total_message = total_males + total_females
    prior_males = total_males/total_message
    prior_females = total_females/total_message

    print("total_males, total_females", total_males, total_females)
    #print("prior_males, prior_females", prior_males, prior_females)
    return prior_males, prior_females


def word_probabilities(counts,total_males,total_females,k=0.5):
    '''turn the word_counts into a list of triplets:
       w , p(w|male),p(w|females)
    '''

    return [(w,
             (male+k)/(total_males + 2 * k),
             (female+k)/ (total_females + 2 *k))
             for w,(male,female) in counts.iteritems()]


def classify_probabiliy(word_probs,message,prior_male,prior_female):
    ''' Classify a message as either spam (1) or not_spam (0)'''
    message_words = tokenize(message)

    log_prob_if_male = log_prob_if_female = 0

    #iterate through each word in our vocab
    for word, prob_if_male, prob_if_female in word_probs:

        if word in message_words:

            log_prob_if_male += math.log(prob_if_male)
            log_prob_if_female += math.log(prob_if_female)

        else:

            log_prob_if_male += math.log(1 - prob_if_male)
            log_prob_if_female += math.log(1 - prob_if_female)

    prob_if_male = math.exp(log_prob_if_male) * prior_male
    prob_if_female = math.exp(log_prob_if_female) * prior_female

    if prob_if_male > prob_if_female:
        return True
    else:
        return False


def classify(word_probs1, prior_male1, prior_female1, message1):
        return classify_probabiliy(word_probs1, message1, prior_male1, prior_female1)


def train(self,training_set):
    #calculate the number of males and females in the trianing data and return the word probabilities
    num_males = 0
    num_females = 0
    for [message, is_male] in training_set:
        if is_male == "0":
            num_males=num_males+1
        else:
            num_females=num_females+1
    print("num males", num_males)
    print("num females", num_females)
    # Count the words in each status message
    word_counts = count_words(training_set)
    #calculate the prior probabilities
    prior_male,prior_female = class_probabiltiies(num_males,num_females)
    print (prior_male,prior_female)
    #calculate the prbabilities for each word
    word_probs = word_probabilities(word_counts,num_males,num_females,self)
#    print("word probs", word_probs)
    return word_probs,prior_male,prior_female



def precision(tp,fp,fn,tn):
    ''' What fraction of postitives were correctly identified '''
    return tp / (tp + fp)



def recall(tp,fp,fn,tn):
    '''How accurate were our positive predictions '''
    return tp/(tp+fn)


def accuracy(tp,fp,fn,tn):
    correct = tp + tn
    total = tp + tn + fn + fp

    return correct/total



def f1_score(tp,fp,fn,tn):
    p = precision(tp,fp,fn,tn)
    r = recall(tp,fp,fn,tn)

    return 2 * p * r / (p+r)

#PROGRAM START
with open("C:\Users\swetha Ch\Desktop\FacebookDataTCSS555Project\TCSS555\Train\profile\profile6.csv", "r") as profiles:
    #read from the profile and write to a list "data".
    count = 0
    next(profiles)
    for line in profiles:
        main = re.split(',', line)
        userID = main[1]
        gender = main[3]
        count = count+1
        with open("C:\Users\swetha Ch\Desktop\FacebookDataTCSS555Project\TCSS555\Train\Text"+"\\"+userID+".txt", "r") as status:
            values = status.read()
            values = values.replace(r"|"," ")
            values = values.replace(r"\n", " ")
            #print ("values", values)
            data.append((values, gender))
            status.close()

    #print("count", count)
profiles.close()

#print (data)

self = 0.5

random.seed(0)
#split the data according to the given split ratio
train_data, test_data = split_data(data,0.70,count)
#print("Train data", train_data)
print("length of Train data", len(train_data))
print("length of Test data", len(test_data))
#print("Test data", test_data)
##calculate the word probabilitites and prior probabilitites for the status message.
word_probs,prior_male,prior_female=train(self, train_data)
print ("prior_male,prior_female", prior_male,prior_female)
#as per the calulated probabilities, predict the gender for status in test data
classified = [(message,is_male,classify(word_probs, prior_male, prior_female, message)) for message, is_male in test_data]
#print ("classified",classified)


fp=fn=tp=tn=0

for x,est,tru in classified:
    #print (est,tru)
    if est == "0" and tru==False:
        fp+=1
    elif est== "0" and tru==True:
        tp+=1
    elif est== "1" and tru == False:
        tn+=1
    else:
        fn+=1

print ("tp,fp,fn,tn", tp,fp,fn,tn)
# error metrics
print ('accuracy: {}'.format(accuracy(tp,fp,fn,tn)))
print ('precision: {}'.format(precision(tp,fp,fn,tn)))
print ('recall: {}'.format(f1_score(tp,fp,fn,tn)))