mcriblikelihood/preprocessing.py at master · andyclee/mcriblikelihood · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
from datacollect import PorkPrices, dateToFloat, MCRIB_DATES
import pandas as pd

"""
PREPROCESSING
"""

def movingAverage(dataArr, numPoints):
    maData = []
    for i in range(numPoints, len(dataArr)):
        curMA = 0
        for j in range(numPoints):
            curMA += maData[i - numPoints + j]

        curMA = curMA / numPoints
        maData.append(curMA)

    return maData

class PreprocessedData():
    data = PorkPrices()
    windowSize = 0
    prices = []
    years = []
    pastYears = []
    trend = []
    mcrib = []
    independentVars = []
    dependentVars = []

    def __init__(self, window):
        self.windowSize = window
        for row in self.data.combinedPrices:
            self.years.append(row[0])
            self.prices.append(row[1])
            self.trend.append(row[2])

        self.years = pd.Series(self.years)
        self.years = self.years.apply(lambda x : x.lower())
        self.prices = self.deseasonalize(self.years, self.prices)
        self.prices = pd.Series(self.prices)
        self.trend = pd.Series(self.trend)

        self.prices = self.prices.rolling(center=True, window=self.windowSize).mean()

        #Format date for regression
        self.years = self.years.apply(dateToFloat)

        pastYears = []
        for row in self.data.historicPrices:
            pastYears.append(row[0])

        mcribArr = []
        for year in pastYears:
            if (year.lower() in MCRIB_DATES):
                mcribArr.append(1)
            else:
                mcribArr.append(0)

        self.mcrib = pd.Series(mcribArr, name="has_mcrib")
        self.dependentVars = self.mcrib

        cols = {"year": self.years, "price": self.prices, "trend": self.trend}
        self.independentVars = pd.DataFrame(data=cols)

        #Drop future values
        self.independentVars.drop(self.independentVars.index[self.dependentVars.shape[0]:], inplace=True)
        maDrops = self.windowSize // 2
        self.independentVars.drop(self.independentVars.index[:maDrops], inplace=True)
        self.years = self.independentVars['year']
        self.prices = self.independentVars['price']
        self.trend = self.independentVars['trend']
        self.dependentVars.drop(self.dependentVars.index[:maDrops], inplace=True)

    def mean(self, data):
        sum = 0
        for point in data:
            sum += point
        return sum / len(data)

    """
    Dates are expected in format of: three-letter-month year
    """
    def deseasonalize(self, dates, values):
        seasons = {}
        cycles = {}
        for idx, date in enumerate(dates):
            month = date[:3]
            year = date[4:]
            if month in seasons:
                seasons[month].append((values[idx], year))
            else:
                seasons[month] = [(values[idx], year)]

            if year in cycles:
                cycles[year].append(values[idx])
            else:
                cycles[year] = [values[idx]]

        #Get average prices per cycle
        for year, prices in cycles.items():
            cycles[year] = self.mean(prices)

        proportions = {}
        #Gets proportions for each month
        for month, prices in seasons.items():
            for price in prices:
                if month in proportions:
                    proportions[month].append(price[0] / cycles[price[1]])
                else:
                    proportions[month] = [price[0] / cycles[price[1]]]

        #Convert from proportions to indices
        for month, props in proportions.items():
            proportions[month] = self.mean(props)

        deseasonalized = []
        for idx, date in enumerate(dates):
            month = date[:3]
            deseasonalized.append(values[idx] / proportions[month])

        return deseasonalized