DistJoin/data_str2num.py at main · GIS-PuppetMaster/DistJoin · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
import numpy as np
import pandas as pd
from tqdm import tqdm

df = pd.read_csv("./data/IMDB/title.csv", sep=',', escapechar='\\', encoding='utf-8',
                 low_memory=False)
res = dict()
df.head()

data_array = np.array(df['phonetic_code'])
data_list = data_array.tolist()
data_list = list(set(data_list))
if np.nan in data_list:
    data_list.remove(np.nan)
data_list = [str(i) for i in data_list]
data_list.sort()
i = 1
for key in data_list:
    res[key] = i
    i += 1
for i in tqdm(range(len(df['phonetic_code']))):
    if (pd.notnull(df['phonetic_code'][i])):
        df['phonetic_code'][i] = res[df['phonetic_code'][i]]

data_array = np.array(df['series_years'])
data_list = data_array.tolist()
data_list = list(set(data_list))
if np.nan in data_list:
    data_list.remove(np.nan)
data_list = [str(i) for i in data_list]
data_list.sort()
i = 1
for key in data_list:
    res[key] = i
    i += 1
for i in tqdm(range(len(df['series_years']))):
    if (pd.notnull(df['series_years'][i])):
        df['series_years'][i] = res[df['series_years'][i]]

data_array = np.array(df['imdb_index'])
data_list = data_array.tolist()
data_list = list(set(data_list))
if np.nan in data_list:
    data_list.remove(np.nan)
data_list = [str(i) for i in data_list]
data_list.sort()
i = 1
for key in data_list:
    res[key] = i
    i += 1
for i in tqdm(range(len(df['imdb_index']))):
    if (pd.notnull(df['imdb_index'][i])):
        df['imdb_index'][i] = res[df['imdb_index'][i]]

df.to_csv("./data/IMDB/title_.csv", header=False, index=False)