From be1eb22421d37877e6f7bf504c86e93015b8bf6d Mon Sep 17 00:00:00 2001 From: icecat Date: Wed, 7 Dec 2016 13:24:34 +0800 Subject: [PATCH] save cwb 36hr data to mongo --- cwb36hr2mongo.py | 86 ++++++++++++++++++++++++++++++++++++++++++++++++ weather.py | 18 +++++++--- 2 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 cwb36hr2mongo.py diff --git a/cwb36hr2mongo.py b/cwb36hr2mongo.py new file mode 100644 index 0000000..7125b80 --- /dev/null +++ b/cwb36hr2mongo.py @@ -0,0 +1,86 @@ +# cwb api type ID F-C0032-001 +# save data to mongo +from pymongo import MongoClient +import json +import uuid + +def F_C0032_001mongo(pure_data): + #connect to mongo + client = MongoClient("mongodb://localhost:27017/") + db = client['nctu_bot'] + collect = db['weather'] + + check_insert = 1 + check_data = 1 + deal_data = [{'_id' : '', 'city_county' : '', 'start_time' : '', 'end_time' : '', 'wx_value' : 0, 'wx_str' : '', 'pop_value' : 0, 'pop_unit' : '', 'ci_str' : '', 'mint_value' : 0, 'mint_unit' : '', 'maxt_value' : 0, 'maxt_unit' : '', 'update' : 0}, + {'_id' : '', 'city_county' : '', 'start_time' : '', 'end_time' : '', 'wx_value' : 0, 'wx_str' : '', 'pop_value' : 0, 'pop_unit' : '', 'ci_str' : '', 'mint_value' : 0, 'mint_unit' : '', 'maxt_value' : 0, 'maxt_unit' : '', 'update' : 0}, + {'_id' : '', 'city_county' : '', 'start_time' : '', 'end_time' : '', 'wx_value' : 0, 'wx_str' : '', 'pop_value' : 0, 'pop_unit' : '', 'ci_str' : '', 'mint_value' : 0, 'mint_unit' : '', 'maxt_value' : 0, 'maxt_unit' : '', 'update' : 0}] + + #reshape data structure + for tt in range(len( pure_data['records']['location'] )): + for tt0 in range(3): + deal_data[tt0]['_id'] = str(uuid.uuid3(uuid.uuid1(), 'javascript')) + #print (deal_data[tt0]['_id']) + deal_data[tt0]['city_county'] = pure_data['records']['location'][tt]['locationName'] + for tt1 in range(len( pure_data['records']['location'][tt]['weatherElement'] )): + if pure_data['records']['location'][tt]['weatherElement'][tt1]['elementName'] == 'Wx': + deal_data[tt0]['start_time'] = pure_data['records']['location'][tt]['weatherElement'][tt1]['time'][tt0]['startTime'] + deal_data[tt0]['end_time'] = pure_data['records']['location'][tt]['weatherElement'][tt1]['time'][tt0]['endTime'] + deal_data[tt0]['wx_value'] = pure_data['records']['location'][tt]['weatherElement'][tt1]['time'][tt0]['parameter']['parameterValue'] + deal_data[tt0]['wx_str'] = pure_data['records']['location'][tt]['weatherElement'][tt1]['time'][tt0]['parameter']['paramterName'] + elif pure_data['records']['location'][tt]['weatherElement'][tt1]['elementName'] == 'PoP': + deal_data[tt0]['pop_value'] = pure_data['records']['location'][tt]['weatherElement'][tt1]['time'][tt0]['parameter']['paramterName'] + deal_data[tt0]['pop_unit'] = pure_data['records']['location'][tt]['weatherElement'][tt1]['time'][tt0]['parameter']['parameterUnit'] + elif pure_data['records']['location'][tt]['weatherElement'][tt1]['elementName'] == 'CI': + deal_data[tt0]['ci_str'] = pure_data['records']['location'][tt]['weatherElement'][tt1]['time'][tt0]['parameter']['paramterName'] + elif pure_data['records']['location'][tt]['weatherElement'][tt1]['elementName'] == 'MinT': + deal_data[tt0]['mint_value'] = pure_data['records']['location'][tt]['weatherElement'][tt1]['time'][tt0]['parameter']['paramterName'] + deal_data[tt0]['mint_unit'] = pure_data['records']['location'][tt]['weatherElement'][tt1]['time'][tt0]['parameter']['parameterUnit'] + elif pure_data['records']['location'][tt]['weatherElement'][tt1]['elementName'] == 'MaxT': + deal_data[tt0]['maxt_value'] = pure_data['records']['location'][tt]['weatherElement'][tt1]['time'][tt0]['parameter']['paramterName'] + deal_data[tt0]['maxt_unit'] = pure_data['records']['location'][tt]['weatherElement'][tt1]['time'][tt0]['parameter']['parameterUnit'] + else : + check_data = 0 + print ('data structure error') + + + + #save data and prevent double + for i in range(3): + querry_data = collect.find({'city_county':deal_data[i]['city_county'],'start_time' : deal_data[i]['start_time']}).sort('update', -1).limit(1) + if querry_data.count() != 0: + if (querry_data[0]['start_time'] == deal_data[i]['start_time'] and + querry_data[0]['end_time'] == deal_data[i]['end_time'] and + querry_data[0]['wx_value'] == deal_data[i]['wx_value'] and + querry_data[0]['wx_str'] == deal_data[i]['wx_str'] and + querry_data[0]['pop_value'] == deal_data[i]['pop_value'] and + querry_data[0]['pop_unit'] == deal_data[i]['pop_unit'] and + querry_data[0]['ci_str'] == deal_data[i]['ci_str'] and + querry_data[0]['mint_value'] == deal_data[i]['mint_value'] and + querry_data[0]['mint_unit'] == deal_data[i]['mint_unit'] and + querry_data[0]['maxt_value'] == deal_data[i]['maxt_value'] and + querry_data[0]['maxt_unit'] == deal_data[i]['maxt_unit']): + #print("data ", end="", flush=True) + #print(i, end="", flush=True) + #print(" is exist") + continue + else : + deal_data[i]['update'] = querry_data[0]['update'] + 1 + + insert_id = collect.insert_one(deal_data[i]).inserted_id + if insert_id is None: + check_insert = 0 + print('error insert data to mongo') + + + client.close() + + + if check_data == 0 and check_insert == 0: + return -2 + elif check_data == 0 and check_insert == 1: + return -3 + elif check_data == 1 and check_insert == 0: + return -1 + else : + return 1 \ No newline at end of file diff --git a/weather.py b/weather.py index e1eb56e..e301900 100644 --- a/weather.py +++ b/weather.py @@ -1,6 +1,6 @@ # encoding=utf-8 -from pymongo import MongoClient from pprint import pprint +from cwb36hr2mongo import F_C0032_001mongo import urllib.request import urllib.parse import json @@ -19,11 +19,21 @@ with open("token","r") as token_file: token = token_file.readline().rstrip('\n') - print(token) + #print(token) target_url.add_header( 'Authorization' , token) fp = urllib.request.urlopen(target_url) pure_data = json.loads(fp.read().decode('utf-8')) -pprint(pure_data) - +#pprint(pure_data) fp.close() + +#save to mongo +result = F_C0032_001mongo(pure_data) +if result == -2: + print ('data structure and mongo insert error!') +elif result == -3: + print ('data structure and mongo data error!') +elif result == -1: + print ('mongo insert error!') +else : + print ('data insert to mongo success!') \ No newline at end of file