Update db

backend/create_old_database.py

@@ -0,0 +1,91 @@
+import pandas as pd
+from sqlalchemy import create_engine
+import pymysql
+import glob
+from sqlalchemy import text
+from plandb_methods import *
+from database_main import *
+from update_util import *
+import os
+import sys 
+import logging
+
+from database_main import compileContrastCurves
+from database_main import compileCompleteness
+
+from update_plandb_method import *
+
+
+"""
+Create old database, 2022 information, to be updated.
+For benchmarking purposes.
+Currently uses MySQL database: (database2022before)
+"""
+
+
+logging.basicConfig(filename="timer_log.txt", level=logging.INFO, format = "%(asctime)s - %(message)s")
+
+cache = False
+datestr = Time.now().datetime.strftime("%Y-%m")
+
+
+start_time_complete = time.time()
+logging.info("Complete database build start")
+
+data = getIPACdata()
+
+
+
+#photometric data
+photdict = loadPhotometryData(infile="plandb.sioslab.com/allphotdata_2015.npz")
+
+#band info
+bandzip = list(genBands())
+
+# TODO: look at genOrbitData method signature, with t0 default, and consider adding t0
+orbdata, orbitfits = genOrbitData(data, bandzip, photdict)
+
+ephemeris_orbitfits, ephemeris_orbdata = addEphemeris(data, orbitfits, orbdata, bandzip, photdict)
+
+quadrature_data = calcQuadratureVals(ephemeris_orbitfits, bandzip, photdict)
+
+exosims_json = 'plandb.sioslab.com/ci_perf_exosims.json'
+contr_data = calcContrastCurves(quadrature_data, exosims_json=exosims_json)
+
+comps, compdict, comps_data = calcPlanetCompleteness(contr_data, bandzip, photdict, exosims_json=exosims_json)
+
+plandata, stdata, orbitfits = generateTables(data, comps_data)
+
+# TODO: Prob replace this path stuff, prob HAVE to plandb.com/cache contr curvecalculations and then compile, or maybe write new compile function
+# TODO: calculate contrast curves and test it here
+contr_curvs_path = Path(f'plandb.sioslab.com/cache/cont_curvs_{datestr.replace("-", "_")}')
+contrast_curves = compileContrastCurves(stdata, contr_curvs_path)
+
+
+
+# newpdfs = pdfs.apply(addId, axis = 1)
+scenarios = pd.read_csv("plandb.sioslab.com/cache/scenario_angles.csv")
+
+# compiled_completeness_path = Path(f"plandb.com/cache/compiled_completeness_{datestr}.p")
+# if compiled_completeness_path.exists():
+#     completeness = pd.read_pickle(compiled_completeness_path)
+# else:
+#     completeness = compileCompleteness()
+#     completeness.to_pickle(compiled_completeness_path)
+
+
+# passwd = input("db password: ")
+# username = 'plandb_admin'
+engineToday = create_engine('mysql+pymysql://'+"andrewchiu"+':'+"Password123!"+'@localhost/TestToday',echo=True)
+
+#TODO: completeness later
+
+writeSQL(engineToday, plandata=plandata, stdata=stdata, orbitfits=orbitfits, orbdata=orbdata, pdfs=None, aliases=None,contrastCurves=None,scenarios=None, completeness=None)
+
+# TODO end time
+end_time_complete = time.time()
+elapsed_time = end_time_complete - start_time_complete
+logging.info(f"complete database build end. Elapsed time: {elapsed_time:.2f} seconds")
+
+print(f"elapsed time: {elapsed_time}" )
+

backend/create_today_database.py

@@ -0,0 +1,84 @@
+import pandas as pd
+from sqlalchemy import create_engine
+import pymysql
+import glob
+from sqlalchemy import text
+from plandb_methods import *
+from database_main import *
+from update_util import *
+import os
+import sys 
+import logging
+
+from database_main import compileContrastCurves
+from database_main import compileCompleteness
+
+from update_plandb_method import *
+
+
+
+
+"""
+Original compile of today's database using all current information.
+Currently uses MySQL database: (TestToday)
+"""
+
+
+logging.basicConfig(filename="timer_log.txt", level=logging.INFO, format = "%(asctime)s - %(message)s")
+
+cache = False
+datestr = Time.now().datetime.strftime("%Y-%m")
+
+
+
+
+start_time_2022 = time.time()
+logging.info("2022 database build start")
+
+#This is prob faster because it's pickled
+
+datestr = "2022-05"
+plandata_path = Path(f'plandb.sioslab.com/cache/plandata_{datestr}.p')
+planetsOld = pd.read_pickle(plandata_path)
+stdata_path = Path(f'plandb.sioslab.com/cache/stdata_{datestr}.p')
+starsOld = pd.read_pickle(stdata_path)
+orbfits_path = Path(f'plandb.sioslab.com/cache/table_orbitfits_{datestr}.p')
+orbitfitsOld = pd.read_pickle(orbfits_path)
+orbdata_path = Path(f'plandb.sioslab.com/cache/ephemeris_orbdata_{datestr}.p')
+orbitsOld = pd.read_pickle(orbdata_path)
+comps_path = Path(f'plandb.sioslab.com/cache/comps_{datestr}.p')
+pdfs = pd.read_pickle(comps_path)
+print(pdfs)
+compiled_contr_curvs_path = Path(f'plandb.sioslab.com/cache/compiled_cont_curvs_{datestr}.p')
+contr_curvs_path = Path(f'plandb.sioslab.com/cache/cont_curvs_{datestr.replace("-", "_")}')
+if compiled_contr_curvs_path.exists():
+    contrast_curvesOld = pd.read_pickle(compiled_contr_curvs_path)
+else:
+    contrast_curvesOld = compileContrastCurves(starsOld, contr_curvs_path)
+    contrast_curvesOld.to_pickle(compiled_contr_curvs_path)
+def addId(r):
+    r['pl_id']= list(planetsOld.index[(planetsOld['pl_name'] == r['Name'])])[0]
+    return r
+
+#TODO Pdfs
+newpdfs = pdfs.apply(addId, axis = 1)
+scenarios = pd.read_csv("plandb.sioslab.com/cache/scenario_angles.csv")
+
+before_engine = create_engine('mysql+pymysql://'+"andrewchiu"+':'+"Password123!"+'@localhost/database2022before',echo=True)
+
+writeSQL(before_engine, plandata=planetsOld, stdata=starsOld, orbitfits=orbitfitsOld, orbdata=orbitsOld, pdfs=None, aliases=None,contrastCurves=None,scenarios=None, completeness=None)
+
+print("Done")
+
+end_time_2022 = time.time()
+elapsed_time_2022 = end_time_2022 - start_time_2022
+logging.info(f"2022 database build complete, elapsed time: {elapsed_time_2022}")
+
+#TODO; compile later
+
+# compiled_completeness_path = Path(f"plandb.com/cache/compiled_completeness_{datestr}.p")
+# if compiled_completeness_path.exists():
+#     completeness = pd.read_pickle(compiled_completeness_path)
+# else:
+#     completeness = compileCompleteness()
+#     completeness.to_pickle(compiled_completeness_path)

backend/database_main.py

@@ -5,9 +5,10 @@
 from sqlalchemy import create_engine
 from tqdm import tqdm
 from plandb_methods import *
+from update_util import *
 
 def compileContrastCurves(stars, cont_path):
-    scenarios = pd.read_csv('cache/scenario_angles.csv')['scenario_name']
+    scenarios = pd.read_csv('plandb.sioslab.com/cache/scenario_angles.csv')['scenario_name']
 
     cols = ['scenario_name', 'st_id', 'r_lamD', 'r_as', 'r_mas', 'contrast', 'dMag', 'lam', 't_int_hr', 'fpp']
     contrastCurves = pd.DataFrame([], columns = cols)
@@ -31,12 +32,12 @@ def compileContrastCurves(stars, cont_path):
         else:
             indCC["scenario_name"] = scen
             indCC["st_id"] = list(stars.index[(stars['st_name'] == st_name)])[0]
-            contrastCurves = contrastCurves.append(indCC, ignore_index=True)
+            contrastCurves = contrastCurves._append(indCC, ignore_index=True)
     return contrastCurves
 
 def compileCompleteness():
     datestr = Time.now().datetime.strftime("%Y-%m")
-    comp_data = pd.read_pickle(f"cache/comps_data_{datestr}.p")
+    comp_data = pd.read_pickle(f"plandb.sioslab.com/cache/comps_data_{datestr}.p")
     col_names = comp_data.columns.values.tolist()
     scenario_names = []
     for x in col_names:
@@ -56,29 +57,30 @@ def compileCompleteness():
                     row['compMaxdMag_' + scenario_name]])
 
                 singleRow = pd.DataFrame(newRows, columns =  ['pl_id', 'completeness',  'scenario_name', 'compMinWA', 'compMaxWA', 'compMindMag', 'compMaxdMag'])
-                completeness = completeness.append(singleRow, ignore_index=True)
+                completeness = completeness._append(singleRow, ignore_index=True)
     # print(completeness)
     return completeness
 
 if __name__ == "__main__":
-    datestr = Time.now().datetime.strftime("%Y-%m")
-    plandata_path = Path(f'cache/plandata_{datestr}.p')
+    datestr = Time.now().datetime.strftime("2022-05")
+    plandata_path = Path(f'plandb.sioslab.com/cache/plandata_{datestr}.p')
     planets = pd.read_pickle(plandata_path)
 
-    stdata_path = Path(f'cache/stdata_{datestr}.p')
+    stdata_path = Path(f'plandb.sioslab.com/cache/stdata_{datestr}.p')
     stars = pd.read_pickle(stdata_path)
 
-    orbfits_path = Path(f'cache/table_orbitfits_{datestr}.p')
+    orbfits_path = Path(f'plandb.sioslab.com/cache/table_orbitfits_{datestr}.p')
     orbitfits = pd.read_pickle(orbfits_path)
 
-    orbdata_path = Path(f'cache/ephemeris_orbdata_{datestr}.p')
+    orbdata_path = Path(f'plandb.sioslab.com/cache/ephemeris_orbdata_{datestr}.p')
     orbits = pd.read_pickle(orbdata_path)
 
-    comps_path = Path(f'cache/comps_{datestr}.p')
+    comps_path = Path(f'plandb.sioslab.com/cache/comps_{datestr}.p')
     pdfs = pd.read_pickle(comps_path)
+    print(pdfs)
 
-    compiled_contr_curvs_path = Path(f'cache/compiled_cont_curvs_{datestr}.p')
-    contr_curvs_path = Path(f'cache/cont_curvs_{datestr.replace("-", "_")}')
+    compiled_contr_curvs_path = Path(f'plandb.sioslab.com/cache/compiled_cont_curvs_{datestr}.p')
+    contr_curvs_path = Path(f'plandb.sioslab.com/cache/cont_curvs_{datestr.replace("-", "_")}')
 
     if compiled_contr_curvs_path.exists():
         contrast_curves = pd.read_pickle(compiled_contr_curvs_path)
@@ -90,17 +92,26 @@ def addId(r):
         return r
     newpdfs = pdfs.apply(addId, axis = 1)
 
-    scenarios = pd.read_csv("cache/scenario_angles.csv")
-    compiled_completeness_path = Path(f"cache/compiled_completeness_{datestr}.p")
+    scenarios = pd.read_csv("plandb.sioslab.com/cache/scenario_angles.csv")
+
+    compiled_completeness_path = Path(f"plandb.sioslab.com/cache/compiled_completeness_{datestr}.p")
     if compiled_completeness_path.exists():
         completeness = pd.read_pickle(compiled_completeness_path)
     else:
         completeness = compileCompleteness()
         completeness.to_pickle(compiled_completeness_path)
 
-    passwd = input("db password: ")
-    username = 'plandb_admin'
+    # passwd = input("db password: ")
+    # username = 'plandb_admin'
 
-    engine = create_engine('mysql+pymysql://'+username+':'+passwd+'@localhost/plandb',echo=False)
+    engine = create_engine('mysql+pymysql://'+"andrewchiu"+':'+"Password123!"+'@localhost/testSios',echo=True)
+
     #pool_pre_ping=True for remote
-    writeSQL(engine, plandata=planets, stdata=stars, orbitfits=orbitfits, orbdata=orbits, pdfs=newpdfs, aliases=None,contrastCurves=contrast_curves,scenarios=scenarios, completeness=completeness)
+    planets.to_excel("main_planets.xlsx")
+    stars.to_excel("main_stars.xlsx")
+    # orbitfits.to_excel("main_orbitfits")
+    # orbits.to_excel('main_orbits.xlsx')
+    contrast_curves.to_excel('main_contrast.xlsx')
+    completeness.to_excel('main_completeness.xlsx')
+
+    writeSQL(engine, plandata=planets, stdata=stars, orbitfits=orbitfits, orbdata=orbits, pdfs=newpdfs, aliases=None,contrastCurves=contrast_curves,scenarios=None, completeness=completeness)