diff --git a/python/examples/data_import/csv/.env.example b/python/examples/data_import/csv/.env.example new file mode 100644 index 000000000..83925d9af --- /dev/null +++ b/python/examples/data_import/csv/.env.example @@ -0,0 +1,4 @@ +SIFT_GRPC_URI= +SIFT_REST_URI= +SIFT_API_KEY= +ASSET_NAME= \ No newline at end of file diff --git a/python/examples/data_import/csv/custom/.env-example b/python/examples/data_import/csv/custom/.env-example deleted file mode 100644 index cdef5f890..000000000 --- a/python/examples/data_import/csv/custom/.env-example +++ /dev/null @@ -1,3 +0,0 @@ -SIFT_API_URI="" -SIFT_API_KEY="" -ASSET_NAME="" \ No newline at end of file diff --git a/python/examples/data_import/csv/custom/main.py b/python/examples/data_import/csv/custom/main.py deleted file mode 100644 index f79a2470d..000000000 --- a/python/examples/data_import/csv/custom/main.py +++ /dev/null @@ -1,70 +0,0 @@ -import csv -import os - -from dotenv import load_dotenv -from sift_py.data_import.config import CsvConfig -from sift_py.data_import.csv import CsvUploadService -from sift_py.data_import.status import DataImportService -from sift_py.rest import SiftRestConfig - -if __name__ == "__main__": - """ - Example of uploading a CSV file into Sift using custom CSV config. - """ - - load_dotenv() - - sift_uri = os.getenv("SIFT_API_URI") - assert sift_uri, "expected 'SIFT_API_URI' environment variable to be set" - - apikey = os.getenv("SIFT_API_KEY") - assert apikey, "expected 'SIFT_API_KEY' environment variable to be set" - - asset_name = os.getenv("ASSET_NAME") - assert asset_name, "expected 'ASSET_NAME' environment variable to be set" - - rest_config: SiftRestConfig = { - "uri": sift_uri, - "apikey": apikey, - } - - csv_upload_service = CsvUploadService(rest_config) - - # Create CSV config. - input_csv = "sample_data.csv" - - # Parse CSV to get channel names. - data_config = {} - with open(input_csv, "r") as f: - reader = csv.DictReader(f) - headers = next(reader) - for i, channel in enumerate(headers): - if channel == "timestamp": - continue - data_config[i + 1] = { - "name": channel, - # This example assumes all channels are doubles. - # Can also use `ChannelDoubleType.DOUBLE` or `double` - "data_type": "CHANNEL_DATA_TYPE_DOUBLE", - "description": f"Example channel {channel}", - } - - csv_config = CsvConfig( - { - "asset_name": asset_name, - "first_data_row": 2, - "time_column": { - "format": "TIME_FORMAT_ABSOLUTE_DATETIME", - # Can also use `TimeFormatType.ABSOLUTE_DATETIME` - "column_number": 1, - }, - "data_columns": data_config, - } - ) - - import_service: DataImportService = csv_upload_service.upload(input_csv, csv_config) - print(import_service.get_data_import()) - - print("Waiting for upload to complete...") - import_service.wait_until_complete() - print("Upload example complete!") diff --git a/python/examples/data_import/csv/custom/requirements.txt b/python/examples/data_import/csv/custom/requirements.txt deleted file mode 100644 index 2dda90fec..000000000 --- a/python/examples/data_import/csv/custom/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -python-dotenv -sift-stack-py diff --git a/python/examples/data_import/csv/main.py b/python/examples/data_import/csv/main.py new file mode 100644 index 000000000..4b6f4fef0 --- /dev/null +++ b/python/examples/data_import/csv/main.py @@ -0,0 +1,48 @@ +"""Import a CSV file into Sift.""" + +import os + +from dotenv import load_dotenv +from sift_client import SiftClient + +if __name__ == "__main__": + load_dotenv() + + grpc_uri = os.getenv("SIFT_GRPC_URI") + assert grpc_uri, "expected 'SIFT_GRPC_URI' environment variable to be set" + + rest_uri = os.getenv("SIFT_REST_URI") + assert rest_uri, "expected 'SIFT_REST_URI' environment variable to be set" + + apikey = os.getenv("SIFT_API_KEY") + assert apikey, "expected 'SIFT_API_KEY' environment variable to be set" + + asset_name = os.getenv("ASSET_NAME") + assert asset_name, "expected 'ASSET_NAME' environment variable to be set" + + client = SiftClient(api_key=apikey, grpc_url=grpc_uri, rest_url=rest_uri) + + # Auto-detect the config and import the file. + import_job = client.data_import.import_from_path( + "sample_data.csv", + asset=asset_name, + ) + + import_job.wait_until_complete() + + # If auto-detect doesn't quite match your file, inspect the config and patch + # it before importing. Common fixes: override a column's data type, change + # the time column or format, or drop a column that shouldn't be imported. + # + # config = client.data_import.detect_config("sample_data.csv") + # print(config) # inspect what was auto-detected + # + # # Example: drop a column from the import + # config.data_columns = [dc for dc in config.data_columns if dc.name != "channel_0"] + # + # import_job = client.data_import.import_from_path( + # "sample_data.csv", + # asset=asset_name, + # config=config, + # ) + # import_job.wait_until_complete() diff --git a/python/examples/data_import/csv/simple/requirements.txt b/python/examples/data_import/csv/requirements.txt similarity index 50% rename from python/examples/data_import/csv/simple/requirements.txt rename to python/examples/data_import/csv/requirements.txt index 2dda90fec..697cffc6f 100644 --- a/python/examples/data_import/csv/simple/requirements.txt +++ b/python/examples/data_import/csv/requirements.txt @@ -1,2 +1,2 @@ python-dotenv -sift-stack-py +sift-stack-py \ No newline at end of file diff --git a/python/examples/data_import/csv/custom/sample_data.csv b/python/examples/data_import/csv/sample_data.csv similarity index 100% rename from python/examples/data_import/csv/custom/sample_data.csv rename to python/examples/data_import/csv/sample_data.csv diff --git a/python/examples/data_import/csv/simple/.env-example b/python/examples/data_import/csv/simple/.env-example deleted file mode 100644 index cdef5f890..000000000 --- a/python/examples/data_import/csv/simple/.env-example +++ /dev/null @@ -1,3 +0,0 @@ -SIFT_API_URI="" -SIFT_API_KEY="" -ASSET_NAME="" \ No newline at end of file diff --git a/python/examples/data_import/csv/simple/main.py b/python/examples/data_import/csv/simple/main.py deleted file mode 100644 index 1ab8351ef..000000000 --- a/python/examples/data_import/csv/simple/main.py +++ /dev/null @@ -1,42 +0,0 @@ -import os - -from dotenv import load_dotenv -from sift_py.data_import.csv import CsvUploadService -from sift_py.data_import.status import DataImportService -from sift_py.rest import SiftRestConfig - -if __name__ == "__main__": - """ - Example of uploading a CSV file into Sift using default CSV config. - """ - - load_dotenv() - - sift_uri = os.getenv("SIFT_API_URI") - assert sift_uri, "expected 'SIFT_API_URI' environment variable to be set" - - apikey = os.getenv("SIFT_API_KEY") - assert apikey, "expected 'SIFT_API_KEY' environment variable to be set" - - asset_name = os.getenv("ASSET_NAME") - assert asset_name, "expected 'ASSET_NAME' environment variable to be set" - - rest_config: SiftRestConfig = { - "uri": sift_uri, - "apikey": apikey, - } - - csv_upload_service = CsvUploadService(rest_config) - - # Can optionally specify units_row=N or description_row=N if these rows exist. - # Must also specify first_data_row=N in this case. - import_service: DataImportService = csv_upload_service.simple_upload( - asset_name, - "sample_data.csv", - ) - data_import = import_service.get_data_import() - print(data_import.model_dump_json(indent=1)) - - print("Waiting for upload to complete...") - import_service.wait_until_complete() - print("Upload example complete!") diff --git a/python/examples/data_import/csv/simple/sample_data.csv b/python/examples/data_import/csv/simple/sample_data.csv deleted file mode 100644 index 974bffb9a..000000000 --- a/python/examples/data_import/csv/simple/sample_data.csv +++ /dev/null @@ -1,51 +0,0 @@ -timestamp,channel_0,channel_1,channel_2,channel_3,channel_4,channel_5,channel_6,channel_7,channel_8,channel_9 -2024-10-07 17:00:09.982126,0.9869788584872923,0.4321820341919653,0.5867135634469265,0.9613042704758855,0.8581117009916057,0.47931312587076513,0.08242174011901193,0.1933231289442503,0.35985209963106657,0.3541647897768103 -2024-10-07 17:00:10.002126,0.5701255316316417,0.5914707762677202,0.2562630025294298,0.2513389890039397,0.42158646662087185,0.3479905929531466,0.26458283424910256,0.4609703329809085,0.6421614421556726,0.8510388436200512 -2024-10-07 17:00:10.022126,0.49446373422349477,0.3195179734137701,0.7871899227553234,0.9344052236947964,0.672805707797897,0.5123445839142331,0.28222507345627657,0.06418497987230987,0.23737333108063496,0.27500526480430076 -2024-10-07 17:00:10.042126,0.6323411689241686,0.12129516635402504,0.9523423895236848,0.6884533600751157,0.7144189711378498,0.3981104884533361,0.7761816578087838,0.8901628333060857,0.30626343283413393,0.6011538466824089 -2024-10-07 17:00:10.062126,0.7105902314226873,0.5099079791743336,0.4802228469605496,0.10704676305717797,0.10138693932861131,0.2650078206727895,0.25211372664734555,0.5767357520495985,0.9286870825289508,0.25497721804082396 -2024-10-07 17:00:10.082126,0.5256323808689144,0.1736180220982083,0.6524881071381322,0.003042953818593541,0.6613972481385193,0.7100947908784161,0.7477606441382524,0.3912805619092947,0.8430490363583497,0.6194785712573339 -2024-10-07 17:00:10.102126,0.9693387841288329,0.903623556498963,0.5100034454529075,0.6407491036361468,0.8569231122512672,0.05056118054309999,0.44232019387397803,0.7469144554405716,0.9727410561790976,0.24048712876411005 -2024-10-07 17:00:10.122126,0.7955698564962665,0.7249446474764154,0.8870730273307796,0.8843604048911804,0.5195901521212246,0.05753648053992644,0.7481253948701445,0.04145468150297349,0.2757433105519955,0.8271431370209696 -2024-10-07 17:00:10.142126,0.44163115772117134,0.3290077544927732,0.2755718544807364,0.772133850506479,0.768188038982342,0.7265904485292926,0.6565012067425104,0.23367690620333514,0.029934154305872096,0.9695809128153028 -2024-10-07 17:00:10.162126,0.8156068002765097,0.29274451770409216,0.5421572486014887,0.13000757637350413,0.022321842369040334,0.8869017316875608,0.4233633405896129,0.9960102139371052,0.11212362102743312,0.32849721419760736 -2024-10-07 17:00:10.182126,0.5834782910613371,0.4146197648932102,0.5813947696667644,0.07047903300825009,0.6602636693291395,0.25604504320586363,0.23331372679399032,0.38001110441597374,0.30182669662997474,0.065460012927686 -2024-10-07 17:00:10.202126,0.4958858227610534,0.03335758086761054,0.18306127159920793,0.2873661888506289,0.3622892776923665,0.499305859523576,0.6819818039499034,0.7077342031985537,0.6166251879949645,0.396436674329553 -2024-10-07 17:00:10.222126,0.5522713559775752,0.059736261748141506,0.6671495700467506,0.28880193876063154,0.5949485419694917,0.15234371456520912,0.8820635740470701,0.2623615814355038,0.05939341425818523,0.33899151593770116 -2024-10-07 17:00:10.242126,0.2548536144767083,0.9446851850635495,0.2503283278262417,0.6603460241625815,0.9426257446009163,0.2335313814529668,0.11325626995463156,0.8561477524963562,0.5714950967235992,0.6208318132502507 -2024-10-07 17:00:10.262126,0.6483317839597722,0.9058103142522725,0.8245270488872087,0.5102024611674935,0.9927493027517336,0.03993821791474972,0.38976451460023287,0.16434407971163412,0.8862485490664651,0.5239478232944098 -2024-10-07 17:00:10.282126,0.8149201339397839,0.7137737976420994,0.5557818190695873,0.4938338815400628,0.415033367396499,0.7035967227531553,0.2799010551744173,0.11133675295610845,0.09614041182273869,0.6749010616348136 -2024-10-07 17:00:10.302126,0.966821236972675,0.6427478499039081,0.08793942901785445,0.983536883456765,0.08017419642633206,0.9628090059077687,0.2261439442554486,0.8616042317097121,0.29768680983311857,0.7099921172794093 -2024-10-07 17:00:10.322126,0.07616495133533874,0.8477361352992079,0.5124562051137337,0.3477102817431239,0.9803013846846018,0.11564621113417217,0.500394919401976,0.9532968441813576,0.6981640095296011,0.11464731475203482 -2024-10-07 17:00:10.342126,0.2816774196886087,0.6946350894468526,0.06032100021328923,0.08671742329512011,0.3703285502256528,0.23098242439121186,0.8024638729107021,0.7519069755902849,0.1567668195054993,0.6188475921954685 -2024-10-07 17:00:10.362126,0.3562879856158999,0.5821585057874376,0.5106635612521454,0.3801483655437152,0.9198726678600827,0.9418166961193987,0.388429410377256,0.2977471804131373,0.9471563180632896,0.14325457954021137 -2024-10-07 17:00:10.382126,0.2567504742394161,0.06216349913585972,0.06597933896982477,0.9929001020720042,0.04246495074234258,0.059639438947922785,0.35082423391626916,0.49816120975354294,0.4793436374437493,0.9044262977225134 -2024-10-07 17:00:10.402126,0.729668044188631,0.87303573486452,0.28241255331422177,0.9844784607945438,0.5628283312476997,0.7772705514326647,0.5621228460418618,0.31655274740604544,0.14667119076481394,0.46479306538035026 -2024-10-07 17:00:10.422126,0.6079166221523755,0.3381935261303928,0.4027552753614071,0.9463221601878666,0.3079585365095189,0.7384939769719963,0.3068195207236858,0.9170954845496502,0.5033755202586822,0.1866483806338577 -2024-10-07 17:00:10.442126,0.6374130109345892,0.18034567217625985,0.3434712128818532,0.17021073481209192,0.3008496813095347,0.9906288845050785,0.1767094160069448,0.10490367522653032,0.451783284411753,0.28951700336392294 -2024-10-07 17:00:10.462126,0.6064278489558738,0.927735760571258,0.5420948784165588,0.6429208279586314,0.6491094970355998,0.4658083998707867,0.5023833964486385,0.31178928614967627,0.7371075546530739,0.20117118757702135 -2024-10-07 17:00:10.482126,0.4513829072880269,0.08750235625396385,0.897429061180492,0.525573337899755,0.4958912577189468,0.6666299042049784,0.6673821543792452,0.24940296325704814,0.39990346596891735,0.03908119199471982 -2024-10-07 17:00:10.502126,0.6526868292235004,0.2396467272906465,0.48876587520447246,0.9600364585017024,0.5645764863120679,0.6525672122039082,0.5892799462667705,0.2623733923630508,0.12644043893423196,0.557038851697632 -2024-10-07 17:00:10.522126,0.8206340728243324,0.3856078258875246,0.10355737982275359,0.40008217561542403,0.775489883125943,0.7809085279365546,0.9728982608591412,0.7437225300634555,0.9064050525860767,0.34954913722287684 -2024-10-07 17:00:10.542126,0.7183869527962506,0.6057076354942508,0.7141305521692453,0.864181892944079,0.3063329533214951,0.5922208449149451,0.2165961724433344,0.361412924700091,0.9195428793118164,0.9117659605065301 -2024-10-07 17:00:10.562126,0.5337432823497181,0.3029799047348265,0.7187998606002997,0.6413294998441611,0.06956711458021725,0.7653584390353513,0.5478444411319665,0.23956816537402514,0.565309371127512,0.09571756811408905 -2024-10-07 17:00:10.582126,0.9438967074410107,0.5947221959103646,0.36845583913223356,0.062139986030510186,0.4868199978665264,0.9933813175937465,0.5802919514297851,0.3290430584865637,0.10596102722542844,0.2548595173495307 -2024-10-07 17:00:10.602126,0.9934730684688866,0.8642964082869516,0.4976780006661251,0.28274284200988264,0.2873345110619544,0.5603090594354533,0.03671323561253215,0.2910425794795004,0.27643557935209917,0.9933617886999094 -2024-10-07 17:00:10.622126,0.7625363022038244,0.15457745224329023,0.14594004187301046,0.5333898421447607,0.06272457694883149,0.6695233127369234,0.11348515414578297,0.5372190720009107,0.8982742712845617,0.0735953901666454 -2024-10-07 17:00:10.642126,0.9752915088836254,0.4212118900250873,0.7517891300677948,0.6173271203730661,0.9484753022335238,0.49003318121474826,0.9673069773540193,0.7340492846172215,0.7752184647782351,0.3540611205129477 -2024-10-07 17:00:10.662126,0.8752711555400344,0.5802701554789678,0.5206563647840734,0.33822713547368,0.9940105706040785,0.22225489571851353,0.09547252426599151,0.6501010981395459,0.11054088308311194,0.9583261706090845 -2024-10-07 17:00:10.682126,0.07347363289278785,0.014646911613324765,0.7320190228788871,0.7709297572291158,0.15512255074438153,0.15390836335702818,0.5177227117538571,0.3236803212166608,0.8070567542605646,0.14637148735742855 -2024-10-07 17:00:10.702126,0.7705283572230599,0.10391905603399265,0.22647839339151277,0.19381828130242862,0.7394259865623566,0.537284895745162,0.665941669144839,3.682024595530464e-05,0.2948538006772726,0.5132436577867977 -2024-10-07 17:00:10.722126,0.5801002058755251,0.5347360901142872,0.9426863263468371,0.10078965364197234,0.18092649132383554,0.4158556131912532,0.48780894559491883,0.30729645738027744,0.8750348655618886,0.7104045822140087 -2024-10-07 17:00:10.742126,0.5643358961498655,0.7476536978422998,0.42184219720188965,0.5986913936301931,0.7995464506487457,0.47011356406382276,0.47165656214479945,0.640485625907709,0.14945533562104174,0.5078522070157255 -2024-10-07 17:00:10.762126,0.689416312998768,0.6869095715975265,0.44233170845065484,0.18918733339531457,0.05840017597923497,0.5475729301622566,0.8653724176218798,0.4307388257626793,0.02484138230351174,0.6041369641600293 -2024-10-07 17:00:10.782126,0.655117523072002,0.7744548323118985,0.12191113923871255,0.7580828943619093,0.5846162089198462,0.10163805596923403,0.9548748239914063,0.17047041185779388,0.3828186387151279,0.2125862018544078 -2024-10-07 17:00:10.802126,0.9978949894880911,0.4407389649883179,0.3874802461581107,0.787917288981986,0.46395079770224845,0.7076803126961722,0.021586569921456245,0.5047605761660581,0.11754763946252733,0.6914425169805245 -2024-10-07 17:00:10.822126,0.18230805546987583,0.7162526432491131,0.21515641083754522,0.011646521130654008,0.28184867409261694,0.9899533951647436,0.8802442501936819,0.15601023898717636,0.43165659645721866,0.8624006609001295 -2024-10-07 17:00:10.842126,0.08772137639860766,0.023610987792976545,0.18152206111415536,0.38963482785942316,0.9412403573232293,0.4134256244939435,0.7480337877841542,0.7004645831294896,0.6068471493001919,0.9784047958687754 -2024-10-07 17:00:10.862126,0.555431255352411,0.32821540235710556,0.8050463718214766,0.39059374606368025,0.901122696046085,0.7502017205918795,0.9967762731875593,0.46418284947619637,0.2502871364121475,0.8883270975329431 -2024-10-07 17:00:10.882126,0.8628168741632191,0.0691464712374249,0.2338115694969456,0.20339096239946164,0.31131589189871856,0.8803184037865743,0.6050595482347587,0.9791982321603043,0.7768227079004556,0.17412397358921428 -2024-10-07 17:00:10.902126,0.7372826427025838,0.47780638494554384,0.9565364953062893,0.8023572786972596,0.697601265722501,0.2996954441916203,0.45187436183282126,0.5050166941863234,0.5971251947486449,0.2663036154763788 -2024-10-07 17:00:10.922126,0.2616462515215068,0.9801170295903668,0.9714067759861551,0.6171366258392272,0.20210955994064783,0.5821302248582718,0.21728119192057127,0.5659070515053762,0.33890253787639546,0.7847785601399289 -2024-10-07 17:00:10.942126,0.30765032091105926,0.3315540842714122,0.8583987435373389,0.8090398019035149,0.053143709883349044,0.6161159934280365,0.9085598110294536,0.22921848109316345,0.09698643931431572,0.9558913981896255 -2024-10-07 17:00:10.962126,0.29892134818962646,0.9746840156550289,0.9189029062113264,0.6774566374705723,0.6404015592597371,0.39860549421962166,0.36998704719579667,0.7438885993373414,0.7012269332301863,0.35944279411072455