csv_processing.py

import pandas as pd
import numpy as np


def update_file_with_fto(file1_path, file2_path, output_path):
    # read file1 and file2 into DataFrames
    df1 = pd.read_csv(file1_path)   # the IPUs data
    df2 = pd.read_csv(file2_path)   # the FTOs data

    # get number of rows in each DataFrame
    n_rows_IPU = df1.shape[0]
    n_rows_FTO = df2.shape[0]



    # list for the IPU durations taken from the IPUs data CSV
    ipu_durations = [0] * n_rows_IPU
    for index, value in df1["IPU_Duration"].items():
        ipu_durations[index] = value
    
    # list for the pre-FT IPU durations taken from the FTOs data CSV
    preFT_durations = [0] * n_rows_FTO
    for index, value in df2["Pre_FT_IPU_Dur"].items():
        preFT_durations[index] = value

    # list for the actual FTOs from the FTOs data CSV
    ftos = [0] * n_rows_FTO
    for index, value in df2["FTO"].items():
        ftos[index] = value

    # create another list to store the FTOs that will be matched
    postIPU_durations = [0] * n_rows_IPU



    # indexes for outer and inner lists
    index2 = 0
    index1 = 0

    # iterate through the lists (there are less FTOs than IPUs since there are less FTs)
    while index2 < len(preFT_durations) and index1 < len(ipu_durations):

        # search through the array of IPU durations
        while index1 < len(ipu_durations):

            # a pre-FT duration in the FTOs data has been matched with an IPU in the IPUs data
            if (ipu_durations[index1] == preFT_durations[index2]):

                # (eventually) put the FTO in the row where the IPU duration was matched in the IPUs data
                postIPU_durations[index1] = ftos[index2]

                index1 += 1
                break
            else:
                index1 += 1
            
        # now search for a match for the next pre-FT IPU duration
        index2 += 1

    # add the list of matched FTOs to the DataFrame for the IPUs data
    df1['Post_IPU_FT_Dur'] = pd.Series(postIPU_durations)

    

    # export the updated DataFrame to a new CSV file
    df1.to_csv(output_path, index=True)
    print(f"Processing complete. Updated file saved as '{output_path}'")


if __name__ == "__main__":
    file1 = "_IPUs_L2_Quiet.csv"                    # the IPUs data, here being the full L2 Quiet
    file2 = "_FTOs_L2_Quiet_PositiveFTOs.csv"       # the FTOs data, here being the full L2 Quiet
    output_file = "updated_IPUs_L2_Quiet.csv"       # copy of the above IPUs data with FTOs added where appropriate
    
    update_file_with_fto(file1, file2, output_file)