Automated_Alignment_Code.py

from output_logging import Logger, signal_handler
import objective_parameters
import gwl_handling
import event_handling
import take_screenshot
import circle_detection
import circle_filtering
import square_detection
import rotation_adjustment
import pyautogui
import time
import datetime
import numpy as np
import cv2
import signal
import sys
if '..' not in sys.path:
    sys.path.append('..')


def main(
         objective = '63x', DiLL = False,  # Specify the microscope objective being used as '63x', '25x', or '10x'. Set DiLL to True if printing in DiLL mode, set it to False if printing in oil immersion mode.
         angle = 31.317, Y_south = False,  # Angle Y-Axis value obtained from NanoWrite. Set Y_south to False if the alignment marker's +Y arrow points upward, set it to True if the +Y arrow points downward.
         jobs = 1,                         # If you have more than one unique job file to print, set jobs to the number of different job files you have, otherwise set jobs = 1.
         rows = 2, columns = 3,            # Number of rows and columns (set rows = 1 and columns = 1 if you only have one structure to print on).
         array_spacing_X = 150,            # Horizontal center-to-center distance (in microns) between the array positions.
         array_spacing_Y = 200,            # Vertical center-to-center distance (in microns) between the array positions.
         move_to_find_interface = 100,     # Distance (in microns) the stage moves away in the +Y direction from the printed structure(s) for interface finding.
         square_side_length = 40,          # Side length (in microns) of the square formed by the four alignment markers.
         detection_params = 'default'):    # Check circle_detection.py to look at the possible set of circle detection parameters (i.e. recipes) that can be used (the one to select depends on the resin and substrate you are printing with as well as the marker diameter).


    # Objective parameters are obtained depending on the objective being used.
    origin_X, origin_Y, focus_value, rough_alignment_dist, alignment_threshold, pixel_size = objective_parameters.main(objective)

    max_alignment_move = rough_alignment_dist  # The maximum distance (in microns) the stage is allowed to move when aligning the print origin with the square's center.
    alignment_iterations = 5                   # Maximum number of times the program is allowed to reposition the stage for alignment before printing.
    job_number = 1                             # A counter is set so that the program knows which job file to print next if there are multiple jobs. This value can also be changed to let the program know which job file to start printing at (ex. set it to 5 if you want it to begin the print sequence at jobfile5.gwl).
    alignment_error_flag = False               # A boolean variable that is set to True if an error is encountered during alignment. Lets the program know if it should skip the current array position.

    program_parameters = np.array([['objective','DiLL','angle','Y_south','jobs','rows','columns','array_spacing_X',
        'array_spacing_Y','move_to_find_interface','square_side_length','detection_params','origin_X','origin_Y',
        'focus_value','rough_alignment_dist','alignment_threshold'],
        [objective,DiLL,angle,Y_south,jobs,rows,columns,array_spacing_X,array_spacing_Y,move_to_find_interface,
        square_side_length,detection_params,origin_X,origin_Y,focus_value,rough_alignment_dist,alignment_threshold]]).T

    print('Parameters used:')
    for name, value in program_parameters:  # Prints out the parameters used for the program.
        print(f'{name} = {value}')
    print('\n')

    if DiLL:  # If printing in DiLL mode, stage movements in Y are reversed. Hence, array_spacing_Y and move_to_find_interface are inverted such that the stage movement is correct.
        array_spacing_Y = -array_spacing_Y
        move_to_find_interface = -move_to_find_interface

    for row in range(rows):
        # # #
        # Stage Movement Between Printed Structures
        # # #
        if row != 0:  # The stage will move to the next row (unless it is on the first row).
            if columns != 1:  # If there is only one column, the stage will not move in X when moving to the next row.
                array_move_X = gwl_handling.GwlFile(f'./exchange/move_X.gwl')  # An empty .gwl file is initialized in the system's memory, whose properties are the filename and path.
                array_move_X.add_movement('X',-array_spacing_X * (columns - 1))  # Adds the MoveStageX command to the gwl file's data in memory.
                array_move_X.write_to_file()  # Creates the .gwl file with the added command(s) in the "exchange" folder.
                array_move_X.start_job()  # Renames the .gwl file to have _new.gwl at the end, (the renaming event triggers ServerMode to execute the .gwl file).
                event_handling.wait_for_job()  # Pauses the program until the .gwl file finishes executing (ServerMode creates a .log file to signal that the .gwl file finished executing).

            array_move_Y = gwl_handling.GwlFile(f'./exchange/move_Y.gwl')
            array_move_Y.add_movement('Y', -array_spacing_Y)
            array_move_Y.write_to_file()
            array_move_Y.start_job()
            event_handling.wait_for_job()

        for col in range(columns):
            if col != 0:  # If the stage is not on the first column, it will move the stage to next column by moving in the +X direction.
                next_column = gwl_handling.GwlFile(f'./exchange/next_column.gwl')
                next_column.add_movement('X', array_spacing_X)
                next_column.write_to_file()
                next_column.start_job()
                event_handling.wait_for_job()

            position = f'X{row + 1}_Y{col + 1}'  # Creates a string to contain the array position, which will be used to name the screenshots saved.
            print(f'Row number: {row + 1}, Column number: {col + 1}')  # Prints out the array position that is currently in view.

            # # #
            # Interface Finding
            # # #
            # The stage first moves in the +Y direction, finds the interface, adjusts the focus (if needed), then moves in the -Y direction back to the structure(s).
            # Interface finding is performed away from pre-existing structures as they may reduce the accuracy of the interface finding.

            find_interface = gwl_handling.GwlFile(f'./exchange/find_interface_away.gwl')
            find_interface.add_movement('Y', move_to_find_interface)
            find_interface.interface_finding(findInterfaceAt=0)
            if focus_value != 0:
                find_interface.adjust_focus(AddZDrivePosition=focus_value)
            find_interface.add_movement('Y', -move_to_find_interface)
            find_interface.write_to_file()  # Creates the .gwl file containing the MoveStageY commands, FindInterfaceAt command, and AddZDrivePosition command.
            find_interface.start_job()
            event_handling.wait_for_job()

            # # #
            # Circle Detection and Locating the Square's Center
            # # #
            alignment_time_start = time.time()  # Will be used to calculate how long the program took to align the stage sufficiently.
            for iteration in range(alignment_iterations):  # This for loop will repeat for a maximum of the specified number of alignment iterations (i.e. if set to 5, the loop will iterate for a maximum of 5 times unless a break command is reached).
                position_i = position + '_' + str(iteration+1)  # Sets a string to contain the array position along with the alignment iteration; ex. X2_Y3_1
                print(f'Current alignment iteration for {position}: {iteration+1}')  # Prints out the iteration the alignment is currently on.

                # Circle Detection
                take_screenshot.capture_screenshot(f'{position_i}.tif')  # Takes a screenshot of the display and saves it.
                take_screenshot.crop_screenshot(f'{position_i}.tif')     # Crops the screenshot to only show the microscope view.
                take_screenshot.resize_screenshot(f'{position_i}.tif')   # Resizes the screenshot to match the microscope camera's resolution.

                img = cv2.imread(f'./exchange/{position_i}.tif', 0)  # Loads the cropped screenshot in memory as a grayscale image.
                cv2.imwrite(f'./exchange/{position_i}_raw.tif',img)  # Saves the image with '_raw' appended to the original filename. This image is a raw screenshot of the microscope view.

                detected_circles = circle_detection.main(position_i, detection_params)  # detected_circles is an array that contains the (x,y) coordinates for each detected circle's center and their radii (units are in pixels).

                try:
                    count = detected_circles[0].shape[0]  # count contains the number of circles detected.
                except TypeError:  # If an error was obtained in the previous line (i.e. when detected_circles has no elements), count is set to 0.
                    count = 0
                if count < 4:  # If the number of circles detected is less than 4, the program prints an error message and skips the current array position and moves to the next one.
                    print(f'\033[0;49;91mLess than 4 circles detected. Aborting alignment for {position}.\033[m \n')
                    alignment_error_flag = True
                    break

                # Drawing of Detected Circles on the Screenshot
                img = cv2.imread(f'./exchange/{position_i}.tif', 1)  # Opens the screenshot image as non-grayscale for drawing in color.
                circles_for_drawing = np.uint16(np.around(detected_circles))  # Before drawing the circles, the x,y,r values of detected_circles are rounded to the nearest integer and then converted to unsigned 16-bit integers.
                for (x, y, r) in circles_for_drawing[0, :]:
                    cv2.circle(img, (x, y), r, (0, 0, 0), 1)  # Draws the outline of the circle with radius r.
                    cv2.circle(img, (x, y), 2, (0, 0, 0), 3)  # Draws the center of the circle as a smaller circle/dot.

                cv2.imwrite(f'./exchange/{position_i}.tif', img)  # The drawn circles are saved to the same screenshot that was opened.
                cv2.imwrite(f'./exchange/{position_i}_circles.tif', img)  # The drawn circles are saved to a new screenshot.
                detected_circles = np.delete(detected_circles[0],2,1)  # Clears the third column of the detected_circles array, which contained the radii of the detected circles.

                # Circle Filtering
                filtered_circles, outer_ring_radius, inner_ring_radius = circle_filtering.main(detected_circles, origin_X, origin_Y, square_side_length, max_alignment_move, pixel_size)  # The detected_circles array is filtered to only contain the coordinates of detected circles that are a specific distance away from the print origin.

                if filtered_circles.shape[0] < 4:  # If the number of circles detected after filtering is less than 4, the program prints an error message and skips the current array position and moves to the next one.
                    print(f'\033[0;49;91mLess than 4 circles remained after filtering. Aborting alignment for {position}.\033[m \n')
                    alignment_error_flag = True
                    break

                # Drawing of Filtered Circles on the Screenshot
                [cv2.circle(img, (x, y), 2, (0, 255, 255), 3) for x, y in filtered_circles[:, :2]]  # The center of each circle in the filtered_circles array is marked by a small yellow circle/dot.

                # Draws two yellow circles that represent the upper and lower boundary used to filter the detected circles.
                #cv2.circle(img, (origin_X, origin_Y), int(outer_ring_radius), (0, 255, 255), 1)
                #cv2.circle(img, (origin_X, origin_Y), int(inner_ring_radius), (0, 255, 255), 1)

                # Square Detection
                squares = square_detection.main(filtered_circles, square_side_length, objective, pixel_size)  # Identifies the coordinates of the four alignment markers, which form a square.

                # Square Ranking
                if squares.shape[0] == 0:  # If no squares are found, the program prints an error message and skips the current array position and moves to the next one.
                    print(f'\033[0;49;91mNo suitable squares found. Aborting alignment for {position}.\033[m \n')
                    alignment_error_flag = True
                    break

                elif squares.shape[0] == 1:  # If only one square was found, square ranking is skipped.
                    squares_ranked = squares

                else:  # If multiple squares were found, the squares array is sorted based on how far away each square's center is from the print origin.
                    squares = np.hstack((squares, np.zeros((squares.shape[0], 1), dtype=squares.dtype)))  # Adds a ninth column to the squares array.
                    for s in range(squares.shape[0]):
                        # Calculates the X and Y coordinates (in pixels) of each square's center by averaging the coordinates of the four corners.
                        square_center_X = sum(squares[s][i] for i in range(0, 8, 2)) / 4
                        square_center_Y = sum(squares[s][i] for i in range(1, 8, 2)) / 4
                        squared_dist = (square_center_X - origin_X) ** 2 + (square_center_Y - origin_Y) ** 2  # Calculates the squared distance of each square's center to the print origin.
                        squares[s][8] = squared_dist  # Adds squared_dist to the 9th column.
                    squares_ranked = squares[np.argsort(squares[:, 8])]  # Sorts the squares array in ascending order based on the squared distance.

                # Calculates the X and Y coordinates (in pixels) of the highest ranked square's center by averaging the coordinates of the four corners.
                X_computed = sum(squares_ranked[0][i] for i in range(0, 8, 2)) / 4
                Y_computed = sum(squares_ranked[0][i] for i in range(1, 8, 2)) / 4

                print(f'Square center located at X = {X_computed} px, Y = {Y_computed} px')  # Prints out the computed pixel coordinates of the initial print's center.

                # Drawing of Detected Square(s) on the Screenshot
                for i in range(squares_ranked.shape[0]):  # The square with the closest distance to the print origin will be shown in red and its center will be shown as a red dot. The other squares not filtered out in the initial step will be shown in increasingly blueish colors.
                    pts = np.array([[squares_ranked[i][j], squares_ranked[i][j + 1]] for j in range(0, 8, 2)], np.int32)  # An array containing the X-Y coordinates of the four points of the square in the i-th row is created.
                    pts = pts.reshape((-1, 1, 2))  # Reshapes the pts array to the shape required by the cv2.polylines function.

                    if i > 0:  # All squares except the first one are colored blue, with the squared being colored less blue the farther away the lower ranked they are.
                        red, green, blue = 255 - (255 * i / (squares_ranked.shape[0] - 1)), 0, 255
                    else:  # The first square in the pts array (which is the square formed by the alignment markers) is colored red.
                        red, green, blue = 255, 0, 0

                    cv2.polylines(img, [pts], True,(int(blue), int(green), int(red)))  # Draws the squares on the screenshot taken.

                cv2.circle(img, (int(X_computed), int(Y_computed)), 2, (0, 0, 255), 3)  # Draws a red dot that shows the location of the initial print's center.
                cv2.circle(img, (int(origin_X), int(origin_Y)), 2, (255, 255, 255), 3)  # Draws a white dot that shows the location of the print origin.
                cv2.imwrite(f'./exchange/{position_i}.tif', img)  # Overwrites the opened screenshot image. The screenshot now includes the drawn square(s), along with the location of the initial print's center and the print origin.

                # # #
                # Stage Movement for Alignment Correction
                # # #
                if DiLL:
                    displacement_px_X = X_computed - origin_X  # The calculated stage movement in the X-direction (in pixels) is equal to the square's center X-coordinate minus the print origin's X-coordinate.
                    displacement_px_Y = origin_Y - Y_computed  # The calculated stage movement in the Y-direction (in pixels) is equal to the print origin's Y-coordinate minus the square's center Y-coordinate.
                else:  # In oil immersion mode, the X-axis appears flipped in the microscope view, so the displacement calculation is flipped to take this into account.
                    displacement_px_X = origin_X - X_computed
                    displacement_px_Y = origin_Y - Y_computed

                # Converts the X and Y displacements from pixels into microns.
                displacement_um_X = float(displacement_px_X / pixel_size)
                displacement_um_Y = float(displacement_px_Y / pixel_size)

                displacement_um_X_new, displacement_um_Y_new = rotation_adjustment.main(displacement_um_X, displacement_um_Y, angle, Y_south, DiLL)  # A rotation matrix is used to convert the calculated displacements from the screenshot into displacements based on the rotation of the virtual coordinate system.

                print(f'Calculated displacement for {position_i}: {displacement_um_X_new} um in X, {displacement_um_Y_new} um in Y')  # Prints out the distance (in microns) the stage will move in X and Y.

                if displacement_um_X_new ** 2  + displacement_um_Y_new ** 2 < alignment_threshold ** 2:  # If the separation between the initial print's center and the print origin is less than the specified alignment threshold, the program breaks the alignment loop and proceeds to the next step of the program.
                    print(f'\033[0;49;92mCalculated displacement is less than {alignment_threshold} um. {position} is aligned sufficiently.\033[m')  # Prints out that the displacement in X and Y is less than 'alignment_threshold' (in bright green color).
                    print(f'Alignment took {round(time.time() - alignment_time_start, 1)} seconds.')  # Prints out how long it took to align the current array position sufficiently.
                    take_screenshot.screenshot_aligned(position)  # Takes a screenshot of the microscope view after alignment and saves the image.

                    if focus_value != 0:  # Readjusts the focus back to the interface position if focus_value is nonzero.
                        focus = gwl_handling.GwlFile(f'./exchange/change_focus.gwl')
                        focus.adjust_focus(AddZDrivePosition=-focus_value)  # Moves the microscope Z-drive back to the interface position.
                        focus.write_to_file()
                        focus.start_job()
                        event_handling.wait_for_job()

                    break

                if displacement_um_X_new ** 2  + displacement_um_Y_new ** 2 > max_alignment_move ** 2:  # If the program wants to move the stage by more than the specified maximum distance it is allowed to move, it breaks the alignment loop and moves to the next array position.
                    print(f'\033[0;49;91mCalculated displacement exceeds {max_alignment_move} um. Aborting alignment for {position}.\033[m \n')  # Prints out that the calculated displacement exceeds the maximum that the program is allowed to move.
                    alignment_error_flag = True
                    break

                if not alignment_error_flag:  # The program will not move the stage for alignment if the alignment_error_flag was raised.
                    aligner = gwl_handling.GwlFile(f'./exchange/alignment.gwl')  # An empty .gwl file is initialized in the system's memory, whose properties are the filename and path.
                    aligner.add_movement('X', displacement_um_X_new)  # Adds the calculated displacement in X, which will be written to the .gwl file in the format "MoveStageX displacement_um_X_new".
                    aligner.add_movement('Y', displacement_um_Y_new)  # "MoveStageY displacement_um_Y_new"
                    aligner.write_to_file()  # Creates alignment.gwl, which has the MoveStageX and MoveStageY commands.
                    aligner.start_job()  # Executes the .gwl, which will move the stage along X and then along Y.
                    event_handling.wait_for_job()

            # # #
            # Addition of Scan Field Offsets and Print Job Execution
            # # #
            if jobs > 1:  # If multiple jobs are to be printed, it sets job_name to be jobfile{job_number} (ex. jobfile1, jobfile2, ...).
                job_name = f'jobfile{job_number}'

                #if not alignment_error_flag:  # Uncomment this if statement and indent the line below if you want the program to not skip over the current job number for the next print if an alignment error is encountered.
                job_number += 1  # Increments job_number by 1. NOTE: If the alignment_error_flag is True, the job number will still increment and so the program will skip over the job file it was supposed to print at that position. To disable this behavior, uncomment the if statement in the previous line and unindent this line.

                if job_number > jobs:  # If job_number is greater than the number of jobs, job_number is set back to 1.
                    job_number = 1  # So if the number of rows multiplied by the number of columns is greater than the number of job files, the program will reset the counter and start printing at jobfile1.gwl

            else:  # If only one job file is to be printed (i.e. when multiple_jobs is set as False), job_name is set as 'jobfile'.
                job_name = 'jobfile'

            if not alignment_error_flag:  # The program will not add scan field offsets to the job file or print the job file if the alignment_error_flag was raised.
                print_job = gwl_handling.GwlFile(f'./exchange/{job_name}.gwl')
                print_job.write_offsets_to_file('XOffset', 'YOffset', f'XOffset {displacement_um_X_new}', f'YOffset {displacement_um_Y_new}')  # Opens the job file and replaces XOffset 0 and YOffset 0 such that the X and Y displacements are set as the offsets.
                time.sleep(0.25)  # The program pauses to ensure the job file has enough time to be edited before executing it.

                print(f'Printing {job_name}.gwl on {position}.\n')  # The program writes out the job file being printed and the position being printing on.
                print_job.start_job()  # Executes the job file.
                event_handling.wait_for_job()  # Pauses the program until the job file finishes printing.

                take_screenshot.screenshot_finished(position)  # Takes a screenshot and saves it after the print job finishes.

                #max_alignment_move = 5  # If the first alignment and print was successful, max_alignment_move is set to 5 microns for subsequent prints.

            alignment_error_flag = False  # Lowers the flag (in case it was raised) for the next alignment.


if __name__ == "__main__":

    sys.stdout = Logger()  # Starts logging the python console output to a .txt file with the date and time.
    signal.signal(signal.SIGINT, signal_handler)  # Saves the console output to the .txt file even if the program was aborted (does not work if the program is aborted while it was waiting for a .gwl file to finish executing).

    start_time = datetime.datetime.now()  # Logs the time the program is executed.
    main()  # Executes the main automated alignment code.
    end_time = datetime.datetime.now()  # Logs the time the program finishes executing.

    print(f'\nAutomated_Alignment.py has finished.\nTime taken: {end_time - start_time}')  # Prints out the total time taken by the program.
    sys.stdout.flush()  # Stops the logging and saves the .txt file with the console output.

    pyautogui.alert(text='Automated_Alignment.py has finished.', title='Automated_Alignment.py', button='OK')  # Displays a message box stating that the program finished executing.