From ebcd91dd476a17aeeb6cbe87330dd6f428e28a97 Mon Sep 17 00:00:00 2001 From: Anthony Lombardi Date: Wed, 12 Apr 2023 14:12:38 -0400 Subject: [PATCH] ENH: Update protocol to accept heuristic method for optimization initialization This commit introduces a backward incompatible changes requiring an updated client. The "optimizeFrame" command is updated to require an additional field called "opt_init_heuristic" allowing to control the optimization process by setting which heuristic should be used for initialization. Accepted values are the following: - CURRENT_FRAME - PREVIOUS_FRAME - LINEAR_EXTRAPOLATION - SPLINE_INTERPOLATION Note that the hard-coded behavior in trackDialog corresponding to "previous frame" is removed from both the python and Matlab client. To support the transition of existing scripts using the python or Matlab client, the default value for "opt_init_heuristic" value is still set to "previous frame". Co-authored-by: Jean-Christophe Fillion-Robin --- .../socket-control-libraries/matlab.md | 4 +- .../socket-control-libraries/pyautoscoper.md | 14 ++++++- autoscoper/src/net/Socket.cpp | 6 ++- scripts/matlab/AutoscoperConnection.m | 20 +++++---- scripts/python/PyAutoscoper/connect.py | 41 +++++++++++++++---- .../python/examples/pyautoscoper-examples.py | 4 ++ 6 files changed, 70 insertions(+), 19 deletions(-) diff --git a/Documentation/socket-control-libraries/matlab.md b/Documentation/socket-control-libraries/matlab.md index aee6566f..0a73ea9e 100644 --- a/Documentation/socket-control-libraries/matlab.md +++ b/Documentation/socket-control-libraries/matlab.md @@ -209,7 +209,8 @@ conn.optimizeFrame( max_stall_itr, dframe, opt_method, - cf_model + cf_model, + opt_init_heuristic ) ``` @@ -225,6 +226,7 @@ All of these arguments are optional and have default values. The default values | dframe | 1 | The amount of frames to skip over. | | opt_method | 0 | The optimization method to use. 0 is Partial Swarm Optimization and 1 is Downhill Simplex. | | cf_model | 0 | The cost function model to use. 0 is NCC and 1 is Sum of Absolute Differences. | +| opt_init_heuristic | 1 | The heuristic used to initialize the optimization, 0 for current frame, 1 for previous frame, 2 linear extrapolation, 3 for spline interpolation. | ### Tracking Dialog diff --git a/Documentation/socket-control-libraries/pyautoscoper.md b/Documentation/socket-control-libraries/pyautoscoper.md index be566d3e..7e7a5015 100644 --- a/Documentation/socket-control-libraries/pyautoscoper.md +++ b/Documentation/socket-control-libraries/pyautoscoper.md @@ -115,7 +115,7 @@ There are two methods for optimizing the tracking data: * `optimizeFrame`: Optimizes the tracking data for a single frame. * `trackingDialog`: Automatically optimizes the tracking data for all given frames. -The `optimizeFrame` method takes ten arguments: +The `optimizeFrame` method takes eleven arguments: * `volume`: The volume index (0-indexed). * `frame`: The frame index (0-indexed). * `repeats`: The number of times to repeat the optimization. @@ -126,6 +126,12 @@ The `optimizeFrame` method takes ten arguments: * `dframe`: The amount of frames to skip backwards for the initial guess. * `opt_method`: The {const}`~PyAutoscoper.connect.OptimizationMethod` to use. * `cf_model` : The {const}`~PyAutoscoper.connect.CostFunction` to use for evaluating the optimization. +* `opt_init_heuristic`: The {const}`~PyAutoscoper.connect.OptimizationInitializationHeuristic`. + +:::{versionadded} 2 + +The `opt_init_heuristic` parameter. +::: ```{literalinclude} ../../scripts/python/examples/pyautoscoper-examples.py :language: python @@ -146,6 +152,12 @@ The `trackingDialog` method takes at least three arguments: * `max_stall_itr`: The maximum number of iterations to stall the optimization. Defaults to 25. * `opt_method`: The {const}`~PyAutoscoper.connect.OptimizationMethod` to use. Defaults to {const}`~PyAutoscoper.connect.OptimizationMethod.PARTICLE_SWARM_OPTIMIZATION`. * `cf_model` : The {const}`~PyAutoscoper.connect.CostFunction` to use for evaluating the optimization. Defaults to {const}`~PyAutoscoper.connect.CostFunction.NORMALIZED_CROSS_CORRELATION`. +* `opt_init_heuristic`: The {const}`~PyAutoscoper.connect.OptimizationInitializationHeuristic`. Default to {const}`~PyAutoscoper.connect.OptimizationInitializationHeuristic.PREVIOUS_FRAME`. + +:::{versionadded} 2 + +The `opt_init_heuristic` parameter. +::: ```{literalinclude} ../../scripts/python/examples/pyautoscoper-examples.py :language: python diff --git a/autoscoper/src/net/Socket.cpp b/autoscoper/src/net/Socket.cpp index 925bb261..087d5405 100644 --- a/autoscoper/src/net/Socket.cpp +++ b/autoscoper/src/net/Socket.cpp @@ -48,8 +48,9 @@ #include #include "filesystem_compat.hpp" #include +#include -#define AUTOSCOPER_SOCKET_VERSION 1 +#define AUTOSCOPER_SOCKET_VERSION 2 Socket::Socket(AutoscoperMainWindow* mainwindow, unsigned long long int listenPort) : m_mainwindow(mainwindow) { @@ -354,9 +355,12 @@ void Socket::handleMessage(QTcpSocket * connection, char* data, qint64 length) SocketReadValuePointerMacro(dframe, qint32); SocketReadValuePointerMacro(opt_method, qint32); SocketReadValuePointerMacro(cf_model, qint32); + SocketReadValuePointerMacro(opt_init_heuristic, qint32); std::cerr << "Running optimization from autoscoper for frame #" << *frame << std::endl; + m_mainwindow->getTracker()->trial()->guess = *opt_init_heuristic; + m_mainwindow->optimizeFrame(*volumeID, *frame, *dframe, *repeats, *opt_method, *max_iter, *min_limit, *max_limit, diff --git a/scripts/matlab/AutoscoperConnection.m b/scripts/matlab/AutoscoperConnection.m index c2e77f48..5428599e 100644 --- a/scripts/matlab/AutoscoperConnection.m +++ b/scripts/matlab/AutoscoperConnection.m @@ -304,7 +304,7 @@ function getImageCropped(obj, volNum, camera, frameNum) data = fread(obj.socket_descriptor, obj.socket_descriptor.BytesAvailable); end - function optimizeFrame(obj, volNum, frameNum, repeats, max_itr, min_lim, max_lim, max_stall_itr, dframe, opt_method, cf_model) + function optimizeFrame(obj, volNum, frameNum, repeats, max_itr, min_lim, max_lim, max_stall_itr, dframe, opt_method, cf_model, opt_init_heuristic) % Optimizes the given frame % only obj, volNum, and frame are required % all other parameters are optional @@ -319,6 +319,7 @@ function optimizeFrame(obj, volNum, frameNum, repeats, max_itr, min_lim, max_lim % dframe: The amount of frames to skip % opt_method: The optimization method to use, 0 for Particle Swarm, 1 for Downhill Simplex % cf_model: The cost function model to use, 0 for NCC (Bone Models), 1 for Sum of Absolute Differences (Implant Models) + % opt_init_heuristic: heuristic for setting the initial frame for the optimization. 0 for current frame, 1 for previous frame, 2 linear extrapolation, 3 for spline interpolation if nargin < 3 error('Not enough input arguments'); @@ -347,6 +348,9 @@ function optimizeFrame(obj, volNum, frameNum, repeats, max_itr, min_lim, max_lim if nargin < 11 cf_model = 0; end + if nargin < 12 + opt_init_heuristic = 0; + end fwrite(obj.socket_descriptor, [ ... 11 ... typecast(int32(volNum), 'uint8') ... @@ -359,6 +363,7 @@ function optimizeFrame(obj, volNum, frameNum, repeats, max_itr, min_lim, max_lim typecast(int32(dframe), 'uint8') ... typecast(int32(opt_method), 'uint8') ... typecast(int32(cf_model), 'uint8') ... + typecast(int32(opt_init_heuristic), 'uint8') ... ]); while obj.socket_descriptor.BytesAvailable == 0 pause(1); @@ -376,7 +381,7 @@ function saveFullDRR(obj) data = fread(obj.socket_descriptor, obj.socket_descriptor.BytesAvailable); end - function trackingDialog(obj, volNum, startframe, endframe, repeats, max_itr, min_lim, max_lim, max_stall_itr, dframe, opt_method, cf_model) + function trackingDialog(obj, volNum, startframe, endframe, repeats, max_itr, min_lim, max_lim, max_stall_itr, dframe, opt_method, cf_model, opt_init_heuristic) % Performs optimization on a range of frames % Only obj, volNum, startframe, and endframe are required % all other parameters are optional @@ -392,6 +397,7 @@ function trackingDialog(obj, volNum, startframe, endframe, repeats, max_itr, min % dframe: The amount of frames to skip % opt_method: The optimization method to use, 0 for Particle Swarm, 1 for Downhill Simplex % cf_model: The cost function model to use, 0 for NCC (Bone Models), 1 for Sum of Absolute Differences (Implant Models) + % opt_init_heuristic: heuristic for setting the initial frame for the optimization. 0 for current frame, 1 for previous frame, 2 linear extrapolation, 3 for spline interpolation if nargin < 4 error('Not enough input arguments'); @@ -420,13 +426,12 @@ function trackingDialog(obj, volNum, startframe, endframe, repeats, max_itr, min if nargin < 12 cf_model = 0; end + if nargin < 13 + opt_init_heuristic = 0; + end for i = startframe:endframe obj.setFrame(i); - if i ~= 0 - pose = obj.getPose(volNum, i - 1); - obj.setPose(volNum, i, pose); - end obj.optimizeFrame( ... volNum, ... i, ... @@ -437,7 +442,8 @@ function trackingDialog(obj, volNum, startframe, endframe, repeats, max_itr, min max_stall_itr, ... dframe, ... opt_method, ... - cf_model ... + cf_model, ... + opt_init_heuristic ... ); end end diff --git a/scripts/python/PyAutoscoper/connect.py b/scripts/python/PyAutoscoper/connect.py index c3ccd967..a58bf581 100644 --- a/scripts/python/PyAutoscoper/connect.py +++ b/scripts/python/PyAutoscoper/connect.py @@ -3,7 +3,7 @@ import struct from enum import Enum -EXPECTED_SERVER_VERSION = 1 +EXPECTED_SERVER_VERSION = 2 class CostFunction(Enum): @@ -13,6 +13,15 @@ class CostFunction(Enum): SUM_OF_ABSOLUTE_DIFFERENCES = 1 +class OptimizationInitializationHeuristic(Enum): + """Enum for the different optimization initialization heuristics available in PyAutoscoper.""" + + CURRENT_FRAME = 0 + PREVIOUS_FRAME = 1 + LINEAR_EXTRAPOLATION = 2 + SPLINE_INTERPOLATION = 3 + + class OptimizationMethod(Enum): """Enum for the different optimization methods available in PyAutoscoper.""" @@ -421,6 +430,7 @@ def optimizeFrame( dframe, opt_method, cf_model, + opt_init_heuristic, ): """ Optimize the pose of the volume at the specified frame. @@ -445,11 +455,20 @@ def optimizeFrame( :type opt_method: int or :const:`~OptimizationMethod` :param cf_model: The cost function to use. :const:`~CostFunction.NORMALIZED_CROSS_CORRELATION` for Bone Models, :const:`~CostFunction.SUM_OF_ABSOLUTE_DIFFERENCES` for Implant Models. :type cf_model: int or :const:`~CostFunction` + :param opt_init_heuristic: The heuristic to initialize the optimization. See :const:`~OptimizationInitializationHeuristic`. + :type opt_init_heuristic: int or :const:`~OptimizationInitializationHeuristic` :raises AutoscoperServerError: If the server fails to optimize the frame :raises AutoscoperConnectionError: If the connection to the server is lost :raises ValueError: If parameters accepting an enum value are incorrectly specified. + + .. versionadded:: 2 + + The `opt_init_heuristic` parameter. """ + if not isinstance(opt_init_heuristic, OptimizationInitializationHeuristic): + opt_init_heuristic = OptimizationInitializationHeuristic(opt_init_heuristic) + if not isinstance(cf_model, CostFunction): cf_model = CostFunction(cf_model) @@ -468,8 +487,9 @@ def optimizeFrame( float(max_lim), max_stall_itr, dframe, - opt_method, - cf_model, + opt_method.value, + cf_model.value, + opt_init_heuristic.value, ) def saveFullDRR(self): @@ -509,11 +529,10 @@ def trackingDialog( max_stall_itr=25, opt_method=OptimizationMethod.PARTICLE_SWARM_OPTIMIZATION, cf_model=CostFunction.NORMALIZED_CROSS_CORRELATION, + opt_init_heuristic=OptimizationInitializationHeuristic.PREVIOUS_FRAME, ): """ - Automatically tracks the volume accross the given frames. - - Currently using previous frame for intial guess. + Automatically tracks the volume across the given frames. :param volume: The id of the volume to be tracked :type volume: int @@ -537,18 +556,21 @@ def trackingDialog( :type opt_method: int or :const:`~OptimizationMethod` :param cf_model: The cost function to use. :const:`~CostFunction.NORMALIZED_CROSS_CORRELATION` for Bone Models, :const:`~CostFunction.SUM_OF_ABSOLUTE_DIFFERENCES` for Implant Models. :type cf_model: int or :const:`~CostFunction` + :param opt_init_heuristic: The heuristic to initialize the optimization. See :const:`~OptimizationInitializationHeuristic`. + :type opt_init_heuristic: int or :const:`~OptimizationInitializationHeuristic` :raises AutoscoperServerError: If the server fails to track the volume :raises AutoscoperConnectionError: If the connection to the server is lost :raises ValueError: If parameters accepting an enum value are incorrectly specified. + + .. versionadded:: 2 + + The `opt_init_heuristic` parameter. """ if self.verbose: print(f"Automated tracking of volume {volume} from frame {start_frame} to {end_frame}.\n") for frame in range(start_frame, end_frame): self.setFrame(frame=frame) - if frame != 0: - pose = self.getPose(volume=volume, frame=(frame - 1)) - self.setPose(volume=volume, frame=frame, pose=pose) self.optimizeFrame( volume=volume, frame=frame, @@ -560,6 +582,7 @@ def trackingDialog( dframe=frame_skip, opt_method=opt_method, cf_model=cf_model, + opt_init_heuristic=opt_init_heuristic, ) def getNumVolumes(self): diff --git a/scripts/python/examples/pyautoscoper-examples.py b/scripts/python/examples/pyautoscoper-examples.py index 7789cdc5..a92f5ca1 100644 --- a/scripts/python/examples/pyautoscoper-examples.py +++ b/scripts/python/examples/pyautoscoper-examples.py @@ -4,6 +4,7 @@ AutoscoperConnection, OptimizationMethod, CostFunction, + OptimizationInitializationHeuristic, ) autoscoperSocket = AutoscoperConnection() @@ -48,6 +49,7 @@ dframe=1, opt_method=OptimizationMethod.PARTICLE_SWARM_OPTIMIZATION, cf_model=CostFunction.NORMALIZED_CROSS_CORRELATION, + opt_init_heuristic=OptimizationInitializationHeuristic.PREVIOUS_FRAME, ) # [Example 5 - End] @@ -61,6 +63,7 @@ AutoscoperConnection, OptimizationMethod, CostFunction, + OptimizationInitializationHeuristic, ) # Create a socket connection to Autoscoper @@ -97,6 +100,7 @@ dframe=1, opt_method=OptimizationMethod.PARTICLE_SWARM_OPTIMIZATION, cf_model=CostFunction.NORMALIZED_CROSS_CORRELATION, + opt_init_heuristic=OptimizationInitializationHeuristic.CURRENT_FRAME, ) autoscoperSocket.saveTracking(volume, f"path/to/tracking_data_volume_{volume}_out.tra")