Merge pull request #66 from YttriLab/dev_3d

Updated In-App Documentation and OpenMonkeyStudio support

New features:
- Added extended in-app documentation for each step
- Added OpenMonkeyStudio support
- Added 3D Feature extraction

Errors:
- Caught some errors when starting but not finishing Refinement

GUI:
- Recovered Feature extraction step in sidebar to avoid confusion when uploading data and auto refresh is missing
- Updated Menu step 5 to show correct image

CleanUp:
- restructured feature extraction code to split functions from extractor class (some redundancy remains)
- split prediction from feature extraction
- started to remove old code

asoid/__init__.py

@@ -1,3 +1,3 @@
 #setting up the version number in here for setup.py to read and use in asoid.__version__
-__version__ = "0.3"
+__version__ = "0.3.1"
 

asoid/app.py

@@ -33,7 +33,8 @@ def index():
     step2_fname = HERE.joinpath("images/feature_extraction_wht.png")
     step3_fname = HERE.joinpath("images/baseline_classifier_wht.png")
     step4_fname = HERE.joinpath("images/active_learning_schematic.png")
-    step5_fname = HERE.joinpath("images/app_discovery.png")
+    step5_fname = HERE.joinpath("images/asoid_logo.png")
+    step6_fname = HERE.joinpath("images/app_discovery.png")
 
     st.markdown(f" <h1 style='text-align: left; color: #f6386d; font-size:30px; "
                 f"font-family:Avenir; font-weight:normal'>Welcome to A-SOiD</h1> "
@@ -123,14 +124,14 @@ def index():
 
     elif selected_step == 'Step 5':
         colL.markdown(f" <h1 style='text-align: left; color: #FFFFFF; font-size:18px; "
-                      f"font-family:Avenir; font-weight:normal'> Step 5: Discover subtle differences within behavior"
+                      f"font-family:Avenir; font-weight:normal'> Step 5: Predict behavior on new data"
                       f"", unsafe_allow_html=True)
         colL.markdown(f" <h1 style='text-align: left; color: #FFFFFF; font-size:18px; "
                       f"font-family:Avenir; font-weight:normal'> In this step,"
-                      f" you can run unsupervised learning on a particular behavior to get "
-                      f"segmented behaviors."
+                      f" you can use the trained classifier to predict the behavior of new data"
+                      f"and visualize some core statistics."
                       f"", unsafe_allow_html=True)
-        colR.markdown("<p style='text-align: right; color: grey; '>" + img_to_html(step5_fname, width=350) + "</p>",
+        colR.markdown("<p style='text-align: right; color: grey; '>" + img_to_html(step5_fname, width=200) + "</p>",
                       unsafe_allow_html=True)
 
     elif selected_step == 'Step 6':
@@ -142,7 +143,7 @@ def index():
                       f" you can run unsupervised learning on a particular behavior to get "
                       f"segmented behaviors."
                       f"", unsafe_allow_html=True)
-        colR.markdown("<p style='text-align: right; color: grey; '>" + img_to_html(step5_fname, width=350) + "</p>",
+        colR.markdown("<p style='text-align: right; color: grey; '>" + img_to_html(step6_fname, width=350) + "</p>",
                       unsafe_allow_html=True)
 
     bottom_cont = st.container()
@@ -192,6 +193,8 @@ def main():
         st.session_state['page'] = 'Step 1'
     if 'config' not in st.session_state:
         st.session_state['config'] = None
+    if "project_name" not in st.session_state:
+        st.session_state["project_name"] = None
 
     # if in main menu, display applications, see above index for item layout
     with st.sidebar:
@@ -209,8 +212,10 @@ def main():
                     project_config.optionxform = str
                     project_config.read_file(stringio)
                     st.session_state['config'] = project_config
+                    st.session_state["project_name"] = project_config["Project"].get("PROJECT_NAME")
                     st.rerun()
             elif st.session_state['config'] is not None:
+                st.info(f"Project :green[{st.session_state['project_name']}] loaded.")
                 cleared = st.form_submit_button(":red[Delete]")
                 if cleared:
                     st.session_state['config'] = None
@@ -226,7 +231,7 @@ def main():
             prefix = st.session_state['config']["Project"].get("PROJECT_NAME")
             data, config = load_data(working_dir,
                                      prefix)
-            menu_options = ['Menu', 'Upload Data', 'Extract Features', 'Active Learning',
+            menu_options = ['Menu', 'View Config', 'Extract Features', 'Active Learning',
                             'Refine Behaviors', 'Create New Dataset', 'Predict', 'Discover']
             icon_options = ['window-desktop',
                             'upload',
@@ -239,11 +244,17 @@ def main():
                             ]
 
         except:
-            menu_options = ['Menu', 'Upload Data', 'Active Learning',
-                            'Refine Behaviors', 'Create New Dataset', 'Predict', 'Discover']
+            menu_options = ['Menu', 'Upload Data'
+                , 'Extract Features'
+                 , 'Active Learning'
+                 ,'Refine Behaviors'
+                 , 'Create New Dataset'
+                 , 'Predict'
+                 , 'Discover'
+                            ]
             icon_options = ['window-desktop',
                             'upload',
-
+                            'bar-chart-line',
                             'diagram-2',
                             'images',
                             'file-earmark-plus',
@@ -276,7 +287,7 @@ def main():
 
     if nav_options == 'Menu':
         index()
-    elif 'Upload Data' in nav_options:
+    elif 'Upload Data' in nav_options or 'View Config' in nav_options:
         if "config" in st.session_state.keys():
             A_data_preprocess.main(config=st.session_state['config'])
         else:

asoid/apps/A_data_preprocess.py

@@ -5,13 +5,31 @@
 
 
 TITLE = "Preprocess data"
-
+PREPROCESS_HELP = ("In this step, you will preprocess the data before extracting features. "
+                   "\n\n The data will be used to train the classifier and predict the behavior in the next steps."
+                   "\n\n---\n\n"
+                   "**Step 1**: Create a new project or upload an existing project configuration."
+
+                   "\n\n **Step 2**: Select a pose estimation origin."
+                   "\n\n **Step 3**: Upload pose files."
+                     "\n\n **Step 4**: Upload annotation files."
+                     "\n\n **Step 5**: Match pose and annotation files in the same order."
+                        "\n\n **Step 6**: Set the config parameters."
+                     "\n\n **Step 7**: Create the project and preprocess the data."
+                   "\n\n **Step 8**: Continue with :orange[Extract Features]."
+                   "\n\n---\n\n"
+                   ":red[Using the same prefix and working directory will result in an overwrite and might cause unintended problems.]"
+                   )
 
 def main(config=None):
     st.markdown("""---""")
+
     if config:
+        st.title("View Project Configuration")
         view_config_md(config)
     else:
+        st.title("Create project and upload data")
+        st.expander("What is this?", expanded=False).markdown(PREPROCESS_HELP)
         processor = Preprocess()
         processor.main()
     st.session_state['page'] = 'Step 2'

asoid/apps/B_extract_features.py

@@ -8,6 +8,17 @@
 
 TITLE = "Extract features"
 
+EXTRACT_FEATURES_HELP = ("In this step, you will extract features from the labeled data you uploaded. "
+                         "\n\n The features will be used to train the classifier and predict the behavior in the next steps."
+                         "\n\n---\n\n"
+                            "**Step 1**: Upload your project config file."
+                            "\n\n **Step 2**: Set the parameters."
+                            "\n\n **Step 3**: Extract the features."
+                            "\n\n **Step 4**: Continue with :orange[Active Learning]."
+                            "\n\n---\n\n"
+                         ":blue[Feature extraction can be repeated but requires new training afterwards.]"
+                         )
+
 
 def prompt_setup(prompt_container, software, framerate, annotation_classes,
                  working_dir, prefix, show_only=False):
@@ -60,13 +71,17 @@ def prompt_setup(prompt_container, software, framerate, annotation_classes,
 def main(config=None):
     st.markdown("""---""")
 
+    st.title("Extract Features")
+    st.expander("What is this?", expanded=False).markdown(EXTRACT_FEATURES_HELP)
+
     if config is not None:
         working_dir = config["Project"].get("PROJECT_PATH")
         prefix = config["Project"].get("PROJECT_NAME")
         annotation_classes = [x.strip() for x in config["Project"].get("CLASSES").split(",")]
         software = config["Project"].get("PROJECT_TYPE")
         framerate = config["Project"].getfloat("FRAMERATE")
         iteration = config["Processing"].getint("ITERATION")
+        is_3d = config["Project"].getboolean("IS_3D")
         project_dir = os.path.join(working_dir, prefix)
         iter_folder = str.join('', ('iteration-', str(iteration)))
         os.makedirs(os.path.join(project_dir, iter_folder), exist_ok=True)
@@ -86,15 +101,15 @@ def main(config=None):
                     prompt_setup(prompt_container, software, framerate, annotation_classes,
                                  working_dir, prefix)
                 if st.button('Extract Features', help = EXTRACT_FEATURES_HELP):
-                    extractor = Extract(working_dir, prefix, frames2integ)
+                    extractor = Extract(working_dir, prefix, frames2integ, is_3d)
                     extractor.main()
         except FileNotFoundError:
             prompt_container = st.container()
             frames2integ = \
                 prompt_setup(prompt_container, software, framerate,
                              annotation_classes, working_dir, prefix)
             if st.button('Extract Features'):
-                extractor = Extract(working_dir, prefix, frames2integ)
+                extractor = Extract(working_dir, prefix, frames2integ, is_3d)
                 extractor.main()
         st.session_state['page'] = 'Step 3'
 

asoid/apps/C_auto_active_learning.py

@@ -9,7 +9,17 @@
 from utils.project_utils import update_config
 
 TITLE = "Active learning"
-
+ACTIVE_LEARNING_HELP = ("In this step, you will train a classifier using a small set of labeled data and then small portions of the remaining training data are fed to the classifier for several iterations."
+                        "\n\n The samples are selected based on the classifier's confidence in its predictions. This process reaches high performance with limited labeled data."
+                        "\n\n :blue[The parameters **Initial sampling ratio**, **Max number of self-learning iterations**, and **Max samples amongst the X classes** can be adjusted to control the active learning process.]"
+                        "\n\n---\n\n"
+                        "**Step 1**: Select an iteration."
+                        "\n\n **Step 2**: Set the parameters."
+                        "\n\n **Step 3**: Train the classifier."
+                        "\n\n **Step 4**: View the results :blue[(might require a refresh by pressing 'R' on your keyboard)]."
+                        "\n\n **Step 5**: Refine the parameters or move to the next step :orange[Predict], :orange[Discover], or :orange[Refine Behaviors]."
+                        "\n\n---\n\n"
+                        ":blue[This classifier can be directly used in the prediction and discovery steps. Alternatively, you can refine the classifier by adding more unlabeled data in the next step:] :orange[Refine Behaviors]")
 
 def prompt_setup(software, train_fx, conf,
                  working_dir, prefix, iteration_dir, exclude_other, annotation_classes):
@@ -84,6 +94,9 @@ def prompt_setup(software, train_fx, conf,
 
 def main(ri=None, config=None):
     st.markdown("""---""")
+    st.title("Active Learning")
+    st.expander("What is this?", expanded=False).markdown(ACTIVE_LEARNING_HELP)
+
 
     if config is not None: