[Perf] Add strategy for performance test

agent/doc/UML/performance_test_design.puml

@@ -0,0 +1,110 @@
+
+@startmindmap performance_test_inspection_entrypoints
+* Performance Test
+** Define the strategy in TestTask
+*** LifeCycle
+****:
+TEST_STARTED
+TEST_FINISHED
+TEST_FAILED
+TEST_RUN_STARTED
+TEST_RUN_FINISHED
+;
+*** Schedule
+** Call inspect() method in a test case
+*** Hydra SDK
+*** T2C
+*** Espresso
+@endmindmap
+@startjson taps_to_cases_json_with_performance_test_actions
+{
+  "drivers": [
+    {
+      "id": "13211FDD400183",
+      "platform": "android",
+'      Deprecated Name : init
+      "setUp": {
+        "launcherApp": ""
+      }
+'      Can also add tearDown here to match the device action design
+    }
+  ],
+'  deprecated name: cases
+  "actions": [
+    {
+      "index": 0,
+      "driverId": "13211FDD400183",
+      "action": {
+        "actionType": "inspection",
+        "arguments": {
+          "inspectorType": "INSPECTOR_ANDROID_MEMORY_INFO",
+          "targetApp": "com.microsoft.appmanager",
+          "description": "Start LTW",
+          "isReset": false
+        }
+      },
+      "isOptional": false
+    },
+    {
+      "index": 1,
+      "driverId": "13211FDD400183",
+      "action": {
+        "actionType": "inspection",
+        "arguments": {
+          "inspectorType": "INSPECTOR_ANDROID_BATTERY_INFO",
+          "targetApp ": "com.microsoft.appmanager",
+          "description ": "Start LTW",
+          "isReset": true
+        }
+      },
+      "isOptional": false
+    }
+  ]
+}
+@endjson
+
+@startuml perf_test_class_design
+class InspectionStrategy {
+
+    + StrategyType strategyType;
+
+    // if the strategyType == TEST_LIFECYCLE
+    + List<WhenType> when;
+
+    // if the strategyType == TEST_SCHEDULE
+    + long interval;
+    + TimeUnit intervalUnit;
+}
+
+interface PerformanceTestListener {
+    void testStarted();
+    void testFinished();
+    void testFailure();
+    void testRunStarted();
+    void testRunFinished();
+}
+
+class PerformanceTestManagementService {
+    Map<String, List<ScheduledFuture<?>>> inspectPerformanceTimerMap
+    Map<String, List<InspectionStrategy>> testLifeCycleStrategyMap
+}
+
+PerformanceTestManagementService -> InspectionStrategy
+PerformanceTestManagementService -u-|> PerformanceTestListener
+
+PerformanceTestManagementService -d-o TestRunner
+AppiumListener -d->  PerformanceTestListener
+Junit5Listener -d->  PerformanceTestListener
+EspressoTestInfoProcessorListener  -d->  PerformanceTestListener
+@enduml
+
+@startuml espresso_performance_test_design
+class EspressoPerformanceParser {
+    void processNewLines()
+}
+EspressoPerformanceParser --|> InstrumentationResultParser
+TestRunner <|-- EspressoRunner
+EspressoRunner -> EspressoPerformanceParser
+EspressoPerformanceParser -> PerformanceInspectionService
+@enduml
+

agent/doc/UML/test_runner_design.puml

@@ -110,7 +110,34 @@ deactivate TaskCompletion
         "isReset": false
     },
     "frameworkType": "JUnit4",
-    "testRunnerName": "androidx.test.runner.AndroidJUnitRunner"
+    "testRunnerName": "androidx.test.runner.AndroidJUnitRunner",
+    "inspectionStrategies": [
+        {
+            "strategyType": "TEST_SCHEDULE",
+            "interval": 300,
+            "intervalUnit": "MILLISECONDS",
+            "inspection": {
+                "inspectorType": "INSPECTOR_ANDROID_BATTERY_INFO",
+                "appId": "com.microsoft.appmanager",
+                "description": "test schedule"
+            }
+        },
+        {
+             "strategyType": "TEST_LIFECYCLE",
+             "when": [
+                 "TEST_STARTED",
+                 "TEST_SUCCESS",
+                 "TEST_FAILURE",
+                 "TEST_RUN_STARTED",
+                 "TEST_RUN_FINISHED"
+             ],
+             "inspection": {
+                 "inspectorType": "INSPECTOR_ANDROID_BATTERY_INFO",
+                 "appId": "com.microsoft.appmanager",
+                 "description": "test schedule"
+             }
+         }
+    ]
 }
 @endjson
 

agent/src/main/java/com/microsoft/hydralab/agent/config/TestRunnerConfig.java

@@ -15,6 +15,7 @@
 import com.microsoft.hydralab.common.entity.common.TestTask;
 import com.microsoft.hydralab.common.management.DeviceManager;
 import com.microsoft.hydralab.common.util.ADBOperateUtil;
+import com.microsoft.hydralab.performance.PerformanceTestManagementService;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.context.annotation.Bean;
@@ -39,38 +40,55 @@ public class TestRunnerConfig {
     );
 
     @Bean
-    public EspressoRunner espressoRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService, ADBOperateUtil adbOperateUtil) {
-        return new EspressoRunner(deviceManager, testTaskEngineService, adbOperateUtil);
+    public PerformanceTestManagementService performanceTestManagementService() {
+        PerformanceTestManagementService performanceTestManagementService = new PerformanceTestManagementService();
+        performanceTestManagementService.initialize();
+        return performanceTestManagementService;
     }
 
     @Bean
-    public AdbMonkeyRunner adbMonkeyRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService, ADBOperateUtil adbOperateUtil) {
-        return new AdbMonkeyRunner(deviceManager, testTaskEngineService, adbOperateUtil);
+    public EspressoRunner espressoRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService,
+                                         PerformanceTestManagementService performanceTestManagementService,
+                                         ADBOperateUtil adbOperateUtil) {
+        return new EspressoRunner(deviceManager, testTaskEngineService, performanceTestManagementService, adbOperateUtil);
     }
 
     @Bean
-    public AppiumMonkeyRunner appiumMonkeyRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService) {
-        return new AppiumMonkeyRunner(deviceManager, testTaskEngineService);
+    public AdbMonkeyRunner adbMonkeyRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService,
+                                           PerformanceTestManagementService performanceTestManagementService,
+                                           ADBOperateUtil adbOperateUtil) {
+        return new AdbMonkeyRunner(deviceManager, testTaskEngineService, performanceTestManagementService, adbOperateUtil);
     }
 
     @Bean
-    public AppiumRunner appiumRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService) {
-        return new AppiumRunner(deviceManager, testTaskEngineService);
+    public AppiumMonkeyRunner appiumMonkeyRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService,
+                                                 PerformanceTestManagementService performanceTestManagementService) {
+        return new AppiumMonkeyRunner(deviceManager, testTaskEngineService, performanceTestManagementService);
     }
 
     @Bean
-    public AppiumCrossRunner appiumCrossRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService) {
-        return new AppiumCrossRunner(deviceManager, testTaskEngineService, agentName);
+    public AppiumRunner appiumRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService,
+                                     PerformanceTestManagementService performanceTestManagementService) {
+        return new AppiumRunner(deviceManager, testTaskEngineService, performanceTestManagementService);
     }
 
     @Bean
-    public SmartRunner smartRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService, SmartTestUtil smartTestUtil) {
-        return new SmartRunner(deviceManager, testTaskEngineService, smartTestUtil);
+    public AppiumCrossRunner appiumCrossRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService,
+                                               PerformanceTestManagementService performanceTestManagementService) {
+        return new AppiumCrossRunner(deviceManager, testTaskEngineService, performanceTestManagementService, agentName);
     }
 
     @Bean
-    public T2CRunner t2cRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService) {
-        return new T2CRunner(deviceManager, testTaskEngineService, agentName);
+    public SmartRunner smartRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService,
+                                   PerformanceTestManagementService performanceTestManagementService,
+                                   SmartTestUtil smartTestUtil) {
+        return new SmartRunner(deviceManager, testTaskEngineService, performanceTestManagementService, smartTestUtil);
+    }
+
+    @Bean
+    public T2CRunner t2cRunner(DeviceManager deviceManager, TestTaskEngineService testTaskEngineService,
+                               PerformanceTestManagementService performanceTestManagementService) {
+        return new T2CRunner(deviceManager, testTaskEngineService, performanceTestManagementService, agentName);
     }
 
     @ConfigurationProperties(prefix = "app.device-script.commands")

agent/src/main/java/com/microsoft/hydralab/agent/runner/TestRunner.java

@@ -13,6 +13,8 @@
 import com.microsoft.hydralab.common.util.LogUtils;
 import com.microsoft.hydralab.common.util.ThreadPoolUtil;
 import com.microsoft.hydralab.common.util.ThreadUtils;
+import com.microsoft.hydralab.performance.InspectionStrategy;
+import com.microsoft.hydralab.performance.PerformanceTestManagementService;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -29,12 +31,15 @@ public abstract class TestRunner {
     protected final Logger log = LoggerFactory.getLogger(DeviceManager.class);
     protected final DeviceManager deviceManager;
     protected final TestTaskRunCallback testTaskRunCallback;
+    protected final PerformanceTestManagementService performanceTestManagementService;
     protected final XmlBuilder xmlBuilder = new XmlBuilder();
     protected final ActionExecutor actionExecutor = new ActionExecutor();
 
-    public TestRunner(DeviceManager deviceManager, TestTaskRunCallback testTaskRunCallback) {
+    public TestRunner(DeviceManager deviceManager, TestTaskRunCallback testTaskRunCallback,
+                      PerformanceTestManagementService performanceTestManagementService) {
         this.deviceManager = deviceManager;
         this.testTaskRunCallback = testTaskRunCallback;
+        this.performanceTestManagementService = performanceTestManagementService;
     }
 
     public void runTestOnDevice(TestTask testTask, DeviceInfo deviceInfo, Logger logger) {
@@ -52,7 +57,6 @@ public void runTestOnDevice(TestTask testTask, DeviceInfo deviceInfo, Logger log
             testRun.getLogger().error(deviceInfo.getSerialNum() + ": " + e.getMessage(), e);
             saveErrorSummary(testRun, e);
         } finally {
-            //TODO: tearDown for performance testing. Android battery: adb shell dumpsys battery reset
             tearDown(deviceInfo, testTask, testRun);
         }
     }
@@ -140,11 +144,22 @@ protected void setUp(DeviceInfo deviceInfo, TestTask testTask, TestRun testRun)
 
         checkTestTaskCancel(testTask);
         deviceManager.getScreenShot(deviceInfo, testRun.getLogger());
+
+        if (performanceTestManagementService != null && testTask.getInspectionStrategies() != null) {
+            for (InspectionStrategy strategy : testTask.getInspectionStrategies()) {
+                performanceTestManagementService.inspectWithStrategy(strategy);
+            }
+        }
     }
 
     protected abstract void run(DeviceInfo deviceInfo, TestTask testTask, TestRun testRun) throws Exception;
 
     protected void tearDown(DeviceInfo deviceInfo, TestTask testTask, TestRun testRun) {
+        // stop performance test
+        if (performanceTestManagementService != null) {
+            performanceTestManagementService.testTearDown(deviceInfo, log);
+        }
+
         //execute actions
         if (testTask.getDeviceActions() != null) {
             testRun.getLogger().info("Start executing tearDown actions.");

agent/src/main/java/com/microsoft/hydralab/agent/runner/appium/AppiumCrossRunner.java

@@ -7,13 +7,15 @@
 import com.microsoft.hydralab.common.entity.common.TestRun;
 import com.microsoft.hydralab.common.entity.common.TestTask;
 import com.microsoft.hydralab.common.management.DeviceManager;
+import com.microsoft.hydralab.performance.PerformanceTestManagementService;
 import org.slf4j.Logger;
 
 public class AppiumCrossRunner extends AppiumRunner {
     String agentName;
 
-    public AppiumCrossRunner(DeviceManager deviceManager, TestTaskRunCallback testTaskRunCallback, String agentName) {
-        super(deviceManager, testTaskRunCallback);
+    public AppiumCrossRunner(DeviceManager deviceManager, TestTaskRunCallback testTaskRunCallback,
+                             PerformanceTestManagementService performanceTestManagementService, String agentName) {
+        super(deviceManager, testTaskRunCallback, performanceTestManagementService);
         this.agentName = agentName;
     }
 

agent/src/main/java/com/microsoft/hydralab/agent/runner/appium/AppiumListener.java

@@ -10,6 +10,7 @@
 import com.microsoft.hydralab.common.logger.LogCollector;
 import com.microsoft.hydralab.common.management.DeviceManager;
 import com.microsoft.hydralab.common.screen.ScreenRecorder;
+import com.microsoft.hydralab.performance.PerformanceTestListener;
 import org.junit.runner.Description;
 import org.junit.runner.Result;
 import org.junit.runner.notification.Failure;
@@ -31,6 +32,7 @@ public class AppiumListener extends RunListener {
     private final AnimatedGifEncoder e = new AnimatedGifEncoder();
     private final String pkgName;
     DeviceManager deviceManager;
+    private final PerformanceTestListener performanceTestListener;
     private long recordingStartTimeMillis;
     private int index;
     private File gifFile;
@@ -43,12 +45,14 @@ public class AppiumListener extends RunListener {
     private int currentTestIndex = 0;
 
 
-    public AppiumListener(DeviceManager deviceManager, DeviceInfo deviceInfo, TestRun testRun, String pkgName, Logger logger) {
+    public AppiumListener(DeviceManager deviceManager, DeviceInfo deviceInfo, TestRun testRun, String pkgName,
+                          PerformanceTestListener performanceTestListener, Logger logger) {
         this.deviceManager = deviceManager;
         this.deviceInfo = deviceInfo;
         this.testRun = testRun;
         this.logger = logger;
         this.pkgName = pkgName;
+        this.performanceTestListener = performanceTestListener;
         logcatCollector = deviceManager.getLogCollector(deviceInfo, pkgName, testRun, logger);
         deviceScreenRecorder = deviceManager.getScreenRecorder(deviceInfo, testRun.getResultFolder(), logger);
     }
@@ -107,6 +111,7 @@ public void testRunStarted(Description description) {
         testRun.addNewTimeTag("testRunStarted", System.currentTimeMillis() - recordingStartTimeMillis);
         deviceInfo.setRunningTestName(runName.substring(runName.lastIndexOf('.') + 1) + ".testRunStarted");
         logEnter(runName, description.testCount());
+        performanceTestListener.testRunStarted();
     }
 
     @Override
@@ -159,6 +164,7 @@ public void testStarted(Description description) {
                 ioException.printStackTrace();
             }
         }), logger);
+        performanceTestListener.testStarted(ongoingTestUnit.getTitle());
     }
 
     @Override
@@ -167,6 +173,7 @@ public void testFailure(Failure failure) {
         logEnter("testFailed", testDisplayName, failure.getTrace());
         ongoingTestUnit.setStack(failure.getTrace());
         ongoingTestUnit.setStatusCode(AndroidTestUnit.StatusCodes.FAILURE);
+        performanceTestListener.testFailure(ongoingTestUnit.getTitle());
         testRun.addNewTimeTag(ongoingTestUnit.getTitle() + ".fail", System.currentTimeMillis() - recordingStartTimeMillis);
         testRun.oneMoreFailure();
     }
@@ -197,6 +204,7 @@ public void testFinished(Description description) {
         ) {
             ongoingTestUnit.setStatusCode(AndroidTestUnit.StatusCodes.OK);
             ongoingTestUnit.setSuccess(true);
+            performanceTestListener.testSuccess(ongoingTestUnit.getTitle());
         }
         ongoingTestUnit.setEndTimeMillis(System.currentTimeMillis());
         ongoingTestUnit.setRelEndTimeInVideo(ongoingTestUnit.getEndTimeMillis() - recordingStartTimeMillis);
@@ -230,6 +238,7 @@ public void testRunFinished(Result result) {
             }
 
         }
+        performanceTestListener.testRunFinished();
 
         logEnter("testRunEnded", elapsedTime, Thread.currentThread().getName());
         synchronized (this) {