Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add cleanup device script #61

Merged
merged 3 commits into from
Jul 3, 2024
Merged

Add cleanup device script #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b60c35b
Add cleanupDeviceScript and usedStorageThresholdInPercents to configu…
Jul 3, 2024
8023457
Remove no longer used `com.android.tools.build:apksig`
Jul 3, 2024
ec93aad
Add DEFAULT_DEVICE_CLEANUP_SCRIPT
Jul 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions buildSrc/src/main/kotlin/Versions.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ object Versions {
val coroutines = "1.7.3"
val ktor = "2.0.3"

val ddmlib = "31.0.2"
val ddmlib = "31.4.2"
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Better to use ddmlib similar to our AGP version

val dexTestParser = "2.3.4"
val kotlinLogging = "1.4.9"
val slf4jAPI = "1.0.0"
Expand Down Expand Up @@ -53,7 +53,6 @@ object Libraries {
val ktorAuth = "io.ktor:ktor-client-auth-jvm:${Versions.ktor}"
val ktorApacheClient = "io.ktor:ktor-client-apache:${Versions.ktor}"
val axmlParser = "com.shazam:axmlparser:${Versions.axmlParser}"
val apkSig = "com.android.tools.build:apksig:${Versions.androidGradleVersion}"
val gson = "com.google.code.gson:gson:${Versions.gson}"
val apacheCommonsText = "org.apache.commons:commons-text:${Versions.apacheCommonsText}"
val apacheCommonsIO = "commons-io:commons-io:${Versions.apacheCommonsIO}"
Expand Down
8 changes: 7 additions & 1 deletion marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/ConfigurationFactory.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@ package com.malinskiy.marathon
import com.malinskiy.marathon.android.AndroidConfiguration
import com.malinskiy.marathon.android.DEFAULT_APPLICATION_PM_CLEAR
import com.malinskiy.marathon.android.DEFAULT_AUTO_GRANT_PERMISSION
import com.malinskiy.marathon.android.DEFAULT_DEVICE_CLEANUP_SCRIPT
import com.malinskiy.marathon.android.DEFAULT_INSTALL_OPTIONS
import com.malinskiy.marathon.android.DEFAULT_USED_STORAGE_THRESHOLD_PERCENTS
import com.malinskiy.marathon.android.defaultInitTimeoutMillis
import com.malinskiy.marathon.android.serial.SerialStrategy
import com.malinskiy.marathon.execution.Configuration
Expand Down Expand Up @@ -96,6 +98,8 @@ private fun createAndroidConfiguration(
}
}
?: SerialStrategy.AUTOMATIC
val cleanupDeviceScript = extension.cleanupDeviceScript ?: DEFAULT_DEVICE_CLEANUP_SCRIPT
val usedStorageThresholdInPercents = extension.usedStorageThresholdInPercents ?: DEFAULT_USED_STORAGE_THRESHOLD_PERCENTS

return AndroidConfiguration(
sdkDirectory,
Expand All @@ -109,6 +113,8 @@ private fun createAndroidConfiguration(
adbInitTimeout,
installOptions,
preferableRecorderType,
serialStrategy
serialStrategy,
cleanupDeviceScript,
usedStorageThresholdInPercents
)
}
2 changes: 2 additions & 0 deletions marathon-gradle-plugin/src/main/kotlin/com/malinskiy/marathon/MarathonExtension.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ open class MarathonExtension {
//Android specific for now
var autoGrantPermission: Boolean? = null
var instrumentationArgs: MutableMap<String, String> = mutableMapOf()
var cleanupDeviceScript: String? = null
var usedStorageThresholdInPercents: Int? = null

//Kotlin way
fun cache(block: CachePluginConfiguration.() -> Unit) {
Expand Down
1 change: 0 additions & 1 deletion vendor/vendor-android/base/build.gradle.kts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ dependencies {
implementation(Libraries.kotlinLogging)
implementation(Libraries.dexTestParser)
implementation(Libraries.axmlParser)
implementation(Libraries.apkSig)
implementation(Libraries.jacksonAnnotations)
implementation(Libraries.scalr)
implementation(project(":core"))
Expand Down
6 changes: 5 additions & 1 deletion ...endor-android/base/src/main/kotlin/com/malinskiy/marathon/android/AndroidConfiguration.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ const val DEFAULT_AUTO_GRANT_PERMISSION = false
const val DEFAULT_APPLICATION_PM_CLEAR = false
const val DEFAULT_TEST_APPLICATION_PM_CLEAR = false
const val DEFAULT_INSTALL_OPTIONS = ""
const val DEFAULT_USED_STORAGE_THRESHOLD_PERCENTS = 80
const val DEFAULT_DEVICE_CLEANUP_SCRIPT = "pm list packages -3 | grep -E '\\.test\$' | tr -d '\\r' | cut -d ':' -f 2 | xargs -n1 -t pm uninstall"

data class AndroidConfiguration(
val androidSdk: File,
Expand All @@ -26,7 +28,9 @@ data class AndroidConfiguration(
val adbInitTimeoutMillis: Int = defaultInitTimeoutMillis,
val installOptions: String = DEFAULT_INSTALL_OPTIONS,
val preferableRecorderType: DeviceFeature? = null,
val serialStrategy: SerialStrategy = SerialStrategy.AUTOMATIC
val serialStrategy: SerialStrategy = SerialStrategy.AUTOMATIC,
val cleanupDeviceScript: String = DEFAULT_DEVICE_CLEANUP_SCRIPT,
val usedStorageThresholdInPercents: Int = DEFAULT_USED_STORAGE_THRESHOLD_PERCENTS
) : VendorConfiguration {

private val koinModules = listOf(androidModule) + implementationModules
Expand Down
4 changes: 2 additions & 2 deletions vendor/vendor-android/base/src/main/kotlin/com/malinskiy/marathon/android/di/Modules.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import com.malinskiy.marathon.android.AndroidComponentCacheKeyProvider
import com.malinskiy.marathon.android.AndroidComponentInfoExtractor
import com.malinskiy.marathon.android.AndroidLogConfigurator
import com.malinskiy.marathon.android.AndroidTestParser
import com.malinskiy.marathon.android.ApkFileHasher
import com.malinskiy.marathon.android.executor.logcat.LogcatCollector
import com.malinskiy.marathon.android.executor.logcat.LogcatListener
import com.malinskiy.marathon.android.executor.logcat.parse.LogcatEventsAdapter
Expand All @@ -13,14 +12,15 @@ import com.malinskiy.marathon.cache.test.key.ComponentCacheKeyProvider
import com.malinskiy.marathon.execution.ComponentInfoExtractor
import com.malinskiy.marathon.execution.TestParser
import com.malinskiy.marathon.io.CachedFileHasher
import com.malinskiy.marathon.io.Md5FileHasher
import com.malinskiy.marathon.log.MarathonLogConfigurator
import com.malinskiy.marathon.report.logs.LogsProvider
import org.koin.dsl.module

val androidModule = module {
single<TestParser?> { AndroidTestParser() }
single<ComponentInfoExtractor?> { AndroidComponentInfoExtractor() }
single<ComponentCacheKeyProvider?> { AndroidComponentCacheKeyProvider(CachedFileHasher(ApkFileHasher())) }
single<ComponentCacheKeyProvider?> { AndroidComponentCacheKeyProvider(CachedFileHasher(Md5FileHasher())) }
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use MD5 in AndroidAppInstaller.isApkInstalled and in AndroidAppInstaller.getHashOnDevice, so ApkFileHasher is doing nothing. Checking MD5 is really fast and I don't see that it takes even more than a second on the build. We also have a hashmap so it is calculated only once.

single<LogcatCollector?> { LogcatCollector() }
single<LogcatEventsListener?> { get<LogcatCollector>() }
single<LogsProvider?> { get<LogcatCollector>() }
Expand Down
28 changes: 0 additions & 28 deletions ...dor-android/base/src/main/kotlin/com/malinskiy/marathon/android/executor/ApkFileHasher.kt
View file
Open in desktop

This file was deleted.

19 changes: 18 additions & 1 deletion ...ndor-android/ddmlib/src/main/kotlin/com/malinskiy/marathon/android/AndroidAppInstaller.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,11 @@ class AndroidAppInstaller(
if (isApkInstalled) {
logger.info("Skipping installation of $appPackage on ${device.serialNumber} - APK is already installed")
} else {
cleanupSpaceBeforeInstallation(device)
logger.info("Installing $appPackage, ${appApk.absolutePath} to ${device.serialNumber}")
val installationStarted = Instant.now()
val installMessage = device.safeInstallPackage(appApk.absolutePath, true, optionalParams(device))
installMessage?.let { logger.debug { it } }
installMessage?.let { logger.info { it } }
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Better to see as much as we can in the logs related to installation

track.installation(device.serialNumber, installationStarted, Instant.now())
installedApps
.getOrPut(device.serialNumber) { hashMapOf() }
Expand All @@ -69,6 +70,22 @@ class AndroidAppInstaller(
}
}

private fun cleanupSpaceBeforeInstallation(device: AndroidDevice) {
val storageUsedPercentage = device
.safeExecuteShellCommand("df /storage/emulated -h | grep '/storage/emulated' | awk '{print \$5}'")
.substringBefore("%")
.toInt()
logger.info { "Used $storageUsedPercentage% of storage on ${device.serialNumber}" }
val usedStorageThresholdInPercents = androidConfiguration.usedStorageThresholdInPercents
if (storageUsedPercentage > usedStorageThresholdInPercents) {
logger.warn { "On ${device.serialNumber} used more than $usedStorageThresholdInPercents% of storage" }
androidConfiguration.cleanupDeviceScript.let {
logger.info { "Launching cleanup shell script `$it`" }
device.safeExecuteShellCommand(it).let { logger.info { it } }
}
}
}

private suspend fun isApkInstalled(device: AndroidDevice, appPackage: String, fileHash: String): Boolean {
if (installedApps[device.serialNumber]?.get(appPackage) == fileHash) {
return true
Expand Down
8 changes: 7 additions & 1 deletion ...d/ddmlib/src/main/kotlin/com/malinskiy/marathon/android/ddmlib/AndroidDeviceTestRunner.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import com.android.ddmlib.ShellCommandUnresponsiveException
import com.android.ddmlib.TimeoutException
import com.android.ddmlib.testrunner.ITestRunListener
import com.android.ddmlib.testrunner.RemoteAndroidTestRunner
import com.android.ddmlib.testrunner.RemoteAndroidTestRunner.StatusReporterMode
import com.android.ddmlib.testrunner.TestIdentifier
import com.malinskiy.marathon.android.AndroidComponentInfo
import com.malinskiy.marathon.android.AndroidConfiguration
Expand Down Expand Up @@ -104,7 +105,12 @@ class AndroidDeviceTestRunner(private val device: DdmlibAndroidDevice) {
testBatch: TestBatch
): RemoteAndroidTestRunner {

val runner = RemoteAndroidTestRunner(info.instrumentationPackage, info.testRunnerClass, device.ddmsDevice)
val runner = RemoteAndroidTestRunner(
info.instrumentationPackage,
info.testRunnerClass,
device.ddmsDevice,
StatusReporterMode.PROTO_STD
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

StatusReporterMode.TEXT is deprecated from API 26 and used by default in 3 arguments constructor. StatusReporterMode.PROTO_STD should be better as it will use less data because of binary format rather than text.

)

val tests = testBatch.tests.map {
val pkg = when {
Expand Down
Loading