external/revanced-patcher
1
0
Fork 0
mirror of https://github.com/ReVanced/revanced-patcher.git synced 2024-11-12 18:04:24 +01:00
Code Issues Projects Releases Packages Wiki Activity

perf: Free memory earlier and remove negligible lookup maps

Browse source 
Negligible lookup maps used for matching fingerprints have been removed to reduce the likelihood of OOM when the maps are instantiated, commonly observed with 400M RAM. Additionally, lookup maps previously kept for the duration of the patcher instance are now cleared before the classes are compiled. This reduces the likelihood of OOM when compiling classes.
On a related note, a linear increase in memory usage is observed with every compiled class until all classes are compiled implying compiled classes not being freed by GC because they are still referenced. After compiling a class, the class is technically free-able though. The classes are assumed to be referenced in the `multidexlib2` library that takes the list of all classes to compile multiple DEX with and seems to hold the reference to all these classes in memory until all DEX are compiled. A clever fix would involve splitting the list of classes into chunks and getting rid of the list of all classes so that after every DEX compilation, the corresponding split of classes can be freed.
This commit is contained in:
oSumAtrIX 2024-10-03 01:31:37 +02:00
parent f1615b7ab5
commit d53aacdad4
No known key found for this signature in database
GPG key ID: A9B3094ACDB604B4
5 changed files with 44 additions and 94 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
src/main/kotlin/app/revanced/patcher/Fingerprint.kt
View file

@ -66,36 +66,15 @@ class Fingerprint internal constructor(
}
// TODO: If only one string is necessary, why not use a single string for every fingerprint?
fun Fingerprint.lookupByStrings() = strings?.firstNotNullOfOrNull { lookupMaps.methodsByStrings[it] }
if (lookupByStrings()?.let(::match) == true) {
if (strings?.firstNotNullOfOrNull { lookupMaps.methodsByStrings[it] }?.let(::match) == true) {
return true
}
// No strings declared or none matched (partial matches are allowed).
// Use signature matching.
fun Fingerprint.lookupBySignature(): MethodClassPairs {
if (accessFlags == null) return lookupMaps.allMethods
var returnTypeValue = returnType
if (returnTypeValue == null) {
if (AccessFlags.CONSTRUCTOR.isSet(accessFlags)) {
// Constructors always have void return type.
returnTypeValue = "V"
} else {
return lookupMaps.allMethods
}
}
val signature =
buildString {
append(accessFlags)
append(returnTypeValue.first())
appendParameters(parameters ?: return@buildString)
}
return lookupMaps.methodsBySignature[signature] ?: return MethodClassPairs()
context.classes.forEach { classDef ->
if (match(context, classDef)) return true
}
return match(lookupBySignature())
return false
}
/**

12
src/main/kotlin/app/revanced/patcher/Patcher.kt
View file

@ -98,14 +98,12 @@ class Patcher(private val config: PatcherConfig) : Closeable {
logger.info("Merging extensions")
context.executablePatches.forEachRecursively { patch ->
if (patch is BytecodePatch && patch.extension != null) {
context.bytecodeContext.merge(patch.extension)
}
}
with(context.bytecodeContext) {
context.executablePatches.mergeExtensions()
// Initialize lookup maps.
context.bytecodeContext.lookupMaps
// Initialize lookup maps.
lookupMaps
}
logger.info("Executing patches")

1
src/main/kotlin/app/revanced/patcher/PatcherConfig.kt
View file

@ -20,6 +20,7 @@ class PatcherConfig(
private val temporaryFilesPath: File = File("revanced-temporary-files"),
aaptBinaryPath: String? = null,
frameworkFileDirectory: String? = null,
@Deprecated("This is going to be removed in the future because it is not needed anymore.")
internal val multithreadingDexFileWriter: Boolean = false,
) {
private val logger = Logger.getLogger(PatcherConfig::class.java.name)

91
src/main/kotlin/app/revanced/patcher/patch/BytecodePatchContext.kt
View file

@ -21,7 +21,6 @@ import lanchon.multidexlib2.MultiDexIO
import lanchon.multidexlib2.RawDexIO
import java.io.Closeable
import java.io.FileFilter
import java.io.InputStream
import java.util.*
import java.util.logging.Logger
@ -60,40 +59,41 @@ class BytecodePatchContext internal constructor(private val config: PatcherConfi
internal val lookupMaps by lazy { LookupMaps(classes) }
/**
* A map for lookup by [merge].
* Merge the extensions for this set of patches.
*/
internal val classesByType = mutableMapOf<String, ClassDef>().apply {
classes.forEach { classDef -> put(classDef.type, classDef) }
}
internal fun Set<Patch<*>>.mergeExtensions() {
// Lookup map for fast checking if a class exists by its type.
val classesByType = mutableMapOf<String, ClassDef>().apply {
classes.forEach { classDef -> put(classDef.type, classDef) }
}
/**
* Merge an extension to [classes].
*
* @param extensionInputStream The input stream of the extension to merge.
*/
internal fun merge(extensionInputStream: InputStream) {
val extension = extensionInputStream.readAllBytes()
forEachRecursively { patch ->
if (patch is BytecodePatch && patch.extension != null) {
RawDexIO.readRawDexFile(extension, 0, null).classes.forEach { classDef ->
val existingClass = classesByType[classDef.type] ?: run {
logger.fine("Adding class \"$classDef\"")
val extension = patch.extension.readAllBytes()
classes += classDef
classesByType[classDef.type] = classDef
RawDexIO.readRawDexFile(extension, 0, null).classes.forEach { classDef ->
val existingClass = classesByType[classDef.type] ?: run {
logger.fine("Adding class \"$classDef\"")
return@forEach
}
classes += classDef
classesByType[classDef.type] = classDef
logger.fine("Class \"$classDef\" exists already. Adding missing methods and fields.")
return@forEach
}
existingClass.merge(classDef, this@BytecodePatchContext).let { mergedClass ->
// If the class was merged, replace the original class with the merged class.
if (mergedClass === existingClass) {
return@let
logger.fine("Class \"$classDef\" exists already. Adding missing methods and fields.")
existingClass.merge(classDef, this@BytecodePatchContext).let { mergedClass ->
// If the class was merged, replace the original class with the merged class.
if (mergedClass === existingClass) {
return@let
}
classes -= existingClass
classes += mergedClass
}
}
classes -= existingClass
classes += mergedClass
}
}
}
@ -145,6 +145,9 @@ class BytecodePatchContext internal constructor(private val config: PatcherConfi
override fun get(): Set<PatcherResult.PatchedDexFile> {
logger.info("Compiling patched dex files")
// Free up memory before compiling the dex files.
lookupMaps.close()
val patchedDexFileResults =
config.patchedFiles.resolve("dex").also {
it.deleteRecursively() // Make sure the directory is empty.
@ -178,21 +181,6 @@ class BytecodePatchContext internal constructor(private val config: PatcherConfi
* @param classes The list of classes to create the lookup maps from.
*/
internal class LookupMaps internal constructor(classes: List<ClassDef>) : Closeable {
/**
* Classes associated by their type.
*/
internal val classesByType = classes.associateBy { it.type }.toMutableMap()
/**
* All methods and the class they are a member of.
*/
internal val allMethods = MethodClassPairs()
/**
* Methods associated by its access flags, return type and parameter.
*/
internal val methodsBySignature = MethodClassPairsLookupMap()
/**
* Methods associated by strings referenced in it.
*/
@ -203,22 +191,6 @@ class BytecodePatchContext internal constructor(private val config: PatcherConfi
classDef.methods.forEach { method ->
val methodClassPair: MethodClassPair = method to classDef
// For fingerprints with no access or return type specified.
allMethods += methodClassPair
val accessFlagsReturnKey = method.accessFlags.toString() + method.returnType.first()
// Add <access><returnType> as the key.
methodsBySignature[accessFlagsReturnKey] = methodClassPair
// Add <access><returnType>[parameters] as the key.
methodsBySignature[
buildString {
append(accessFlagsReturnKey)
appendParameters(method.parameterTypes)
},
] = methodClassPair
// Add strings contained in the method as the key.
method.instructionsOrNull?.forEach instructions@{ instruction ->
if (instruction.opcode != Opcode.CONST_STRING && instruction.opcode != Opcode.CONST_STRING_JUMBO) {
@ -259,15 +231,12 @@ class BytecodePatchContext internal constructor(private val config: PatcherConfi
}
override fun close() {
allMethods.clear()
methodsBySignature.clear()
methodsByStrings.clear()
}
}
override fun close() {
lookupMaps.close()
classesByType.clear()
classes.clear()
}
}

3
src/test/kotlin/app/revanced/patcher/PatcherTest.kt
View file

@ -6,7 +6,9 @@ import app.revanced.patcher.util.ProxyClassList
import com.android.tools.smali.dexlib2.immutable.ImmutableClassDef
import com.android.tools.smali.dexlib2.immutable.ImmutableMethod
import io.mockk.every
import io.mockk.just
import io.mockk.mockk
import io.mockk.runs
import kotlinx.coroutines.flow.toList
import kotlinx.coroutines.runBlocking
import org.junit.jupiter.api.BeforeEach
@ -193,6 +195,7 @@ internal object PatcherTest {
private operator fun Set<Patch<*>>.invoke(): List<PatchResult> {
every { patcher.context.executablePatches } returns toMutableSet()
every { patcher.context.bytecodeContext.lookupMaps } returns LookupMaps(patcher.context.bytecodeContext.classes)
every { with(patcher.context.bytecodeContext) { any<Set<Patch<*>>>().mergeExtensions() } } just runs
return runBlocking { patcher().toList() }
}