Skip to content

Commit

Permalink
[Xamarin.Android.Build.Tasks] Extract BuildArchive from BuildApk (#…
Browse files Browse the repository at this point in the history 
…9556)

Break down the mega-task `BuildApk` into smaller separation
of concerns:

  - `BuildApk`: Generate files that need to be placed in the APK.
  - `BuildArchive`: Place files in the APK.

This works via the `@(FilesToAddToArchive)` item group which is defined as:

	<FilesToAddToArchive
	    <!-- Path of file on disk to add -->
	    Include="obj\Release\net9.0-android\app_shared_libraries\arm64-v8a\libxamarin-app.so"
	    <!-- Path to place file in APK -->
	    ArchivePath="lib/arm64-v8a/libxamarin-app.so"
	    <!-- Compression level (optional) -->
	    Compression="Store"
	/>

The `%(FilesToAddToArchive.Compression)` metadata is optional and is
really only needed for handling the assembly compression information
[stored in `RegisterTaskObject()`][0].  The valid values are in
[`CompressionMethod.cs`][1].  Hopefully this will not be needed in
the final version.

This change will allow future PRs to break down the `<BuildApk/>`
task into separate tasks for each type of files that belong in the
`.apk`, facilitating better incremental builds.  For example, if only
the Dalvik file changes we don't need to reprocess every managed
assembly, etc.

~~ Implementation ~~

To keep the diff more manageable at this point, create the temporary
`ZipArchiveFileListBuilder` as a drop-in replacement for
`ZipArchiveEx`.  Instead of adding files to the `.apk` it collects
the file name and apk path so that the `<BuildArchive/>` task can do
the actual adding.

Additionally, there is code that looks in every referenced `.jar`
file and copies non-code files into the final apk.  This is currently
handled by specifying the `%(Include)=path/to/my.jar#ExtraFile.txt`.
In a final future state we would probably unzip the `.jar`s to disk
and add the files from there.  This would allow us to avoid unzipping
the `.jar`s on every build.

~~ Performance ~~

Although performance isn't the focus of this PR, there is a nice win
from eliminating extraneous `ZipArchiveEx.Flush()` calls.
(Note that the AutoFlush logic still exists.)

| _BuildApkEmbed target                     |    .NET 9 |   This PR |
| :---------------------------------------- | --------: | --------: |
| Debug (FastDev - _BuildApkFastDev target) |    2.684s |    2.670s |
| Debug (EmbedAssembliesIntoApk = true)     |   79.981s |   75.720s |
| Release                                   |    8.130s |    6.121s | 

[0]: https://github.com/dotnet/android/blob/e98d20f08ef1a3521a35749754a3ae6a61314d0b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs#L355-L361
[1]: https://github.com/dotnet/android-libzipsharp/blob/57558e9e9b5e6e460fcbaca663de1ca80ca467a0/LibZipSharp/Xamarin.Tools.Zip/CompressionMethod.cs
  • Loading branch information
@jpobst
jpobst authored Dec 11, 2024
1 parent 2fa7954 commit 894ecea
Show file tree
Hide file tree
Showing 6 changed files with 456 additions and 249 deletions.
126 changes: 26 additions & 100 deletions src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,6 @@ public class BuildApk : AndroidTask
[Required]
public string [] SupportedAbis { get; set; }

public bool CreatePackagePerAbi { get; set; }

public bool EmbedAssemblies { get; set; }

public bool BundleAssemblies { get; set; }
Expand Down Expand Up @@ -121,6 +119,13 @@ public class BuildApk : AndroidTask
[Output]
public ITaskItem[] OutputFiles { get; set; }

[Output]
public ITaskItem[] OutputApkFiles { get; set; }

[Output]
public ITaskItem [] DSODirectoriesToDelete { get; set; }


bool _Debug {
get {
return string.Equals (Debug, "true", StringComparison.OrdinalIgnoreCase);
Expand All @@ -133,13 +138,13 @@ bool _Debug {
HashSet<string> uncompressedFileExtensions;

// Do not use trailing / in the path
protected virtual string RootPath => "";
public string RootPath { get; set; } = "";

protected virtual string DalvikPath => "";
public string DalvikPath { get; set; } = "";

protected virtual CompressionMethod UncompressedMethod => CompressionMethod.Store;

protected virtual void FixupArchive (ZipArchiveEx zip) { }
protected virtual void FixupArchive (ZipArchiveFileListBuilder zip) { }

List<string> existingEntries = new List<string> ();

Expand All @@ -150,59 +155,8 @@ protected virtual void FixupArchive (ZipArchiveEx zip) { }
void ExecuteWithAbi (DSOWrapperGenerator.Config dsoWrapperConfig, string [] supportedAbis, string apkInputPath, string apkOutputPath, bool debug, bool compress, IDictionary<AndroidTargetArch, Dictionary<string, CompressedAssemblyInfo>> compressedAssembliesInfo, string assemblyStoreApkName)
{
ArchiveFileList files = new ArchiveFileList ();
bool refresh = true;
if (apkInputPath != null && File.Exists (apkInputPath) && !File.Exists (apkOutputPath)) {
Log.LogDebugMessage ($"Copying {apkInputPath} to {apkOutputPath}");
File.Copy (apkInputPath, apkOutputPath, overwrite: true);
refresh = false;
}
using (var apk = new ZipArchiveEx (apkOutputPath, File.Exists (apkOutputPath) ? FileMode.Open : FileMode.Create )) {
if (int.TryParse (ZipFlushFilesLimit, out int flushFilesLimit)) {
apk.ZipFlushFilesLimit = flushFilesLimit;
}
if (int.TryParse (ZipFlushSizeLimit, out int flushSizeLimit)) {
apk.ZipFlushSizeLimit = flushSizeLimit;
}
if (refresh) {
for (long i = 0; i < apk.Archive.EntryCount; i++) {
ZipEntry e = apk.Archive.ReadEntry ((ulong) i);
Log.LogDebugMessage ($"Registering item {e.FullName}");
existingEntries.Add (e.FullName);
}
}
if (apkInputPath != null && File.Exists (apkInputPath) && refresh) {
var lastWriteOutput = File.Exists (apkOutputPath) ? File.GetLastWriteTimeUtc (apkOutputPath) : DateTime.MinValue;
var lastWriteInput = File.GetLastWriteTimeUtc (apkInputPath);
using (var packaged = new ZipArchiveEx (apkInputPath, FileMode.Open)) {
foreach (var entry in packaged.Archive) {
// NOTE: aapt2 is creating zip entries on Windows such as `assets\subfolder/asset2.txt`
var entryName = entry.FullName;
if (entryName.Contains ("\\")) {
entryName = entryName.Replace ('\\', '/');
Log.LogDebugMessage ($"Fixing up malformed entry `{entry.FullName}` -> `{entryName}`");
}
Log.LogDebugMessage ($"Deregistering item {entryName}");
existingEntries.Remove (entryName);
if (lastWriteInput <= lastWriteOutput) {
Log.LogDebugMessage ($"Skipping to next item. {lastWriteInput} <= {lastWriteOutput}.");
continue;
}
if (apk.Archive.ContainsEntry (entryName)) {
ZipEntry e = apk.Archive.ReadEntry (entryName);
// check the CRC values as the ModifiedDate is always 01/01/1980 in the aapt generated file.
if (entry.CRC == e.CRC && entry.CompressedSize == e.CompressedSize) {
Log.LogDebugMessage ($"Skipping {entryName} from {apkInputPath} as its up to date.");
continue;
}
}
var ms = new MemoryStream ();
entry.Extract (ms);
Log.LogDebugMessage ($"Refreshing {entryName} from {apkInputPath}");
apk.Archive.AddStream (ms, entryName, compressionMethod: entry.CompressionMethod);
}
}
}
apk.FixupWindowsPathSeparators ((a, b) => Log.LogDebugMessage ($"Fixing up malformed entry `{a}` -> `{b}`"));

using (var apk = new ZipArchiveFileListBuilder (apkOutputPath, File.Exists (apkOutputPath) ? FileMode.Open : FileMode.Create)) {

// Add classes.dx
CompressionMethod dexCompressionMethod = GetCompressionMethod (".dex");
Expand Down Expand Up @@ -281,7 +235,7 @@ void ExecuteWithAbi (DSOWrapperGenerator.Config dsoWrapperConfig, string [] supp
}
if (!forceInclude) {
foreach (var pattern in excludePatterns) {
if(pattern.IsMatch (path)) {
if (pattern.IsMatch (path)) {
Log.LogDebugMessage ($"Ignoring jar entry '{name}' from '{Path.GetFileName (jarFile)}'. Filename matched the exclude pattern '{pattern}'.");
exclude = true;
break;
Expand All @@ -294,30 +248,15 @@ void ExecuteWithAbi (DSOWrapperGenerator.Config dsoWrapperConfig, string [] supp
Log.LogDebugMessage ("Ignoring jar entry {0} from {1}: the same file already exists in the apk", name, Path.GetFileName (jarFile));
continue;
}
if (apk.Archive.Any (e => e.FullName == path)) {
Log.LogDebugMessage ("Failed to add jar entry {0} from {1}: the same file already exists in the apk", name, Path.GetFileName (jarFile));
continue;
}
byte [] data;
using (var d = new MemoryStream ()) {
jarItem.Extract (d);
data = d.ToArray ();
}
Log.LogDebugMessage ($"Adding {path} from {jarFile} as the archive file is out of date.");
apk.AddEntryAndFlush (data, path);

apk.AddJavaEntryAndFlush (jarFile, jarItem.FullName, path);
}
}
}
// Clean up Removed files.
foreach (var entry in existingEntries) {
// never remove an AndroidManifest. It may be renamed when using aab.
if (string.Compare (Path.GetFileName (entry), "AndroidManifest.xml", StringComparison.OrdinalIgnoreCase) == 0)
continue;
Log.LogDebugMessage ($"Removing {entry} as it is not longer required.");
apk.Archive.DeleteEntry (entry);
}
apk.Flush ();
FixupArchive (apk);

OutputApkFiles = apk.ApkFiles.ToArray ();

}
}

Expand Down Expand Up @@ -363,24 +302,11 @@ public override bool RunTask ()
DSOWrapperGenerator.Config dsoWrapperConfig = DSOWrapperGenerator.GetConfig (Log, AndroidBinUtilsDirectory, IntermediateOutputPath);
ExecuteWithAbi (dsoWrapperConfig, SupportedAbis, ApkInputPath, ApkOutputPath, debug, compress, compressedAssembliesInfo, assemblyStoreApkName: null);
outputFiles.Add (ApkOutputPath);
if (CreatePackagePerAbi && SupportedAbis.Length > 1) {
var abiArray = new string[] { String.Empty };
foreach (var abi in SupportedAbis) {
existingEntries.Clear ();
var path = Path.GetDirectoryName (ApkOutputPath);
var apk = Path.GetFileNameWithoutExtension (ApkOutputPath);
abiArray[0] = abi;
ExecuteWithAbi (dsoWrapperConfig, abiArray, String.Format ("{0}-{1}", ApkInputPath, abi),
Path.Combine (path, String.Format ("{0}-{1}.apk", apk, abi)),
debug, compress, compressedAssembliesInfo, assemblyStoreApkName: abi);
outputFiles.Add (Path.Combine (path, String.Format ("{0}-{1}.apk", apk, abi)));
}
}

OutputFiles = outputFiles.Select (a => new TaskItem (a)).ToArray ();

Log.LogDebugTaskItems (" [Output] OutputFiles :", OutputFiles);
DSOWrapperGenerator.CleanUp (dsoWrapperConfig);
DSODirectoriesToDelete = DSOWrapperGenerator.GetDirectoriesToCleanUp (dsoWrapperConfig).Select (d => new TaskItem (d)).ToArray ();

return !Log.HasLoggedErrors;
}
Expand All @@ -403,7 +329,7 @@ static Regex FileGlobToRegEx (string fileGlob, RegexOptions options)
return new Regex (sb.ToString (), options);
}

void AddRuntimeConfigBlob (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx apk)
void AddRuntimeConfigBlob (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveFileListBuilder apk)
{
// We will place rc.bin in the `lib` directory next to the blob, to make startup slightly faster, as we will find the config file right after we encounter
// our assembly store. Not only that, but also we'll be able to skip scanning the `base.apk` archive when split configs are enabled (which they are in 99%
Expand All @@ -420,7 +346,7 @@ void AddRuntimeConfigBlob (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchi
}
}

void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx apk, bool debug, bool compress, IDictionary<AndroidTargetArch, Dictionary<string, CompressedAssemblyInfo>> compressedAssembliesInfo, string assemblyStoreApkName)
void AddAssemblies (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveFileListBuilder apk, bool debug, bool compress, IDictionary<AndroidTargetArch, Dictionary<string, CompressedAssemblyInfo>> compressedAssembliesInfo, string assemblyStoreApkName)
{
string sourcePath;
AssemblyCompression.AssemblyData compressedAssembly = null;
Expand Down Expand Up @@ -519,7 +445,7 @@ string CompressAssembly (ITaskItem assembly)
}
}

bool AddFileToArchiveIfNewer (ZipArchiveEx apk, string file, string inArchivePath, CompressionMethod compressionMethod = CompressionMethod.Default)
bool AddFileToArchiveIfNewer (ZipArchiveFileListBuilder apk, string file, string inArchivePath, CompressionMethod compressionMethod = CompressionMethod.Default)
{
existingEntries.Remove (inArchivePath.Replace (Path.DirectorySeparatorChar, '/'));
if (apk.SkipExistingFile (file, inArchivePath, compressionMethod)) {
Expand All @@ -531,7 +457,7 @@ bool AddFileToArchiveIfNewer (ZipArchiveEx apk, string file, string inArchivePat
return true;
}

void AddAssemblyConfigEntry (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveEx apk, AndroidTargetArch arch, string assemblyPath, string configFile)
void AddAssemblyConfigEntry (DSOWrapperGenerator.Config dsoWrapperConfig, ZipArchiveFileListBuilder apk, AndroidTargetArch arch, string assemblyPath, string configFile)
{
string inArchivePath = MonoAndroidHelper.MakeDiscreteAssembliesEntryName (assemblyPath + Path.GetFileName (configFile));
existingEntries.Remove (inArchivePath);
Expand Down Expand Up @@ -607,7 +533,7 @@ CompressionMethod GetCompressionMethod (string fileName)
return uncompressedFileExtensions.Contains (Path.GetExtension (fileName)) ? UncompressedMethod : CompressionMethod.Default;
}

void AddNativeLibraryToArchive (ZipArchiveEx apk, string abi, string filesystemPath, string inArchiveFileName, ITaskItem taskItem)
void AddNativeLibraryToArchive (ZipArchiveFileListBuilder apk, string abi, string filesystemPath, string inArchiveFileName, ITaskItem taskItem)
{
string archivePath = MakeArchiveLibPath (abi, inArchiveFileName);
existingEntries.Remove (archivePath);
Expand All @@ -618,10 +544,10 @@ void AddNativeLibraryToArchive (ZipArchiveEx apk, string abi, string filesystemP
}
Log.LogDebugMessage ($"Adding native library: {filesystemPath} (APK path: {archivePath})");
ELFHelper.AssertValidLibraryAlignment (Log, ZipAlignmentPages, filesystemPath, taskItem);
apk.AddEntryAndFlush (archivePath, File.OpenRead (filesystemPath), compressionMethod);
apk.AddFileAndFlush (filesystemPath, archivePath, compressionMethod);
}

void AddRuntimeLibraries (ZipArchiveEx apk, string [] supportedAbis)
void AddRuntimeLibraries (ZipArchiveFileListBuilder apk, string [] supportedAbis)
{
foreach (var abi in supportedAbis) {
foreach (ITaskItem item in ApplicationSharedLibraries) {
Expand Down
Loading

0 comments on commit 894ecea

Please sign in to comment.