Native runtime reorganization (#9113)

Context: #9006

This commit prepares ground for dynamic native runtime linking at application build time by
moving code around and making it more modular, thus allowing the on-build linking of the 
runtime to work properly. 

In particular, this applies to the `p/invoke` handling code which is going to be different between the
pre-linked (and shipped) runtime and the one linked on demand during application build.  The difference
lies in the fact that on-demand build will scan the application for p/invoke usage (after trimming) and will
only include functions that are actually called from the managed land.   Results of the scan will be used to
generate LLVM IR code at build time that references the required functions, which will allow the native linker 
to remove the remaining ones. 

Changes implemented here can be committed to `main` and, possibly, released with .NET9
without changing how things currently work.

Author: grendello

build-tools/scripts/generate-pinvoke-tables.sh

@@ -3,7 +3,7 @@ MY_DIR="$(dirname $0)"
 HOST="$(uname | tr A-Z a-z)"
 
 NATIVE_DIR="${MY_DIR}/../../src/native"
-MONODROID_SOURCE_DIR="${NATIVE_DIR}/monodroid"
+MONODROID_SOURCE_DIR="${NATIVE_DIR}/pinvoke-override"
 GENERATOR_SOURCE="${MONODROID_SOURCE_DIR}/generate-pinvoke-tables.cc"
 GENERATOR_BINARY="${MONODROID_SOURCE_DIR}/generate-pinvoke-tables"
 TARGET_FILE="${MONODROID_SOURCE_DIR}/pinvoke-tables.include"

src/native/CMakeLists.txt

@@ -145,7 +145,7 @@ file(REAL_PATH "../../" REPO_ROOT_DIR)
 set(EXTERNAL_DIR "${REPO_ROOT_DIR}/external")
 set(JAVA_INTEROP_SRC_PATH "${EXTERNAL_DIR}/Java.Interop/src/java-interop")
 set(LIBUNWIND_SOURCE_DIR "${EXTERNAL_DIR}/libunwind")
-
+set(ROBIN_MAP_DIR "${EXTERNAL_DIR}/robin-map")
 
 #
 # Include directories
@@ -449,6 +449,7 @@ add_subdirectory(java-interop)
 add_subdirectory(xamarin-app-stub)
 add_subdirectory(runtime-base)
 add_subdirectory(tracing)
+add_subdirectory(pinvoke-override)
 
 if(DEBUG_BUILD)
   add_subdirectory(xamarin-debug-app-helper)

src/native/java-interop/CMakeLists.txt

@@ -37,10 +37,12 @@ target_compile_options(
   ${XA_COMMON_CXX_ARGS}
 )
 
-set_target_properties(
-  ${LIB_NAME}
-  PROPERTIES
-  ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
-)
+if(DEBUG_BUILD)
+  set_target_properties(
+    ${LIB_NAME}
+    PROPERTIES
+    ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+  )
+endif()
 
 xa_add_compile_definitions(${LIB_NAME})

src/native/libstub/CMakeLists.txt

@@ -38,6 +38,7 @@ endmacro()
 
 xa_add_stub_library(c)
 xa_add_stub_library(m)
+xa_add_stub_library(z)
 
 # These two are used by the marshal methods tracing library when linking libxamarin-app.so
 xa_add_stub_library(log)

src/native/lz4/CMakeLists.txt

@@ -30,10 +30,12 @@ target_include_directories(
   "$<BUILD_INTERFACE:${LZ4_INCLUDE_DIR}>"
 )
 
-set_target_properties(
-  ${LIB_NAME}
-  PROPERTIES
-  ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
-)
+if(DEBUG_BUILD)
+  set_target_properties(
+    ${LIB_NAME}
+    PROPERTIES
+    ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}"
+  )
+endif()
 
 xa_add_compile_definitions(${LIB_NAME})

src/native/monodroid/CMakeLists.txt

@@ -8,7 +8,7 @@ include(CheckCXXSymbolExists)
 # Paths
 
 set(BIONIC_SOURCES_DIR "${REPO_ROOT_DIR}/src-ThirdParty/bionic")
-set(ROBIN_MAP_DIR "${EXTERNAL_DIR}/robin-map")
+set(BUILD_STATIC_LIBRARY OFF)
 
 if(DEBUG_BUILD)
   # Convince NDK to really optimize our Debug builds. Without this, NDK's cmake toolchain definition
@@ -17,6 +17,8 @@ if(DEBUG_BUILD)
   set(XA_COMPILER_FLAGS_DEBUG "-fno-limit-debug-info -O2")
   set(CMAKE_C_FLAGS_DEBUG ${XA_COMPILER_FLAGS_DEBUG})
   set(CMAKE_CXX_FLAGS_DEBUG ${XA_COMPILER_FLAGS_DEBUG})
+elseif(NOT ANALYZERS_ENABLED)
+  set(BUILD_STATIC_LIBRARY ON)
 endif()
 
 # Library directories
@@ -42,13 +44,15 @@ endif()
 # Sources
 string(TOLOWER ${CMAKE_BUILD_TYPE} XAMARIN_MONO_ANDROID_SUFFIX)
 set(XAMARIN_MONO_ANDROID_LIB "mono-android${CHECKED_BUILD_INFIX}.${XAMARIN_MONO_ANDROID_SUFFIX}")
+set(XAMARIN_MONO_ANDROID_STATIC_LIB "${XAMARIN_MONO_ANDROID_LIB}-static")
 
 set(XAMARIN_MONODROID_SOURCES
   debug-constants.cc
   debug.cc
   embedded-assemblies-zip.cc
   embedded-assemblies.cc
   globals.cc
+  internal-pinvokes.cc
   jni-remapping.cc
   mono-log-adapter.cc
   monodroid-glue.cc
@@ -58,9 +62,7 @@ set(XAMARIN_MONODROID_SOURCES
   osbridge.cc
   pinvoke-override-api.cc
   runtime-util.cc
-  timing.cc
   timezones.cc
-  timing-internal.cc
   xamarin_getifaddrs.cc
 )
 
@@ -93,90 +95,107 @@ add_library(
   SHARED ${XAMARIN_MONODROID_SOURCES}
 )
 
-target_compile_definitions(
-  ${XAMARIN_MONO_ANDROID_LIB}
-  PRIVATE
-  HAVE_CONFIG_H
-  HAVE_LZ4
-  JI_DLL_EXPORT
-  JI_NO_VISIBILITY
-  MONO_DLL_EXPORT
-  NET
-  TSL_NO_EXCEPTIONS
-)
-
-if(DONT_INLINE)
-  target_compile_definitions(
-    ${XAMARIN_MONO_ANDROID_LIB}
-    PRIVATE
-    NO_INLINE
+if(BUILD_STATIC_LIBRARY)
+  add_library(
+    ${XAMARIN_MONO_ANDROID_STATIC_LIB}
+    STATIC
+    ${XAMARIN_MONODROID_SOURCES}
   )
 endif()
 
-if(DEBUG_BUILD AND NOT DISABLE_DEBUG)
+macro(lib_target_options TARGET_NAME)
   target_compile_definitions(
-    ${XAMARIN_MONO_ANDROID_LIB}
+    ${TARGET_NAME}
     PRIVATE
-    DEBUG
+    HAVE_CONFIG_H
+    HAVE_LZ4
+    JI_DLL_EXPORT
+    JI_NO_VISIBILITY
+    MONO_DLL_EXPORT
+    NET
+    TSL_NO_EXCEPTIONS
   )
-endif()
 
-if (ENABLE_TIMING)
-  target_compile_definitions(
-    ${XAMARIN_MONO_ANDROID_LIB}
+  if(DONT_INLINE)
+    target_compile_definitions(
+      ${TARGET_NAME}
+      PRIVATE
+      NO_INLINE
+    )
+  endif()
+
+  if(DEBUG_BUILD AND NOT DISABLE_DEBUG)
+    target_compile_definitions(
+      ${TARGET_NAME}
+      PRIVATE
+      DEBUG
+    )
+  endif()
+
+  if (ENABLE_TIMING)
+    target_compile_definitions(
+      ${TARGET_NAME}
+      PRIVATE
+      MONODROID_TIMING
+    )
+  endif()
+
+  target_compile_options(
+    ${TARGET_NAME}
     PRIVATE
-    MONODROID_TIMING
+    ${XA_DEFAULT_SYMBOL_VISIBILITY}
+    ${XA_COMMON_CXX_ARGS}
   )
-endif()
 
-target_compile_options(
-  ${XAMARIN_MONO_ANDROID_LIB}
-  PRIVATE
-  ${XA_DEFAULT_SYMBOL_VISIBILITY}
-  ${XA_COMMON_CXX_ARGS}
-)
-
-target_include_directories(
-  ${XAMARIN_MONO_ANDROID_LIB} BEFORE
-  PRIVATE
-  ${CMAKE_CURRENT_BINARY_DIR}/include
-  ${EXTERNAL_DIR}
-  ${ROBIN_MAP_DIR}/include
-)
+  target_include_directories(
+    ${TARGET_NAME} BEFORE
+    PRIVATE
+    ${CMAKE_CURRENT_BINARY_DIR}/include
+    ${EXTERNAL_DIR}
+    ${ROBIN_MAP_DIR}/include
+  )
 
-target_include_directories(
-  ${XAMARIN_MONO_ANDROID_LIB}
-  SYSTEM PRIVATE
-  ${SYSROOT_CXX_INCLUDE_DIR}
-  ${MONO_RUNTIME_INCLUDE_DIR}
-  ${NATIVE_TRACING_INCLUDE_DIRS}
-  ${LIBUNWIND_INCLUDE_DIRS}
-)
+  target_include_directories(
+    ${TARGET_NAME}
+    SYSTEM PRIVATE
+    ${SYSROOT_CXX_INCLUDE_DIR}
+    ${MONO_RUNTIME_INCLUDE_DIR}
+    ${NATIVE_TRACING_INCLUDE_DIRS}
+    ${LIBUNWIND_INCLUDE_DIRS}
+  )
 
-target_link_directories(
-  ${XAMARIN_MONO_ANDROID_LIB}
-  PRIVATE
-  ${NET_RUNTIME_DIR}/native
-)
+  target_link_directories(
+    ${TARGET_NAME}
+    PRIVATE
+    ${NET_RUNTIME_DIR}/native
+  )
 
-target_link_options(
-  ${XAMARIN_MONO_ANDROID_LIB}
-  PRIVATE
-  ${XA_DEFAULT_SYMBOL_VISIBILITY}
-  ${XA_COMMON_CXX_LINKER_ARGS}
-  ${XA_CXX_DSO_LINKER_ARGS}
-)
+  target_link_options(
+    ${TARGET_NAME}
+    PRIVATE
+    ${XA_DEFAULT_SYMBOL_VISIBILITY}
+    ${XA_COMMON_CXX_LINKER_ARGS}
+    ${XA_CXX_DSO_LINKER_ARGS}
+  )
 
-target_link_libraries(
-  ${XAMARIN_MONO_ANDROID_LIB}
-  ${LINK_LIBS}
-  xa::xamarin-app
-  ${SHARED_LIB_NAME}
-  xa::runtime-base
-  xa::java-interop
-  xa::lz4
-  -lmonosgen-2.0
-  -llog
-)
+  target_link_libraries(
+    ${TARGET_NAME}
+    ${LINK_LIBS}
+    xa::xamarin-app
+    ${SHARED_LIB_NAME}
+    xa::runtime-base
+    xa::java-interop
+    xa::pinvoke-override-precompiled
+    xa::lz4
+    -lmonosgen-2.0
+    -llog
+  )
+endmacro ()
 
+lib_target_options(${XAMARIN_MONO_ANDROID_LIB})
 xa_add_compile_definitions(${XAMARIN_MONO_ANDROID_LIB})
+
+if(BUILD_STATIC_LIBRARY)
+  lib_target_options(${XAMARIN_MONO_ANDROID_STATIC_LIB})
+  xa_add_compile_definitions(${XAMARIN_MONO_ANDROID_STATIC_LIB})
+endif()

src/native/monodroid/embedded-assemblies.cc

@@ -33,6 +33,7 @@
 #include "xamarin-app.hh"
 #include "cpp-util.hh"
 #include "monodroid-glue-internal.hh"
+#include "monodroid-state.hh"
 #include "startup-aware-lock.hh"
 #include "timing-internal.hh"
 #include "search.hh"
@@ -179,7 +180,7 @@ EmbeddedAssemblies::map_runtime_file (XamarinAndroidBundledAssembly& file) noexc
 		close (fd);
 	}
 
-	if (MonodroidRuntime::is_startup_in_progress ()) {
+	if (MonodroidState::is_startup_in_progress ()) {
 		file.data = static_cast<uint8_t*>(map_info.area);
 	} else {
 		uint8_t *expected_null = nullptr;