BaseTools: Adding cross compilation of BaseTool for Windows ARM/ARM64

This change adds the support of crossbuilding basetool for Windows ARM/
ARM64 systems, which will enable the generally available pipeline agents
to build binary tools and make releases as they see fit.

The EDK2 base tools build script is also updated to support cross
compilation using this script.

The crossbuilt binary output is tested on Windows ARM based hardware
systems.

Cc: Rebecca Cran <rebecca@bsdio.com>
Cc: Liming Gao <gaoliming@byosoft.com.cn>
Cc: Bob Feng <bob.c.feng@intel.com>
Cc: Yuwei Chen <yuwei.chen@intel.com>
Signed-off-by: Kun Qin <kun.qin@microsoft.com>

5 files changed, 135 insertions, 14 deletions

diff --git a/BaseTools/.gitignore b/BaseTools/.gitignore
index ec9c69f8b2..0186128250 100644
--- a/BaseTools/.gitignore
+++ b/BaseTools/.gitignore
@@ -17,5 +17,6 @@ Source/C/VfrCompile/VfrTokens.h
 Source/C/bin/

 Source/C/libs/

 Bin/Win32

+Bin/Win64

 Lib

 BaseToolsBuild/
\ No newline at end of file
diff --git a/BaseTools/Edk2ToolsBuild.py b/BaseTools/Edk2ToolsBuild.py
index 2a8a53b0d6..d511b1a2a3 100644
--- a/BaseTools/Edk2ToolsBuild.py
+++ b/BaseTools/Edk2ToolsBuild.py
@@ -12,10 +12,11 @@ import sys
 import logging

 import argparse

 import multiprocessing

+import shutil

 from edk2toolext import edk2_logging

-from edk2toolext.environment import self_describing_environment

+from edk2toolext.environment import self_describing_environment, shell_environment

 from edk2toolext.base_abstract_invocable import BaseAbstractInvocable

-from edk2toollib.utility_functions import RunCmd

+from edk2toollib.utility_functions import RunCmd, GetHostInfo

 from edk2toollib.windows.locate_tools import QueryVcVariables

 

 

@@ -26,8 +27,12 @@ class Edk2ToolsBuild(BaseAbstractInvocable):
         ParserObj = argparse.ArgumentParser()

         ParserObj.add_argument("-t", "--tool_chain_tag", dest="tct", default="VS2022",

                                help="Set the toolchain used to compile the build tools")

+        ParserObj.add_argument("-a", "--target_arch", dest="arch", default=None, choices=[None, 'IA32', 'X64', 'ARM', 'AARCH64'],

+                               help="Specify the architecture of the built base tools. Not specifying this will fall back to the default "

+                               "behavior, for Windows builds, IA32 target will be built, for Linux builds, target arch will be the same as host arch.")

         args = ParserObj.parse_args()

         self.tool_chain_tag = args.tct

+        self.target_arch = args.arch

 

     def GetWorkspaceRoot(self):

         ''' Return the workspace root for initializing the SDE '''

@@ -110,6 +115,103 @@ class Edk2ToolsBuild(BaseAbstractInvocable):
         shell_env.set_shell_var("PYTHON_COMMAND", pc)

 

         if self.tool_chain_tag.lower().startswith("vs"):

+            if self.target_arch is None:

+                # Put a default as IA32

+                self.target_arch = "IA32"

+

+            if self.target_arch == "IA32":

+                VcToolChainArch = "x86"

+                TargetInfoArch = "x86"

+                OutputDir = "Win32"

+            elif self.target_arch == "ARM":

+                VcToolChainArch = "x86_arm"

+                TargetInfoArch = "ARM"

+                OutputDir = "Win32"

+            elif self.target_arch == "X64":

+                VcToolChainArch = "amd64"

+                TargetInfoArch = "x86"

+                OutputDir = "Win64"

+            elif self.target_arch == "AARCH64":

+                VcToolChainArch = "amd64_arm64"

+                TargetInfoArch = "ARM"

+                OutputDir = "Win64"

+            else:

+                raise NotImplementedError()

+

+            self.OutputDir = os.path.join(

+                shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", OutputDir)

+

+            # compiled tools need to be added to path because antlr is referenced

+            HostInfo = GetHostInfo()

+            if TargetInfoArch == HostInfo.arch:

+                # not cross compiling

+                shell_env.insert_path(self.OutputDir)

+            else:

+                # cross compiling:

+                # as the VfrCompile tool is needed in the build process, we need

+                # to build one for the host system, then add the path to the

+                # tools to the PATH environment variable

+                shell_environment.CheckpointBuildVars()

+                if HostInfo.arch == "x86" and HostInfo.bit == "64":

+                    host_arch = "X64"

+                    host_toolchain_arch = "amd64"

+                    TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64")

+                elif HostInfo.arch == "x86" and HostInfo.bit == "32":

+                    host_arch = "IA32"

+                    host_toolchain_arch = "x86"

+                    TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")

+                elif HostInfo.arch == "ARM" and HostInfo.bit == "64":

+                    host_arch = "AARCH64"

+                    host_toolchain_arch = "amd64_arm64"

+                    TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64")

+                elif HostInfo.arch == "ARM" and HostInfo.bit == "32":

+                    host_arch = "ARM"

+                    host_toolchain_arch = "x86_arm"

+                    TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")

+                else:

+                    raise Exception("Unsupported host system. %s %s" % (HostInfo.arch, HostInfo.bit))

+

+                interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]

+                interesting_keys.extend(

+                    ["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"])

+                interesting_keys.extend(

+                    ["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])

+                vc_vars = QueryVcVariables(

+                    interesting_keys, host_toolchain_arch, vs_version=self.tool_chain_tag.lower())

+                for key in vc_vars.keys():

+                    logging.debug(f"Var - {key} = {vc_vars[key]}")

+                    if key.lower() == 'path':

+                        shell_env.set_path(vc_vars[key])

+                    else:

+                        shell_env.set_shell_var(key, vc_vars[key])

+

+                # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where

+                # this script is BUILDING the base tools.

+                shell_env.set_shell_var('HOST_ARCH', host_arch)

+                shell_env.insert_path(TempOutputDir)

+

+                # All set, build the tools for the host system.

+                ret = RunCmd('nmake.exe', None,

+                             workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))

+                if ret != 0:

+                    raise Exception("Failed to build base tools for host system.")

+

+                # Copy the output to a temp directory

+                TempFolder = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "BaseToolsBuild", "Temp")

+                if not os.path.exists(TempFolder):

+                    os.makedirs(TempFolder)

+                for file in os.listdir(TempOutputDir):

+                    shutil.copy(os.path.join(TempOutputDir, file), TempFolder)

+

+                # Clean up the build output

+                ret = RunCmd('nmake.exe', 'cleanall',

+                             workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))

+

+                # Remove the entire TempOutputDir

+                shutil.rmtree(TempOutputDir)

+

+                shell_environment.RevertBuildVars()

+                shell_env.insert_path(TempFolder)

 

             # # Update environment with required VC vars.

             interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]

@@ -118,7 +220,7 @@ class Edk2ToolsBuild(BaseAbstractInvocable):
             interesting_keys.extend(

                 ["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])

             vc_vars = QueryVcVariables(

-                interesting_keys, 'x86', vs_version=self.tool_chain_tag.lower())

+                interesting_keys, VcToolChainArch, vs_version=self.tool_chain_tag.lower())

             for key in vc_vars.keys():

                 logging.debug(f"Var - {key} = {vc_vars[key]}")

                 if key.lower() == 'path':

@@ -126,11 +228,9 @@ class Edk2ToolsBuild(BaseAbstractInvocable):
                 else:

                     shell_env.set_shell_var(key, vc_vars[key])

 

-            self.OutputDir = os.path.join(

-                shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")

-

-            # compiled tools need to be added to path because antlr is referenced

-            shell_env.insert_path(self.OutputDir)

+            # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where

+            # this script is BUILDING the base tools.

+            shell_env.set_shell_var('HOST_ARCH', self.target_arch)

 

             # Actually build the tools.

             output_stream = edk2_logging.create_output_stream()

diff --git a/BaseTools/Source/C/Makefile b/BaseTools/Source/C/Makefile
index a376d32e22..0f26de0aa1 100644
--- a/BaseTools/Source/C/Makefile
+++ b/BaseTools/Source/C/Makefile
@@ -4,7 +4,9 @@
 # Copyright (c) 2009 - 2018, Intel Corporation. All rights reserved.<BR>

 # SPDX-License-Identifier: BSD-2-Clause-Patent

 #

+!IFNDEF HOST_ARCH

 HOST_ARCH = IA32

+!ENDIF

 

 !INCLUDE Makefiles\ms.common

 

diff --git a/BaseTools/Source/C/VfrCompile/Pccts/antlr/AntlrMS.mak b/BaseTools/Source/C/VfrCompile/Pccts/antlr/AntlrMS.mak
index 6fc4d5c15d..26c590adf4 100644
--- a/BaseTools/Source/C/VfrCompile/Pccts/antlr/AntlrMS.mak
+++ b/BaseTools/Source/C/VfrCompile/Pccts/antlr/AntlrMS.mak
@@ -8,6 +8,15 @@ PCCTS_HOME=$(BASE_TOOLS_PATH)\Source\C\VfrCompile\Pccts
 ANTLR_SRC=$(PCCTS_HOME)\antlr

 PCCTS_H=$(PCCTS_HOME)\h

 

+!IFNDEF HOST_ARCH

+HOST_ARCH = IA32

+!ENDIF

+

+!IF "$(HOST_ARCH)"=="IA32" || "$(HOST_ARCH)"=="ARM"

+SYS_BIN_PATH=$(EDK_TOOLS_PATH)\Bin\Win32

+!ELSE

+SYS_BIN_PATH=$(EDK_TOOLS_PATH)\Bin\Win64

+!ENDIF

 

 # Support directories

 SET=$(PCCTS_HOME)\support\set

@@ -27,10 +36,10 @@ SUPPORT_OBJS = set.obj
 

 # Dependencies

 

-$(EDK_TOOLS_PATH)\Bin\Win32\antlr.exe: $(ANTLR_OBJS) $(SUPPORT_OBJS)

+$(SYS_BIN_PATH)\antlr.exe: $(ANTLR_OBJS) $(SUPPORT_OBJS)

     $(CC) $(CFLAGS) -Feantlr.exe $(ANTLR_OBJS) $(SUPPORT_OBJS)

-    -@if not exist $(EDK_TOOLS_PATH)\Bin\Win32 mkdir $(EDK_TOOLS_PATH)\Bin\Win32

-		copy antlr.exe $(EDK_TOOLS_PATH)\Bin\Win32

+    -@if not exist $(SYS_BIN_PATH) mkdir $(SYS_BIN_PATH)

+		copy antlr.exe $(SYS_BIN_PATH)

 

 

 antlr.obj: $(ANTLR_SRC)\antlr.c \

diff --git a/BaseTools/Source/C/VfrCompile/Pccts/dlg/DlgMS.mak b/BaseTools/Source/C/VfrCompile/Pccts/dlg/DlgMS.mak
index c2cac00f53..f2a31a775e 100644
--- a/BaseTools/Source/C/VfrCompile/Pccts/dlg/DlgMS.mak
+++ b/BaseTools/Source/C/VfrCompile/Pccts/dlg/DlgMS.mak
@@ -12,6 +12,15 @@ PCCTS_H=$(PCCTS_HOME)\h
 # Support directories

 SET=$(PCCTS_HOME)\support\set

 

+!IFNDEF HOST_ARCH

+HOST_ARCH = IA32

+!ENDIF

+

+!IF "$(HOST_ARCH)"=="IA32" || "$(HOST_ARCH)"=="ARM"

+SYS_BIN_PATH=$(EDK_TOOLS_PATH)\Bin\Win32

+!ELSE

+SYS_BIN_PATH=$(EDK_TOOLS_PATH)\Bin\Win64

+!ENDIF

 

 # Compiler stuff

 CC = cl

@@ -26,10 +35,10 @@ SUPPORT_OBJS = set.obj
 

 # Dependencies

 

-$(EDK_TOOLS_PATH)\Bin\Win32\dlg.exe: $(DLG_OBJS) $(SUPPORT_OBJS)

+$(SYS_BIN_PATH)\dlg.exe: $(DLG_OBJS) $(SUPPORT_OBJS)

     $(CC) $(CFLAGS) -Fedlg.exe $(DLG_OBJS) $(SUPPORT_OBJS)

-    -@if not exist $(EDK_TOOLS_PATH)\Bin\Win32 mkdir $(EDK_TOOLS_PATH)\Bin\Win32

-		copy dlg.exe $(EDK_TOOLS_PATH)\Bin\Win32

+    -@if not exist $(SYS_BIN_PATH) mkdir $(SYS_BIN_PATH)

+		copy dlg.exe $(SYS_BIN_PATH)

 

 dlg_p.obj: $(DLG_SRC)\dlg_p.c \

 					$(PCCTS_H)\antlr.h \