VirtualBox

source: vbox/trunk/src/VBox/Devices/EFI/FirmwareNew/BaseTools/Edk2ToolsBuild.py@ 108794

Last change on this file since 108794 was 108794, checked in by vboxsync, 7 days ago

Devices/EFI/FirmwareNew: Merge edk2-stable202502 from the vendor branch and make it build for the important platforms, bugref:4643

  • Property svn:eol-style set to native
File size: 16.6 KB
Line 
1# @file Edk2ToolsBuild.py
2# Invocable class that builds the basetool c files.
3#
4# Supports VS2019, VS2022, and GCC5
5##
6# Copyright (c) Microsoft Corporation
7#
8# SPDX-License-Identifier: BSD-2-Clause-Patent
9##
10import os
11import sys
12import logging
13import argparse
14import multiprocessing
15import shutil
16from edk2toolext import edk2_logging
17from edk2toolext.environment import self_describing_environment, shell_environment
18from edk2toolext.base_abstract_invocable import BaseAbstractInvocable
19from edk2toollib.utility_functions import RunCmd, GetHostInfo
20from edk2toollib.windows.locate_tools import QueryVcVariables
21
22
23class Edk2ToolsBuild(BaseAbstractInvocable):
24
25 def ParseCommandLineOptions(self):
26 ''' parse arguments '''
27 ParserObj = argparse.ArgumentParser()
28 ParserObj.add_argument("-t", "--tool_chain_tag", dest="tct", default="VS2022",
29 help="Set the toolchain used to compile the build tools")
30 ParserObj.add_argument("-a", "--target_arch", dest="arch", default=None, choices=[None, 'IA32', 'X64', 'ARM', 'AARCH64'],
31 help="Specify the architecture of the built base tools. Not specifying this will fall back to the default "
32 "behavior, for Windows builds, IA32 target will be built, for Linux builds, target arch will be the same as host arch.")
33 args = ParserObj.parse_args()
34 self.tool_chain_tag = args.tct
35 self.target_arch = args.arch
36
37 def GetWorkspaceRoot(self):
38 ''' Return the workspace root for initializing the SDE '''
39
40 # this is the bastools dir...not the traditional EDK2 workspace root
41 return os.path.dirname(os.path.abspath(__file__))
42
43 def GetActiveScopes(self):
44 ''' return tuple containing scopes that should be active for this process '''
45
46 # Adding scope for cross compilers when building for ARM/AARCH64
47 scopes = ('global',)
48 if GetHostInfo().os == "Linux" and self.tool_chain_tag.lower().startswith("gcc"):
49 if self.target_arch is None:
50 return scopes
51 if "AARCH64" in self.target_arch:
52 scopes += ("gcc_aarch64_linux",)
53 if "ARM" in self.target_arch:
54 scopes += ("gcc_arm_linux",)
55 return scopes
56
57 def GetLoggingLevel(self, loggerType):
58 ''' Get the logging level for a given type (return Logging.Level)
59 base == lowest logging level supported
60 con == Screen logging
61 txt == plain text file logging
62 md == markdown file logging
63 '''
64 if(loggerType == "con"):
65 return logging.ERROR
66 else:
67 return logging.DEBUG
68
69 def GetLoggingFolderRelativeToRoot(self):
70 ''' Return a path to folder for log files '''
71 return "BaseToolsBuild"
72
73 def GetVerifyCheckRequired(self):
74 ''' Will call self_describing_environment.VerifyEnvironment if this returns True '''
75 return True
76
77 def GetLoggingFileName(self, loggerType):
78 ''' Get the logging file name for the type.
79 Return None if the logger shouldn't be created
80
81 base == lowest logging level supported
82 con == Screen logging
83 txt == plain text file logging
84 md == markdown file logging
85 '''
86 return "BASETOOLS_BUILD"
87
88 def WritePathEnvFile(self, OutputDir):
89 ''' Write a PyTool path env file for future PyTool based edk2 builds'''
90 content = '''##
91# Set shell variable EDK_TOOLS_BIN to this folder
92#
93# Autogenerated by Edk2ToolsBuild.py
94#
95# Copyright (c), Microsoft Corporation
96# SPDX-License-Identifier: BSD-2-Clause-Patent
97##
98{
99 "id": "You-Built-BaseTools",
100 "scope": "edk2-build",
101 "flags": ["set_shell_var", "set_path"],
102 "var_name": "EDK_TOOLS_BIN"
103}
104'''
105 with open(os.path.join(OutputDir, "basetoolsbin_path_env.yaml"), "w") as f:
106 f.write(content)
107
108 def Go(self):
109 logging.info("Running Python version: " + str(sys.version_info))
110
111 (build_env, shell_env) = self_describing_environment.BootstrapEnvironment(
112 self.GetWorkspaceRoot(), self.GetActiveScopes())
113
114 # # Bind our current execution environment into the shell vars.
115 ph = os.path.dirname(sys.executable)
116 if " " in ph:
117 ph = '"' + ph + '"'
118 shell_env.set_shell_var("PYTHON_HOME", ph)
119 # PYTHON_COMMAND is required to be set for using edk2 python builds.
120 pc = sys.executable
121 if " " in pc:
122 pc = '"' + pc + '"'
123 shell_env.set_shell_var("PYTHON_COMMAND", pc)
124
125 if self.tool_chain_tag.lower().startswith("vs"):
126 if self.target_arch is None:
127 # Put a default as IA32
128 self.target_arch = "IA32"
129
130 if self.target_arch == "IA32":
131 VcToolChainArch = "x86"
132 TargetInfoArch = "x86"
133 OutputDir = "Win32"
134 elif self.target_arch == "ARM":
135 VcToolChainArch = "x86_arm"
136 TargetInfoArch = "ARM"
137 OutputDir = "Win32"
138 elif self.target_arch == "X64":
139 VcToolChainArch = "amd64"
140 TargetInfoArch = "x86"
141 OutputDir = "Win64"
142 elif self.target_arch == "AARCH64":
143 VcToolChainArch = "amd64_arm64"
144 TargetInfoArch = "ARM"
145 OutputDir = "Win64"
146 else:
147 raise NotImplementedError()
148
149 self.OutputDir = os.path.join(
150 shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", OutputDir)
151
152 # compiled tools need to be added to path because antlr is referenced
153 HostInfo = GetHostInfo()
154 if TargetInfoArch == HostInfo.arch:
155 # not cross compiling
156 shell_env.insert_path(self.OutputDir)
157 else:
158 # cross compiling:
159 # as the VfrCompile tool is needed in the build process, we need
160 # to build one for the host system, then add the path to the
161 # tools to the PATH environment variable
162 shell_environment.CheckpointBuildVars()
163 if HostInfo.arch == "x86" and HostInfo.bit == "64":
164 host_arch = "X64"
165 host_toolchain_arch = "amd64"
166 TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64")
167 elif HostInfo.arch == "x86" and HostInfo.bit == "32":
168 host_arch = "IA32"
169 host_toolchain_arch = "x86"
170 TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")
171 elif HostInfo.arch == "ARM" and HostInfo.bit == "64":
172 host_arch = "AARCH64"
173 host_toolchain_arch = "amd64_arm64"
174 TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64")
175 elif HostInfo.arch == "ARM" and HostInfo.bit == "32":
176 host_arch = "ARM"
177 host_toolchain_arch = "x86_arm"
178 TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")
179 else:
180 raise Exception("Unsupported host system. %s %s" % (HostInfo.arch, HostInfo.bit))
181
182 interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]
183 interesting_keys.extend(
184 ["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"])
185 interesting_keys.extend(
186 ["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])
187 vc_vars = QueryVcVariables(
188 interesting_keys, host_toolchain_arch, vs_version=self.tool_chain_tag.lower())
189 for key in vc_vars.keys():
190 logging.debug(f"Var - {key} = {vc_vars[key]}")
191 if key.lower() == 'path':
192 shell_env.set_path(vc_vars[key])
193 else:
194 shell_env.set_shell_var(key, vc_vars[key])
195
196 # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
197 # this script is BUILDING the base tools.
198 shell_env.set_shell_var('HOST_ARCH', host_arch)
199 shell_env.insert_path(TempOutputDir)
200
201 # All set, build the tools for the host system.
202 ret = RunCmd('nmake.exe', None,
203 workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
204 if ret != 0:
205 raise Exception("Failed to build base tools for host system.")
206
207 # Copy the output to a temp directory
208 TempFolder = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "BaseToolsBuild", "Temp")
209 if not os.path.exists(TempFolder):
210 os.makedirs(TempFolder)
211 for file in os.listdir(TempOutputDir):
212 shutil.copy(os.path.join(TempOutputDir, file), TempFolder)
213
214 # Clean up the build output
215 ret = RunCmd('nmake.exe', 'cleanall',
216 workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
217
218 # Remove the entire TempOutputDir
219 shutil.rmtree(TempOutputDir)
220
221 shell_environment.RevertBuildVars()
222 shell_env.insert_path(TempFolder)
223
224 # # Update environment with required VC vars.
225 interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]
226 interesting_keys.extend(
227 ["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"])
228 interesting_keys.extend(
229 ["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])
230 vc_vars = QueryVcVariables(
231 interesting_keys, VcToolChainArch, vs_version=self.tool_chain_tag.lower())
232 for key in vc_vars.keys():
233 logging.debug(f"Var - {key} = {vc_vars[key]}")
234 if key.lower() == 'path':
235 shell_env.set_path(vc_vars[key])
236 else:
237 shell_env.set_shell_var(key, vc_vars[key])
238
239 # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
240 # this script is BUILDING the base tools.
241 shell_env.set_shell_var('HOST_ARCH', self.target_arch)
242
243 # Actually build the tools.
244 output_stream = edk2_logging.create_output_stream()
245 ret = RunCmd('nmake.exe', None,
246 workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
247 edk2_logging.remove_output_stream(output_stream)
248 problems = edk2_logging.scan_compiler_output(output_stream)
249 for level, problem in problems:
250 logging.log(level, problem)
251 if ret != 0:
252 raise Exception("Failed to build.")
253
254 self.WritePathEnvFile(self.OutputDir)
255 return ret
256
257 elif self.tool_chain_tag.lower().startswith("gcc"):
258 # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
259 # this script is BUILDING the base tools.
260 HostInfo = GetHostInfo()
261 prefix = None
262 TargetInfoArch = None
263 if self.target_arch is not None:
264 shell_env.set_shell_var('HOST_ARCH', self.target_arch)
265
266 if "AARCH64" in self.target_arch:
267 prefix = shell_env.get_shell_var("GCC5_AARCH64_PREFIX")
268 if prefix == None:
269 # now check for install dir. If set then set the Prefix
270 install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_INSTALL")
271
272 # make GCC5_AARCH64_PREFIX to align with tools_def.txt
273 prefix = os.path.join(install_path, "bin", "aarch64-none-linux-gnu-")
274
275 shell_environment.GetEnvironment().set_shell_var("GCC_PREFIX", prefix)
276 TargetInfoArch = "ARM"
277
278 elif "ARM" in self.target_arch:
279 prefix = shell_env.get_shell_var("GCC5_ARM_PREFIX")
280 if prefix == None:
281 # now check for install dir. If set then set the Prefix
282 install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_INSTALL")
283
284 # make GCC5_ARM_PREFIX to align with tools_def.txt
285 prefix = os.path.join(install_path, "bin", "arm-none-linux-gnueabihf-")
286
287 shell_environment.GetEnvironment().set_shell_var("GCC_PREFIX", prefix)
288 TargetInfoArch = "ARM"
289
290 else:
291 TargetInfoArch = "x86"
292 else:
293 self.target_arch = HostInfo.arch
294 TargetInfoArch = HostInfo.arch
295 # Otherwise, the built binary arch will be consistent with the host system
296
297 # Added logic to support cross compilation scenarios
298 if TargetInfoArch != HostInfo.arch:
299 # this is defaulting to the version that comes with Ubuntu 20.04
300 ver = shell_environment.GetBuildVars().GetValue("LIBUUID_VERSION", "2.34")
301 work_dir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), self.GetLoggingFolderRelativeToRoot())
302 pack_name = f"util-linux-{ver}"
303 unzip_dir = os.path.join(work_dir, pack_name)
304
305 if os.path.isfile(os.path.join(work_dir, f"{pack_name}.tar.gz")):
306 os.remove(os.path.join(work_dir, f"{pack_name}.tar.gz"))
307 if os.path.isdir(unzip_dir):
308 shutil.rmtree(unzip_dir)
309
310 # cross compiling, need to rebuild libuuid for the target
311 ret = RunCmd("wget", f"https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v{ver}/{pack_name}.tar.gz", workingdir=work_dir)
312 if ret != 0:
313 raise Exception(f"Failed to download libuuid version {ver} - {ret}")
314
315 ret = RunCmd("tar", f"xvzf {pack_name}.tar.gz", workingdir=work_dir)
316 if ret != 0:
317 raise Exception(f"Failed to untar the downloaded file {ret}")
318
319 # configure the source to use the cross compiler
320 pack_name = f"util-linux-{ver}"
321 if "AARCH64" in self.target_arch:
322 ret = RunCmd("sh", f"./configure --host=aarch64-linux -disable-all-programs --enable-libuuid CC={prefix}gcc", workingdir=unzip_dir)
323 elif "ARM" in self.target_arch:
324 ret = RunCmd("sh", f"./configure --host=arm-linux -disable-all-programs --enable-libuuid CC={prefix}gcc", workingdir=unzip_dir)
325 if ret != 0:
326 raise Exception(f"Failed to configure the util-linux to build with our gcc {ret}")
327
328 ret = RunCmd("make", "", workingdir=unzip_dir)
329 if ret != 0:
330 raise Exception(f"Failed to build the libuuid with our gcc {ret}")
331
332 shell_environment.GetEnvironment().set_shell_var("CROSS_LIB_UUID", unzip_dir)
333 shell_environment.GetEnvironment().set_shell_var("CROSS_LIB_UUID_INC", os.path.join(unzip_dir, "libuuid", "src"))
334
335 ret = RunCmd("make", "clean", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
336 if ret != 0:
337 raise Exception("Failed to build.")
338
339 cpu_count = self.GetCpuThreads()
340
341 output_stream = edk2_logging.create_output_stream()
342 ret = RunCmd("make", f"-C . -j {cpu_count}", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
343 edk2_logging.remove_output_stream(output_stream)
344 problems = edk2_logging.scan_compiler_output(output_stream)
345 for level, problem in problems:
346 logging.log(level, problem)
347 if ret != 0:
348 raise Exception("Failed to build.")
349
350 self.OutputDir = os.path.join(
351 shell_env.get_shell_var("EDK_TOOLS_PATH"), "Source", "C", "bin")
352
353 self.WritePathEnvFile(self.OutputDir)
354 return ret
355
356 logging.critical("Tool Chain not supported")
357 return -1
358
359 def GetCpuThreads(self) -> int:
360 ''' Function to return number of cpus. If error return 1'''
361 cpus = 1
362 try:
363 cpus = multiprocessing.cpu_count()
364 except:
365 # from the internet there are cases where cpu_count is not implemented.
366 # will handle error by just doing single proc build
367 pass
368 return cpus
369
370
371
372def main():
373 Edk2ToolsBuild().Invoke()
374
375
376if __name__ == "__main__":
377 main()
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette