1 | # @file Edk2ToolsBuild.py
|
---|
2 | # Invocable class that builds the basetool c files.
|
---|
3 | #
|
---|
4 | # Supports VS2019, VS2022, and GCC5
|
---|
5 | ##
|
---|
6 | # Copyright (c) Microsoft Corporation
|
---|
7 | #
|
---|
8 | # SPDX-License-Identifier: BSD-2-Clause-Patent
|
---|
9 | ##
|
---|
10 | import os
|
---|
11 | import sys
|
---|
12 | import logging
|
---|
13 | import argparse
|
---|
14 | import multiprocessing
|
---|
15 | import shutil
|
---|
16 | from edk2toolext import edk2_logging
|
---|
17 | from edk2toolext.environment import self_describing_environment, shell_environment
|
---|
18 | from edk2toolext.base_abstract_invocable import BaseAbstractInvocable
|
---|
19 | from edk2toollib.utility_functions import RunCmd, GetHostInfo
|
---|
20 | from edk2toollib.windows.locate_tools import QueryVcVariables
|
---|
21 |
|
---|
22 |
|
---|
23 | class Edk2ToolsBuild(BaseAbstractInvocable):
|
---|
24 |
|
---|
25 | def ParseCommandLineOptions(self):
|
---|
26 | ''' parse arguments '''
|
---|
27 | ParserObj = argparse.ArgumentParser()
|
---|
28 | ParserObj.add_argument("-t", "--tool_chain_tag", dest="tct", default="VS2022",
|
---|
29 | help="Set the toolchain used to compile the build tools")
|
---|
30 | ParserObj.add_argument("-a", "--target_arch", dest="arch", default=None, choices=[None, 'IA32', 'X64', 'ARM', 'AARCH64'],
|
---|
31 | help="Specify the architecture of the built base tools. Not specifying this will fall back to the default "
|
---|
32 | "behavior, for Windows builds, IA32 target will be built, for Linux builds, target arch will be the same as host arch.")
|
---|
33 | args = ParserObj.parse_args()
|
---|
34 | self.tool_chain_tag = args.tct
|
---|
35 | self.target_arch = args.arch
|
---|
36 |
|
---|
37 | def GetWorkspaceRoot(self):
|
---|
38 | ''' Return the workspace root for initializing the SDE '''
|
---|
39 |
|
---|
40 | # this is the bastools dir...not the traditional EDK2 workspace root
|
---|
41 | return os.path.dirname(os.path.abspath(__file__))
|
---|
42 |
|
---|
43 | def GetActiveScopes(self):
|
---|
44 | ''' return tuple containing scopes that should be active for this process '''
|
---|
45 |
|
---|
46 | # Adding scope for cross compilers when building for ARM/AARCH64
|
---|
47 | scopes = ('global',)
|
---|
48 | if GetHostInfo().os == "Linux" and self.tool_chain_tag.lower().startswith("gcc"):
|
---|
49 | if self.target_arch is None:
|
---|
50 | return scopes
|
---|
51 | if "AARCH64" in self.target_arch:
|
---|
52 | scopes += ("gcc_aarch64_linux",)
|
---|
53 | if "ARM" in self.target_arch:
|
---|
54 | scopes += ("gcc_arm_linux",)
|
---|
55 | return scopes
|
---|
56 |
|
---|
57 | def GetLoggingLevel(self, loggerType):
|
---|
58 | ''' Get the logging level for a given type (return Logging.Level)
|
---|
59 | base == lowest logging level supported
|
---|
60 | con == Screen logging
|
---|
61 | txt == plain text file logging
|
---|
62 | md == markdown file logging
|
---|
63 | '''
|
---|
64 | if(loggerType == "con"):
|
---|
65 | return logging.ERROR
|
---|
66 | else:
|
---|
67 | return logging.DEBUG
|
---|
68 |
|
---|
69 | def GetLoggingFolderRelativeToRoot(self):
|
---|
70 | ''' Return a path to folder for log files '''
|
---|
71 | return "BaseToolsBuild"
|
---|
72 |
|
---|
73 | def GetVerifyCheckRequired(self):
|
---|
74 | ''' Will call self_describing_environment.VerifyEnvironment if this returns True '''
|
---|
75 | return True
|
---|
76 |
|
---|
77 | def GetLoggingFileName(self, loggerType):
|
---|
78 | ''' Get the logging file name for the type.
|
---|
79 | Return None if the logger shouldn't be created
|
---|
80 |
|
---|
81 | base == lowest logging level supported
|
---|
82 | con == Screen logging
|
---|
83 | txt == plain text file logging
|
---|
84 | md == markdown file logging
|
---|
85 | '''
|
---|
86 | return "BASETOOLS_BUILD"
|
---|
87 |
|
---|
88 | def WritePathEnvFile(self, OutputDir):
|
---|
89 | ''' Write a PyTool path env file for future PyTool based edk2 builds'''
|
---|
90 | content = '''##
|
---|
91 | # Set shell variable EDK_TOOLS_BIN to this folder
|
---|
92 | #
|
---|
93 | # Autogenerated by Edk2ToolsBuild.py
|
---|
94 | #
|
---|
95 | # Copyright (c), Microsoft Corporation
|
---|
96 | # SPDX-License-Identifier: BSD-2-Clause-Patent
|
---|
97 | ##
|
---|
98 | {
|
---|
99 | "id": "You-Built-BaseTools",
|
---|
100 | "scope": "edk2-build",
|
---|
101 | "flags": ["set_shell_var", "set_path"],
|
---|
102 | "var_name": "EDK_TOOLS_BIN"
|
---|
103 | }
|
---|
104 | '''
|
---|
105 | with open(os.path.join(OutputDir, "basetoolsbin_path_env.yaml"), "w") as f:
|
---|
106 | f.write(content)
|
---|
107 |
|
---|
108 | def Go(self):
|
---|
109 | logging.info("Running Python version: " + str(sys.version_info))
|
---|
110 |
|
---|
111 | (build_env, shell_env) = self_describing_environment.BootstrapEnvironment(
|
---|
112 | self.GetWorkspaceRoot(), self.GetActiveScopes())
|
---|
113 |
|
---|
114 | # # Bind our current execution environment into the shell vars.
|
---|
115 | ph = os.path.dirname(sys.executable)
|
---|
116 | if " " in ph:
|
---|
117 | ph = '"' + ph + '"'
|
---|
118 | shell_env.set_shell_var("PYTHON_HOME", ph)
|
---|
119 | # PYTHON_COMMAND is required to be set for using edk2 python builds.
|
---|
120 | pc = sys.executable
|
---|
121 | if " " in pc:
|
---|
122 | pc = '"' + pc + '"'
|
---|
123 | shell_env.set_shell_var("PYTHON_COMMAND", pc)
|
---|
124 |
|
---|
125 | if self.tool_chain_tag.lower().startswith("vs"):
|
---|
126 | if self.target_arch is None:
|
---|
127 | # Put a default as IA32
|
---|
128 | self.target_arch = "IA32"
|
---|
129 |
|
---|
130 | if self.target_arch == "IA32":
|
---|
131 | VcToolChainArch = "x86"
|
---|
132 | TargetInfoArch = "x86"
|
---|
133 | OutputDir = "Win32"
|
---|
134 | elif self.target_arch == "ARM":
|
---|
135 | VcToolChainArch = "x86_arm"
|
---|
136 | TargetInfoArch = "ARM"
|
---|
137 | OutputDir = "Win32"
|
---|
138 | elif self.target_arch == "X64":
|
---|
139 | VcToolChainArch = "amd64"
|
---|
140 | TargetInfoArch = "x86"
|
---|
141 | OutputDir = "Win64"
|
---|
142 | elif self.target_arch == "AARCH64":
|
---|
143 | VcToolChainArch = "amd64_arm64"
|
---|
144 | TargetInfoArch = "ARM"
|
---|
145 | OutputDir = "Win64"
|
---|
146 | else:
|
---|
147 | raise NotImplementedError()
|
---|
148 |
|
---|
149 | self.OutputDir = os.path.join(
|
---|
150 | shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", OutputDir)
|
---|
151 |
|
---|
152 | # compiled tools need to be added to path because antlr is referenced
|
---|
153 | HostInfo = GetHostInfo()
|
---|
154 | if TargetInfoArch == HostInfo.arch:
|
---|
155 | # not cross compiling
|
---|
156 | shell_env.insert_path(self.OutputDir)
|
---|
157 | else:
|
---|
158 | # cross compiling:
|
---|
159 | # as the VfrCompile tool is needed in the build process, we need
|
---|
160 | # to build one for the host system, then add the path to the
|
---|
161 | # tools to the PATH environment variable
|
---|
162 | shell_environment.CheckpointBuildVars()
|
---|
163 | if HostInfo.arch == "x86" and HostInfo.bit == "64":
|
---|
164 | host_arch = "X64"
|
---|
165 | host_toolchain_arch = "amd64"
|
---|
166 | TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64")
|
---|
167 | elif HostInfo.arch == "x86" and HostInfo.bit == "32":
|
---|
168 | host_arch = "IA32"
|
---|
169 | host_toolchain_arch = "x86"
|
---|
170 | TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")
|
---|
171 | elif HostInfo.arch == "ARM" and HostInfo.bit == "64":
|
---|
172 | host_arch = "AARCH64"
|
---|
173 | host_toolchain_arch = "amd64_arm64"
|
---|
174 | TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win64")
|
---|
175 | elif HostInfo.arch == "ARM" and HostInfo.bit == "32":
|
---|
176 | host_arch = "ARM"
|
---|
177 | host_toolchain_arch = "x86_arm"
|
---|
178 | TempOutputDir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "Bin", "Win32")
|
---|
179 | else:
|
---|
180 | raise Exception("Unsupported host system. %s %s" % (HostInfo.arch, HostInfo.bit))
|
---|
181 |
|
---|
182 | interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]
|
---|
183 | interesting_keys.extend(
|
---|
184 | ["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"])
|
---|
185 | interesting_keys.extend(
|
---|
186 | ["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])
|
---|
187 | vc_vars = QueryVcVariables(
|
---|
188 | interesting_keys, host_toolchain_arch, vs_version=self.tool_chain_tag.lower())
|
---|
189 | for key in vc_vars.keys():
|
---|
190 | logging.debug(f"Var - {key} = {vc_vars[key]}")
|
---|
191 | if key.lower() == 'path':
|
---|
192 | shell_env.set_path(vc_vars[key])
|
---|
193 | else:
|
---|
194 | shell_env.set_shell_var(key, vc_vars[key])
|
---|
195 |
|
---|
196 | # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
|
---|
197 | # this script is BUILDING the base tools.
|
---|
198 | shell_env.set_shell_var('HOST_ARCH', host_arch)
|
---|
199 | shell_env.insert_path(TempOutputDir)
|
---|
200 |
|
---|
201 | # All set, build the tools for the host system.
|
---|
202 | ret = RunCmd('nmake.exe', None,
|
---|
203 | workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
|
---|
204 | if ret != 0:
|
---|
205 | raise Exception("Failed to build base tools for host system.")
|
---|
206 |
|
---|
207 | # Copy the output to a temp directory
|
---|
208 | TempFolder = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), "BaseToolsBuild", "Temp")
|
---|
209 | if not os.path.exists(TempFolder):
|
---|
210 | os.makedirs(TempFolder)
|
---|
211 | for file in os.listdir(TempOutputDir):
|
---|
212 | shutil.copy(os.path.join(TempOutputDir, file), TempFolder)
|
---|
213 |
|
---|
214 | # Clean up the build output
|
---|
215 | ret = RunCmd('nmake.exe', 'cleanall',
|
---|
216 | workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
|
---|
217 |
|
---|
218 | # Remove the entire TempOutputDir
|
---|
219 | shutil.rmtree(TempOutputDir)
|
---|
220 |
|
---|
221 | shell_environment.RevertBuildVars()
|
---|
222 | shell_env.insert_path(TempFolder)
|
---|
223 |
|
---|
224 | # # Update environment with required VC vars.
|
---|
225 | interesting_keys = ["ExtensionSdkDir", "INCLUDE", "LIB"]
|
---|
226 | interesting_keys.extend(
|
---|
227 | ["LIBPATH", "Path", "UniversalCRTSdkDir", "UCRTVersion", "WindowsLibPath", "WindowsSdkBinPath"])
|
---|
228 | interesting_keys.extend(
|
---|
229 | ["WindowsSdkDir", "WindowsSdkVerBinPath", "WindowsSDKVersion", "VCToolsInstallDir"])
|
---|
230 | vc_vars = QueryVcVariables(
|
---|
231 | interesting_keys, VcToolChainArch, vs_version=self.tool_chain_tag.lower())
|
---|
232 | for key in vc_vars.keys():
|
---|
233 | logging.debug(f"Var - {key} = {vc_vars[key]}")
|
---|
234 | if key.lower() == 'path':
|
---|
235 | shell_env.set_path(vc_vars[key])
|
---|
236 | else:
|
---|
237 | shell_env.set_shell_var(key, vc_vars[key])
|
---|
238 |
|
---|
239 | # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
|
---|
240 | # this script is BUILDING the base tools.
|
---|
241 | shell_env.set_shell_var('HOST_ARCH', self.target_arch)
|
---|
242 |
|
---|
243 | # Actually build the tools.
|
---|
244 | output_stream = edk2_logging.create_output_stream()
|
---|
245 | ret = RunCmd('nmake.exe', None,
|
---|
246 | workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
|
---|
247 | edk2_logging.remove_output_stream(output_stream)
|
---|
248 | problems = edk2_logging.scan_compiler_output(output_stream)
|
---|
249 | for level, problem in problems:
|
---|
250 | logging.log(level, problem)
|
---|
251 | if ret != 0:
|
---|
252 | raise Exception("Failed to build.")
|
---|
253 |
|
---|
254 | self.WritePathEnvFile(self.OutputDir)
|
---|
255 | return ret
|
---|
256 |
|
---|
257 | elif self.tool_chain_tag.lower().startswith("gcc"):
|
---|
258 | # Note: This HOST_ARCH is in respect to the BUILT base tools, not the host arch where
|
---|
259 | # this script is BUILDING the base tools.
|
---|
260 | HostInfo = GetHostInfo()
|
---|
261 | prefix = None
|
---|
262 | TargetInfoArch = None
|
---|
263 | if self.target_arch is not None:
|
---|
264 | shell_env.set_shell_var('HOST_ARCH', self.target_arch)
|
---|
265 |
|
---|
266 | if "AARCH64" in self.target_arch:
|
---|
267 | prefix = shell_env.get_shell_var("GCC5_AARCH64_PREFIX")
|
---|
268 | if prefix == None:
|
---|
269 | # now check for install dir. If set then set the Prefix
|
---|
270 | install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_AARCH64_INSTALL")
|
---|
271 |
|
---|
272 | # make GCC5_AARCH64_PREFIX to align with tools_def.txt
|
---|
273 | prefix = os.path.join(install_path, "bin", "aarch64-none-linux-gnu-")
|
---|
274 |
|
---|
275 | shell_environment.GetEnvironment().set_shell_var("GCC_PREFIX", prefix)
|
---|
276 | TargetInfoArch = "ARM"
|
---|
277 |
|
---|
278 | elif "ARM" in self.target_arch:
|
---|
279 | prefix = shell_env.get_shell_var("GCC5_ARM_PREFIX")
|
---|
280 | if prefix == None:
|
---|
281 | # now check for install dir. If set then set the Prefix
|
---|
282 | install_path = shell_environment.GetEnvironment().get_shell_var("GCC5_ARM_INSTALL")
|
---|
283 |
|
---|
284 | # make GCC5_ARM_PREFIX to align with tools_def.txt
|
---|
285 | prefix = os.path.join(install_path, "bin", "arm-none-linux-gnueabihf-")
|
---|
286 |
|
---|
287 | shell_environment.GetEnvironment().set_shell_var("GCC_PREFIX", prefix)
|
---|
288 | TargetInfoArch = "ARM"
|
---|
289 |
|
---|
290 | else:
|
---|
291 | TargetInfoArch = "x86"
|
---|
292 | else:
|
---|
293 | self.target_arch = HostInfo.arch
|
---|
294 | TargetInfoArch = HostInfo.arch
|
---|
295 | # Otherwise, the built binary arch will be consistent with the host system
|
---|
296 |
|
---|
297 | # Added logic to support cross compilation scenarios
|
---|
298 | if TargetInfoArch != HostInfo.arch:
|
---|
299 | # this is defaulting to the version that comes with Ubuntu 20.04
|
---|
300 | ver = shell_environment.GetBuildVars().GetValue("LIBUUID_VERSION", "2.34")
|
---|
301 | work_dir = os.path.join(shell_env.get_shell_var("EDK_TOOLS_PATH"), self.GetLoggingFolderRelativeToRoot())
|
---|
302 | pack_name = f"util-linux-{ver}"
|
---|
303 | unzip_dir = os.path.join(work_dir, pack_name)
|
---|
304 |
|
---|
305 | if os.path.isfile(os.path.join(work_dir, f"{pack_name}.tar.gz")):
|
---|
306 | os.remove(os.path.join(work_dir, f"{pack_name}.tar.gz"))
|
---|
307 | if os.path.isdir(unzip_dir):
|
---|
308 | shutil.rmtree(unzip_dir)
|
---|
309 |
|
---|
310 | # cross compiling, need to rebuild libuuid for the target
|
---|
311 | ret = RunCmd("wget", f"https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/v{ver}/{pack_name}.tar.gz", workingdir=work_dir)
|
---|
312 | if ret != 0:
|
---|
313 | raise Exception(f"Failed to download libuuid version {ver} - {ret}")
|
---|
314 |
|
---|
315 | ret = RunCmd("tar", f"xvzf {pack_name}.tar.gz", workingdir=work_dir)
|
---|
316 | if ret != 0:
|
---|
317 | raise Exception(f"Failed to untar the downloaded file {ret}")
|
---|
318 |
|
---|
319 | # configure the source to use the cross compiler
|
---|
320 | pack_name = f"util-linux-{ver}"
|
---|
321 | if "AARCH64" in self.target_arch:
|
---|
322 | ret = RunCmd("sh", f"./configure --host=aarch64-linux -disable-all-programs --enable-libuuid CC={prefix}gcc", workingdir=unzip_dir)
|
---|
323 | elif "ARM" in self.target_arch:
|
---|
324 | ret = RunCmd("sh", f"./configure --host=arm-linux -disable-all-programs --enable-libuuid CC={prefix}gcc", workingdir=unzip_dir)
|
---|
325 | if ret != 0:
|
---|
326 | raise Exception(f"Failed to configure the util-linux to build with our gcc {ret}")
|
---|
327 |
|
---|
328 | ret = RunCmd("make", "", workingdir=unzip_dir)
|
---|
329 | if ret != 0:
|
---|
330 | raise Exception(f"Failed to build the libuuid with our gcc {ret}")
|
---|
331 |
|
---|
332 | shell_environment.GetEnvironment().set_shell_var("CROSS_LIB_UUID", unzip_dir)
|
---|
333 | shell_environment.GetEnvironment().set_shell_var("CROSS_LIB_UUID_INC", os.path.join(unzip_dir, "libuuid", "src"))
|
---|
334 |
|
---|
335 | ret = RunCmd("make", "clean", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
|
---|
336 | if ret != 0:
|
---|
337 | raise Exception("Failed to build.")
|
---|
338 |
|
---|
339 | cpu_count = self.GetCpuThreads()
|
---|
340 |
|
---|
341 | output_stream = edk2_logging.create_output_stream()
|
---|
342 | ret = RunCmd("make", f"-C . -j {cpu_count}", workingdir=shell_env.get_shell_var("EDK_TOOLS_PATH"))
|
---|
343 | edk2_logging.remove_output_stream(output_stream)
|
---|
344 | problems = edk2_logging.scan_compiler_output(output_stream)
|
---|
345 | for level, problem in problems:
|
---|
346 | logging.log(level, problem)
|
---|
347 | if ret != 0:
|
---|
348 | raise Exception("Failed to build.")
|
---|
349 |
|
---|
350 | self.OutputDir = os.path.join(
|
---|
351 | shell_env.get_shell_var("EDK_TOOLS_PATH"), "Source", "C", "bin")
|
---|
352 |
|
---|
353 | self.WritePathEnvFile(self.OutputDir)
|
---|
354 | return ret
|
---|
355 |
|
---|
356 | logging.critical("Tool Chain not supported")
|
---|
357 | return -1
|
---|
358 |
|
---|
359 | def GetCpuThreads(self) -> int:
|
---|
360 | ''' Function to return number of cpus. If error return 1'''
|
---|
361 | cpus = 1
|
---|
362 | try:
|
---|
363 | cpus = multiprocessing.cpu_count()
|
---|
364 | except:
|
---|
365 | # from the internet there are cases where cpu_count is not implemented.
|
---|
366 | # will handle error by just doing single proc build
|
---|
367 | pass
|
---|
368 | return cpus
|
---|
369 |
|
---|
370 |
|
---|
371 |
|
---|
372 | def main():
|
---|
373 | Edk2ToolsBuild().Invoke()
|
---|
374 |
|
---|
375 |
|
---|
376 | if __name__ == "__main__":
|
---|
377 | main()
|
---|