diff options
| author | kx <kx@radix.pro> | 2024-06-04 06:43:08 +0300 |
|---|---|---|
| committer | kx <kx@radix.pro> | 2024-06-04 06:43:08 +0300 |
| commit | 7e2ccccace24636f29ddc210b94606abd4c7e42b (patch) | |
| tree | 545d43ea12671048956cafeb12303e84d601e571 /BaseTools/Source/Python/AutoGen/ModuleAutoGen.py | |
| parent | 27b044605cd5f6b33a3d231576003850b3fe305b (diff) | |
| download | edk2-trunk.tar.xz | |
Renormalized end-of-lines from master@27b044605cd5f6b33a3d231576003850b3fe305btrunk
Diffstat (limited to 'BaseTools/Source/Python/AutoGen/ModuleAutoGen.py')
| -rwxr-xr-x | BaseTools/Source/Python/AutoGen/ModuleAutoGen.py | 4892 |
1 files changed, 2446 insertions, 2446 deletions
diff --git a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py index 65a2176ca9..7e410164e0 100755 --- a/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py +++ b/BaseTools/Source/Python/AutoGen/ModuleAutoGen.py @@ -1,2446 +1,2446 @@ -## @file
-# Create makefile for MS nmake and GNU make
-#
-# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-#
-from __future__ import absolute_import
-from AutoGen.AutoGen import AutoGen
-from Common.LongFilePathSupport import LongFilePath, CopyLongFilePath
-from Common.BuildToolError import *
-from Common.DataType import *
-from Common.Misc import *
-from Common.StringUtils import NormPath,GetSplitList
-from collections import defaultdict
-from Workspace.WorkspaceCommon import OrderedListDict
-import os.path as path
-import copy
-import hashlib
-from . import InfSectionParser
-from . import GenC
-from . import GenMake
-from . import GenDepex
-from io import BytesIO
-from GenPatchPcdTable.GenPatchPcdTable import parsePcdInfoFromMapFile
-from Workspace.MetaFileCommentParser import UsageList
-from .GenPcdDb import CreatePcdDatabaseCode
-from Common.caching import cached_class_function
-from AutoGen.ModuleAutoGenHelper import PlatformInfo,WorkSpaceInfo
-import json
-import tempfile
-
-## Mapping Makefile type
-gMakeTypeMap = {TAB_COMPILER_MSFT:"nmake", "GCC":"gmake"}
-#
-# Regular expression for finding Include Directories, the difference between MSFT and INTEL/GCC
-# is the former use /I , the Latter used -I to specify include directories
-#
-gBuildOptIncludePatternMsft = re.compile(r"(?:.*?)/I[ \t]*([^ ]*)", re.MULTILINE | re.DOTALL)
-gBuildOptIncludePatternOther = re.compile(r"(?:.*?)-I[ \t]*([^ ]*)", re.MULTILINE | re.DOTALL)
-
-## default file name for AutoGen
-gAutoGenCodeFileName = "AutoGen.c"
-gAutoGenHeaderFileName = "AutoGen.h"
-gAutoGenStringFileName = "%(module_name)sStrDefs.h"
-gAutoGenStringFormFileName = "%(module_name)sStrDefs.hpk"
-gAutoGenDepexFileName = "%(module_name)s.depex"
-gAutoGenImageDefFileName = "%(module_name)sImgDefs.h"
-gAutoGenIdfFileName = "%(module_name)sIdf.hpk"
-gInfSpecVersion = "0x00010017"
-
-#
-# Match name = variable
-#
-gEfiVarStoreNamePattern = re.compile(r"\s*name\s*=\s*(\w+)")
-#
-# The format of guid in efivarstore statement likes following and must be correct:
-# guid = {0xA04A27f4, 0xDF00, 0x4D42, {0xB5, 0x52, 0x39, 0x51, 0x13, 0x02, 0x11, 0x3D}}
-#
-gEfiVarStoreGuidPattern = re.compile(r"\s*guid\s*=\s*({.*?{.*?}\s*})")
-
-#
-# Template string to generic AsBuilt INF
-#
-gAsBuiltInfHeaderString = TemplateString("""${header_comments}
-
-# DO NOT EDIT
-# FILE auto-generated
-
-[Defines]
- INF_VERSION = ${module_inf_version}
- BASE_NAME = ${module_name}
- FILE_GUID = ${module_guid}
- MODULE_TYPE = ${module_module_type}${BEGIN}
- VERSION_STRING = ${module_version_string}${END}${BEGIN}
- PCD_IS_DRIVER = ${pcd_is_driver_string}${END}${BEGIN}
- UEFI_SPECIFICATION_VERSION = ${module_uefi_specification_version}${END}${BEGIN}
- PI_SPECIFICATION_VERSION = ${module_pi_specification_version}${END}${BEGIN}
- ENTRY_POINT = ${module_entry_point}${END}${BEGIN}
- UNLOAD_IMAGE = ${module_unload_image}${END}${BEGIN}
- CONSTRUCTOR = ${module_constructor}${END}${BEGIN}
- DESTRUCTOR = ${module_destructor}${END}${BEGIN}
- SHADOW = ${module_shadow}${END}${BEGIN}
- PCI_VENDOR_ID = ${module_pci_vendor_id}${END}${BEGIN}
- PCI_DEVICE_ID = ${module_pci_device_id}${END}${BEGIN}
- PCI_CLASS_CODE = ${module_pci_class_code}${END}${BEGIN}
- PCI_REVISION = ${module_pci_revision}${END}${BEGIN}
- BUILD_NUMBER = ${module_build_number}${END}${BEGIN}
- SPEC = ${module_spec}${END}${BEGIN}
- UEFI_HII_RESOURCE_SECTION = ${module_uefi_hii_resource_section}${END}${BEGIN}
- MODULE_UNI_FILE = ${module_uni_file}${END}
-
-[Packages.${module_arch}]${BEGIN}
- ${package_item}${END}
-
-[Binaries.${module_arch}]${BEGIN}
- ${binary_item}${END}
-
-[PatchPcd.${module_arch}]${BEGIN}
- ${patchablepcd_item}
-${END}
-
-[Protocols.${module_arch}]${BEGIN}
- ${protocol_item}
-${END}
-
-[Ppis.${module_arch}]${BEGIN}
- ${ppi_item}
-${END}
-
-[Guids.${module_arch}]${BEGIN}
- ${guid_item}
-${END}
-
-[PcdEx.${module_arch}]${BEGIN}
- ${pcd_item}
-${END}
-
-[LibraryClasses.${module_arch}]
-## @LIB_INSTANCES${BEGIN}
-# ${libraryclasses_item}${END}
-
-${depexsection_item}
-
-${userextension_tianocore_item}
-
-${tail_comments}
-
-[BuildOptions.${module_arch}]
-## @AsBuilt${BEGIN}
-## ${flags_item}${END}
-""")
-#
-# extend lists contained in a dictionary with lists stored in another dictionary
-# if CopyToDict is not derived from DefaultDict(list) then this may raise exception
-#
-def ExtendCopyDictionaryLists(CopyToDict, CopyFromDict):
- for Key in CopyFromDict:
- CopyToDict[Key].extend(CopyFromDict[Key])
-
-# Create a directory specified by a set of path elements and return the full path
-def _MakeDir(PathList):
- RetVal = path.join(*PathList)
- CreateDirectory(RetVal)
- return RetVal
-
-#
-# Convert string to C format array
-#
-def _ConvertStringToByteArray(Value):
- Value = Value.strip()
- if not Value:
- return None
- if Value[0] == '{':
- if not Value.endswith('}'):
- return None
- Value = Value.replace(' ', '').replace('{', '').replace('}', '')
- ValFields = Value.split(',')
- try:
- for Index in range(len(ValFields)):
- ValFields[Index] = str(int(ValFields[Index], 0))
- except ValueError:
- return None
- Value = '{' + ','.join(ValFields) + '}'
- return Value
-
- Unicode = False
- if Value.startswith('L"'):
- if not Value.endswith('"'):
- return None
- Value = Value[1:]
- Unicode = True
- elif not Value.startswith('"') or not Value.endswith('"'):
- return None
-
- Value = eval(Value) # translate escape character
- NewValue = '{'
- for Index in range(0, len(Value)):
- if Unicode:
- NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ','
- else:
- NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ','
- Value = NewValue + '0}'
- return Value
-
-## ModuleAutoGen class
-#
-# This class encapsules the AutoGen behaviors for the build tools. In addition to
-# the generation of AutoGen.h and AutoGen.c, it will generate *.depex file according
-# to the [depex] section in module's inf file.
-#
-class ModuleAutoGen(AutoGen):
- # call super().__init__ then call the worker function with different parameter count
- def __init__(self, Workspace, MetaFile, Target, Toolchain, Arch, *args, **kwargs):
- if not hasattr(self, "_Init"):
- self._InitWorker(Workspace, MetaFile, Target, Toolchain, Arch, *args)
- self._Init = True
-
- ## Cache the timestamps of metafiles of every module in a class attribute
- #
- TimeDict = {}
-
- def __new__(cls, Workspace, MetaFile, Target, Toolchain, Arch, *args, **kwargs):
-# check if this module is employed by active platform
- if not PlatformInfo(Workspace, args[0], Target, Toolchain, Arch,args[-1]).ValidModule(MetaFile):
- EdkLogger.verbose("Module [%s] for [%s] is not employed by active platform\n" \
- % (MetaFile, Arch))
- return None
- return super(ModuleAutoGen, cls).__new__(cls, Workspace, MetaFile, Target, Toolchain, Arch, *args, **kwargs)
-
- ## Initialize ModuleAutoGen
- #
- # @param Workspace EdkIIWorkspaceBuild object
- # @param ModuleFile The path of module file
- # @param Target Build target (DEBUG, RELEASE)
- # @param Toolchain Name of tool chain
- # @param Arch The arch the module supports
- # @param PlatformFile Platform meta-file
- #
- def _InitWorker(self, Workspace, ModuleFile, Target, Toolchain, Arch, PlatformFile,DataPipe):
- EdkLogger.debug(EdkLogger.DEBUG_9, "AutoGen module [%s] [%s]" % (ModuleFile, Arch))
- GlobalData.gProcessingFile = "%s [%s, %s, %s]" % (ModuleFile, Arch, Toolchain, Target)
-
- self.Workspace = Workspace
- self.WorkspaceDir = ""
- self.PlatformInfo = None
- self.DataPipe = DataPipe
- self.__init_platform_info__()
- self.MetaFile = ModuleFile
- self.SourceDir = self.MetaFile.SubDir
- self.SourceDir = mws.relpath(self.SourceDir, self.WorkspaceDir)
-
- self.ToolChain = Toolchain
- self.BuildTarget = Target
- self.Arch = Arch
- self.ToolChainFamily = self.PlatformInfo.ToolChainFamily
- self.BuildRuleFamily = self.PlatformInfo.BuildRuleFamily
-
- self.IsCodeFileCreated = False
- self.IsAsBuiltInfCreated = False
- self.DepexGenerated = False
-
- self.BuildDatabase = self.Workspace.BuildDatabase
- self.BuildRuleOrder = None
- self.BuildTime = 0
-
- self._GuidComments = OrderedListDict()
- self._ProtocolComments = OrderedListDict()
- self._PpiComments = OrderedListDict()
- self._BuildTargets = None
- self._IntroBuildTargetList = None
- self._FinalBuildTargetList = None
- self._FileTypes = None
-
- self.AutoGenDepSet = set()
- self.ReferenceModules = []
- self.ConstPcd = {}
- self.FileDependCache = {}
-
- def __init_platform_info__(self):
- pinfo = self.DataPipe.Get("P_Info")
- self.WorkspaceDir = pinfo.get("WorkspaceDir")
- self.PlatformInfo = PlatformInfo(self.Workspace,pinfo.get("ActivePlatform"),pinfo.get("Target"),pinfo.get("ToolChain"),pinfo.get("Arch"),self.DataPipe)
- ## hash() operator of ModuleAutoGen
- #
- # The module file path and arch string will be used to represent
- # hash value of this object
- #
- # @retval int Hash value of the module file path and arch
- #
- @cached_class_function
- def __hash__(self):
- return hash((self.MetaFile, self.Arch, self.ToolChain,self.BuildTarget))
- def __repr__(self):
- return "%s [%s]" % (self.MetaFile, self.Arch)
-
- # Get FixedAtBuild Pcds of this Module
- @cached_property
- def FixedAtBuildPcds(self):
- RetVal = []
- for Pcd in self.ModulePcdList:
- if Pcd.Type != TAB_PCDS_FIXED_AT_BUILD:
- continue
- if Pcd not in RetVal:
- RetVal.append(Pcd)
- return RetVal
-
- @cached_property
- def FixedVoidTypePcds(self):
- RetVal = {}
- for Pcd in self.FixedAtBuildPcds:
- if Pcd.DatumType == TAB_VOID:
- if '.'.join((Pcd.TokenSpaceGuidCName, Pcd.TokenCName)) not in RetVal:
- RetVal['.'.join((Pcd.TokenSpaceGuidCName, Pcd.TokenCName))] = Pcd.DefaultValue
- return RetVal
-
- @property
- def UniqueBaseName(self):
- ModuleNames = self.DataPipe.Get("M_Name")
- if not ModuleNames:
- return self.Name
- return ModuleNames.get((self.Name,self.MetaFile),self.Name)
-
- # Macros could be used in build_rule.txt (also Makefile)
- @cached_property
- def Macros(self):
- return OrderedDict((
- ("WORKSPACE" ,self.WorkspaceDir),
- ("MODULE_NAME" ,self.Name),
- ("MODULE_NAME_GUID" ,self.UniqueBaseName),
- ("MODULE_GUID" ,self.Guid),
- ("MODULE_VERSION" ,self.Version),
- ("MODULE_TYPE" ,self.ModuleType),
- ("MODULE_FILE" ,str(self.MetaFile)),
- ("MODULE_FILE_BASE_NAME" ,self.MetaFile.BaseName),
- ("MODULE_RELATIVE_DIR" ,self.SourceDir),
- ("MODULE_DIR" ,self.SourceDir),
- ("BASE_NAME" ,self.Name),
- ("ARCH" ,self.Arch),
- ("TOOLCHAIN" ,self.ToolChain),
- ("TOOLCHAIN_TAG" ,self.ToolChain),
- ("TOOL_CHAIN_TAG" ,self.ToolChain),
- ("TARGET" ,self.BuildTarget),
- ("BUILD_DIR" ,self.PlatformInfo.BuildDir),
- ("BIN_DIR" ,os.path.join(self.PlatformInfo.BuildDir, self.Arch)),
- ("LIB_DIR" ,os.path.join(self.PlatformInfo.BuildDir, self.Arch)),
- ("MODULE_BUILD_DIR" ,self.BuildDir),
- ("OUTPUT_DIR" ,self.OutputDir),
- ("DEBUG_DIR" ,self.DebugDir),
- ("DEST_DIR_OUTPUT" ,self.OutputDir),
- ("DEST_DIR_DEBUG" ,self.DebugDir),
- ("PLATFORM_NAME" ,self.PlatformInfo.Name),
- ("PLATFORM_GUID" ,self.PlatformInfo.Guid),
- ("PLATFORM_VERSION" ,self.PlatformInfo.Version),
- ("PLATFORM_RELATIVE_DIR" ,self.PlatformInfo.SourceDir),
- ("PLATFORM_DIR" ,mws.join(self.WorkspaceDir, self.PlatformInfo.SourceDir)),
- ("PLATFORM_OUTPUT_DIR" ,self.PlatformInfo.OutputDir),
- ("FFS_OUTPUT_DIR" ,self.FfsOutputDir)
- ))
-
- ## Return the module build data object
- @cached_property
- def Module(self):
- return self.BuildDatabase[self.MetaFile, self.Arch, self.BuildTarget, self.ToolChain]
-
- ## Return the module name
- @cached_property
- def Name(self):
- return self.Module.BaseName
-
- ## Return the module DxsFile if exist
- @cached_property
- def DxsFile(self):
- return self.Module.DxsFile
-
- ## Return the module meta-file GUID
- @cached_property
- def Guid(self):
- #
- # To build same module more than once, the module path with FILE_GUID overridden has
- # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path
- # in DSC. The overridden GUID can be retrieved from file name
- #
- if os.path.basename(self.MetaFile.File) != os.path.basename(self.MetaFile.Path):
- #
- # Length of GUID is 36
- #
- return os.path.basename(self.MetaFile.Path)[:36]
- return self.Module.Guid
-
- ## Return the module version
- @cached_property
- def Version(self):
- return self.Module.Version
-
- ## Return the module type
- @cached_property
- def ModuleType(self):
- return self.Module.ModuleType
-
- ## Return the component type (for Edk.x style of module)
- @cached_property
- def ComponentType(self):
- return self.Module.ComponentType
-
- ## Return the build type
- @cached_property
- def BuildType(self):
- return self.Module.BuildType
-
- ## Return the PCD_IS_DRIVER setting
- @cached_property
- def PcdIsDriver(self):
- return self.Module.PcdIsDriver
-
- ## Return the autogen version, i.e. module meta-file version
- @cached_property
- def AutoGenVersion(self):
- return self.Module.AutoGenVersion
-
- ## Check if the module is library or not
- @cached_property
- def IsLibrary(self):
- return bool(self.Module.LibraryClass)
-
- ## Check if the module is binary module or not
- @cached_property
- def IsBinaryModule(self):
- return self.Module.IsBinaryModule
-
- ## Return the directory to store intermediate files of the module
- @cached_property
- def BuildDir(self):
- return _MakeDir((
- self.PlatformInfo.BuildDir,
- self.Arch,
- self.SourceDir,
- self.MetaFile.BaseName
- ))
-
- ## Return the directory to store the intermediate object files of the module
- @cached_property
- def OutputDir(self):
- return _MakeDir((self.BuildDir, "OUTPUT"))
-
- ## Return the directory path to store ffs file
- @cached_property
- def FfsOutputDir(self):
- if GlobalData.gFdfParser:
- return path.join(self.PlatformInfo.BuildDir, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name)
- return ''
-
- ## Return the directory to store auto-gened source files of the module
- @cached_property
- def DebugDir(self):
- return _MakeDir((self.BuildDir, "DEBUG"))
-
- ## Return the path of custom file
- @cached_property
- def CustomMakefile(self):
- RetVal = {}
- for Type in self.Module.CustomMakefile:
- MakeType = gMakeTypeMap[Type] if Type in gMakeTypeMap else 'nmake'
- File = os.path.join(self.SourceDir, self.Module.CustomMakefile[Type])
- RetVal[MakeType] = File
- return RetVal
-
- ## Return the directory of the makefile
- #
- # @retval string The directory string of module's makefile
- #
- @cached_property
- def MakeFileDir(self):
- return self.BuildDir
-
- ## Return build command string
- #
- # @retval string Build command string
- #
- @cached_property
- def BuildCommand(self):
- return self.PlatformInfo.BuildCommand
-
- ## Get Module package and Platform package
- #
- # @retval list The list of package object
- #
- @cached_property
- def PackageList(self):
- PkagList = []
- if self.Module.Packages:
- PkagList.extend(self.Module.Packages)
- Platform = self.BuildDatabase[self.PlatformInfo.MetaFile, self.Arch, self.BuildTarget, self.ToolChain]
- for Package in Platform.Packages:
- if Package in PkagList:
- continue
- PkagList.append(Package)
- return PkagList
-
- ## Get object list of all packages the module and its dependent libraries belong to and the Platform depends on
- #
- # @retval list The list of package object
- #
- @cached_property
- def DerivedPackageList(self):
- PackageList = []
- PackageList.extend(self.PackageList)
- for M in self.DependentLibraryList:
- for Package in M.Packages:
- if Package in PackageList:
- continue
- PackageList.append(Package)
- return PackageList
-
- ## Get the depex string
- #
- # @return : a string contain all depex expression.
- def _GetDepexExpresionString(self):
- DepexStr = ''
- DepexList = []
- ## DPX_SOURCE IN Define section.
- if self.Module.DxsFile:
- return DepexStr
- for M in [self.Module] + self.DependentLibraryList:
- Filename = M.MetaFile.Path
- InfObj = InfSectionParser.InfSectionParser(Filename)
- DepexExpressionList = InfObj.GetDepexExpresionList()
- for DepexExpression in DepexExpressionList:
- for key in DepexExpression:
- Arch, ModuleType = key
- DepexExpr = [x for x in DepexExpression[key] if not str(x).startswith('#')]
- # the type of build module is USER_DEFINED.
- # All different DEPEX section tags would be copied into the As Built INF file
- # and there would be separate DEPEX section tags
- if self.ModuleType.upper() == SUP_MODULE_USER_DEFINED or self.ModuleType.upper() == SUP_MODULE_HOST_APPLICATION:
- if (Arch.upper() == self.Arch.upper()) and (ModuleType.upper() != TAB_ARCH_COMMON):
- DepexList.append({(Arch, ModuleType): DepexExpr})
- else:
- if Arch.upper() == TAB_ARCH_COMMON or \
- (Arch.upper() == self.Arch.upper() and \
- ModuleType.upper() in [TAB_ARCH_COMMON, self.ModuleType.upper()]):
- DepexList.append({(Arch, ModuleType): DepexExpr})
-
- #the type of build module is USER_DEFINED.
- if self.ModuleType.upper() == SUP_MODULE_USER_DEFINED or self.ModuleType.upper() == SUP_MODULE_HOST_APPLICATION:
- for Depex in DepexList:
- for key in Depex:
- DepexStr += '[Depex.%s.%s]\n' % key
- DepexStr += '\n'.join('# '+ val for val in Depex[key])
- DepexStr += '\n\n'
- if not DepexStr:
- return '[Depex.%s]\n' % self.Arch
- return DepexStr
-
- #the type of build module not is USER_DEFINED.
- Count = 0
- for Depex in DepexList:
- Count += 1
- if DepexStr != '':
- DepexStr += ' AND '
- DepexStr += '('
- for D in Depex.values():
- DepexStr += ' '.join(val for val in D)
- Index = DepexStr.find('END')
- if Index > -1 and Index == len(DepexStr) - 3:
- DepexStr = DepexStr[:-3]
- DepexStr = DepexStr.strip()
- DepexStr += ')'
- if Count == 1:
- DepexStr = DepexStr.lstrip('(').rstrip(')').strip()
- if not DepexStr:
- return '[Depex.%s]\n' % self.Arch
- return '[Depex.%s]\n# ' % self.Arch + DepexStr
-
- ## Merge dependency expression
- #
- # @retval list The token list of the dependency expression after parsed
- #
- @cached_property
- def DepexList(self):
- if self.DxsFile or self.IsLibrary or TAB_DEPENDENCY_EXPRESSION_FILE in self.FileTypes:
- return {}
-
- DepexList = []
- #
- # Append depex from dependent libraries, if not "BEFORE", "AFTER" expression
- #
- FixedVoidTypePcds = {}
- for M in [self] + self.LibraryAutoGenList:
- FixedVoidTypePcds.update(M.FixedVoidTypePcds)
- for M in [self] + self.LibraryAutoGenList:
- Inherited = False
- for D in M.Module.Depex[self.Arch, self.ModuleType]:
- if DepexList != []:
- DepexList.append('AND')
- DepexList.append('(')
- #replace D with value if D is FixedAtBuild PCD
- NewList = []
- for item in D:
- if '.' not in item:
- NewList.append(item)
- else:
- try:
- Value = FixedVoidTypePcds[item]
- if len(Value.split(',')) != 16:
- EdkLogger.error("build", FORMAT_INVALID,
- "{} used in [Depex] section should be used as FixedAtBuild type and VOID* datum type and 16 bytes in the module.".format(item))
- NewList.append(Value)
- except:
- EdkLogger.error("build", FORMAT_INVALID, "{} used in [Depex] section should be used as FixedAtBuild type and VOID* datum type in the module.".format(item))
-
- DepexList.extend(NewList)
- if DepexList[-1] == 'END': # no need of a END at this time
- DepexList.pop()
- DepexList.append(')')
- Inherited = True
- if Inherited:
- EdkLogger.verbose("DEPEX[%s] (+%s) = %s" % (self.Name, M.Module.BaseName, DepexList))
- if 'BEFORE' in DepexList or 'AFTER' in DepexList:
- break
- if len(DepexList) > 0:
- EdkLogger.verbose('')
- return {self.ModuleType:DepexList}
-
- ## Merge dependency expression
- #
- # @retval list The token list of the dependency expression after parsed
- #
- @cached_property
- def DepexExpressionDict(self):
- if self.DxsFile or self.IsLibrary or TAB_DEPENDENCY_EXPRESSION_FILE in self.FileTypes:
- return {}
-
- DepexExpressionString = ''
- #
- # Append depex from dependent libraries, if not "BEFORE", "AFTER" expresion
- #
- for M in [self.Module] + self.DependentLibraryList:
- Inherited = False
- for D in M.DepexExpression[self.Arch, self.ModuleType]:
- if DepexExpressionString != '':
- DepexExpressionString += ' AND '
- DepexExpressionString += '('
- DepexExpressionString += D
- DepexExpressionString = DepexExpressionString.rstrip('END').strip()
- DepexExpressionString += ')'
- Inherited = True
- if Inherited:
- EdkLogger.verbose("DEPEX[%s] (+%s) = %s" % (self.Name, M.BaseName, DepexExpressionString))
- if 'BEFORE' in DepexExpressionString or 'AFTER' in DepexExpressionString:
- break
- if len(DepexExpressionString) > 0:
- EdkLogger.verbose('')
-
- return {self.ModuleType:DepexExpressionString}
-
- # Get the tiano core user extension, it is contain dependent library.
- # @retval: a list contain tiano core userextension.
- #
- def _GetTianoCoreUserExtensionList(self):
- TianoCoreUserExtentionList = []
- for M in [self.Module] + self.DependentLibraryList:
- Filename = M.MetaFile.Path
- InfObj = InfSectionParser.InfSectionParser(Filename)
- TianoCoreUserExtenList = InfObj.GetUserExtensionTianoCore()
- for TianoCoreUserExtent in TianoCoreUserExtenList:
- for Section in TianoCoreUserExtent:
- ItemList = Section.split(TAB_SPLIT)
- Arch = self.Arch
- if len(ItemList) == 4:
- Arch = ItemList[3]
- if Arch.upper() == TAB_ARCH_COMMON or Arch.upper() == self.Arch.upper():
- TianoCoreList = []
- TianoCoreList.extend([TAB_SECTION_START + Section + TAB_SECTION_END])
- TianoCoreList.extend(TianoCoreUserExtent[Section][:])
- TianoCoreList.append('\n')
- TianoCoreUserExtentionList.append(TianoCoreList)
-
- return TianoCoreUserExtentionList
-
- ## Return the list of specification version required for the module
- #
- # @retval list The list of specification defined in module file
- #
- @cached_property
- def Specification(self):
- return self.Module.Specification
-
- ## Tool option for the module build
- #
- # @param PlatformInfo The object of PlatformBuildInfo
- # @retval dict The dict containing valid options
- #
- @cached_property
- def BuildOption(self):
- RetVal, self.BuildRuleOrder = self.PlatformInfo.ApplyBuildOption(self.Module)
- if self.BuildRuleOrder:
- self.BuildRuleOrder = ['.%s' % Ext for Ext in self.BuildRuleOrder.split()]
- return RetVal
-
- ## Get include path list from tool option for the module build
- #
- # @retval list The include path list
- #
- @cached_property
- def BuildOptionIncPathList(self):
- #
- # Regular expression for finding Include Directories, the difference between MSFT and INTEL/GCC
- # is the former use /I , the Latter used -I to specify include directories
- #
- if self.PlatformInfo.ToolChainFamily in (TAB_COMPILER_MSFT):
- BuildOptIncludeRegEx = gBuildOptIncludePatternMsft
- elif self.PlatformInfo.ToolChainFamily in ('INTEL', 'GCC'):
- BuildOptIncludeRegEx = gBuildOptIncludePatternOther
- else:
- #
- # New ToolChainFamily, don't known whether there is option to specify include directories
- #
- return []
-
- RetVal = []
- for Tool in ('CC', 'PP', 'VFRPP', 'ASLPP', 'ASLCC', 'APP', 'ASM'):
- try:
- FlagOption = self.BuildOption[Tool]['FLAGS']
- except KeyError:
- FlagOption = ''
-
- IncPathList = [NormPath(Path, self.Macros) for Path in BuildOptIncludeRegEx.findall(FlagOption)]
-
- #
- # EDK II modules must not reference header files outside of the packages they depend on or
- # within the module's directory tree. Report error if violation.
- #
- if GlobalData.gDisableIncludePathCheck == False:
- for Path in IncPathList:
- if (Path not in self.IncludePathList) and (CommonPath([Path, self.MetaFile.Dir]) != self.MetaFile.Dir):
- ErrMsg = "The include directory for the EDK II module in this line is invalid %s specified in %s FLAGS '%s'" % (Path, Tool, FlagOption)
- EdkLogger.error("build",
- PARAMETER_INVALID,
- ExtraData=ErrMsg,
- File=str(self.MetaFile))
- RetVal += IncPathList
- return RetVal
-
- ## Return a list of files which can be built from source
- #
- # What kind of files can be built is determined by build rules in
- # $(CONF_DIRECTORY)/build_rule.txt and toolchain family.
- #
- @cached_property
- def SourceFileList(self):
- RetVal = []
- ToolChainTagSet = {"", TAB_STAR, self.ToolChain}
- ToolChainFamilySet = {"", TAB_STAR, self.ToolChainFamily, self.BuildRuleFamily}
- for F in self.Module.Sources:
- # match tool chain
- if F.TagName not in ToolChainTagSet:
- EdkLogger.debug(EdkLogger.DEBUG_9, "The toolchain [%s] for processing file [%s] is found, "
- "but [%s] is currently used" % (F.TagName, str(F), self.ToolChain))
- continue
- # match tool chain family or build rule family
- if F.ToolChainFamily not in ToolChainFamilySet:
- EdkLogger.debug(
- EdkLogger.DEBUG_0,
- "The file [%s] must be built by tools of [%s], " \
- "but current toolchain family is [%s], buildrule family is [%s]" \
- % (str(F), F.ToolChainFamily, self.ToolChainFamily, self.BuildRuleFamily))
- continue
-
- # add the file path into search path list for file including
- if F.Dir not in self.IncludePathList:
- self.IncludePathList.insert(0, F.Dir)
- RetVal.append(F)
-
- self._MatchBuildRuleOrder(RetVal)
-
- for F in RetVal:
- self._ApplyBuildRule(F, TAB_UNKNOWN_FILE)
- return RetVal
-
- def _MatchBuildRuleOrder(self, FileList):
- Order_Dict = {}
- self.BuildOption
- for SingleFile in FileList:
- if self.BuildRuleOrder and SingleFile.Ext in self.BuildRuleOrder and SingleFile.Ext in self.BuildRules:
- key = SingleFile.Path.rsplit(SingleFile.Ext,1)[0]
- if key in Order_Dict:
- Order_Dict[key].append(SingleFile.Ext)
- else:
- Order_Dict[key] = [SingleFile.Ext]
-
- RemoveList = []
- for F in Order_Dict:
- if len(Order_Dict[F]) > 1:
- Order_Dict[F].sort(key=lambda i: self.BuildRuleOrder.index(i))
- for Ext in Order_Dict[F][1:]:
- RemoveList.append(F + Ext)
-
- for item in RemoveList:
- FileList.remove(item)
-
- return FileList
-
- ## Return the list of unicode files
- @cached_property
- def UnicodeFileList(self):
- return self.FileTypes.get(TAB_UNICODE_FILE,[])
-
- ## Return the list of vfr files
- @cached_property
- def VfrFileList(self):
- return self.FileTypes.get(TAB_VFR_FILE, [])
-
- ## Return the list of Image Definition files
- @cached_property
- def IdfFileList(self):
- return self.FileTypes.get(TAB_IMAGE_FILE,[])
-
- ## Return a list of files which can be built from binary
- #
- # "Build" binary files are just to copy them to build directory.
- #
- # @retval list The list of files which can be built later
- #
- @cached_property
- def BinaryFileList(self):
- RetVal = []
- for F in self.Module.Binaries:
- if F.Target not in [TAB_ARCH_COMMON, TAB_STAR] and F.Target != self.BuildTarget:
- continue
- RetVal.append(F)
- self._ApplyBuildRule(F, F.Type, BinaryFileList=RetVal)
- return RetVal
-
- @cached_property
- def BuildRules(self):
- RetVal = {}
- BuildRuleDatabase = self.PlatformInfo.BuildRule
- for Type in BuildRuleDatabase.FileTypeList:
- #first try getting build rule by BuildRuleFamily
- RuleObject = BuildRuleDatabase[Type, self.BuildType, self.Arch, self.BuildRuleFamily]
- if not RuleObject:
- # build type is always module type, but ...
- if self.ModuleType != self.BuildType:
- RuleObject = BuildRuleDatabase[Type, self.ModuleType, self.Arch, self.BuildRuleFamily]
- #second try getting build rule by ToolChainFamily
- if not RuleObject:
- RuleObject = BuildRuleDatabase[Type, self.BuildType, self.Arch, self.ToolChainFamily]
- if not RuleObject:
- # build type is always module type, but ...
- if self.ModuleType != self.BuildType:
- RuleObject = BuildRuleDatabase[Type, self.ModuleType, self.Arch, self.ToolChainFamily]
- if not RuleObject:
- continue
- RuleObject = RuleObject.Instantiate(self.Macros)
- RetVal[Type] = RuleObject
- for Ext in RuleObject.SourceFileExtList:
- RetVal[Ext] = RuleObject
- return RetVal
-
- def _ApplyBuildRule(self, File, FileType, BinaryFileList=None):
- if self._BuildTargets is None:
- self._IntroBuildTargetList = set()
- self._FinalBuildTargetList = set()
- self._BuildTargets = defaultdict(set)
- self._FileTypes = defaultdict(set)
-
- if not BinaryFileList:
- BinaryFileList = self.BinaryFileList
-
- SubDirectory = os.path.join(self.OutputDir, File.SubDir)
- if not os.path.exists(SubDirectory):
- CreateDirectory(SubDirectory)
- TargetList = set()
- FinalTargetName = set()
- RuleChain = set()
- SourceList = [File]
- Index = 0
- #
- # Make sure to get build rule order value
- #
- self.BuildOption
-
- while Index < len(SourceList):
- # Reset the FileType if not the first iteration.
- if Index > 0:
- FileType = TAB_UNKNOWN_FILE
- Source = SourceList[Index]
- Index = Index + 1
-
- if Source != File:
- CreateDirectory(Source.Dir)
-
- if File.IsBinary and File == Source and File in BinaryFileList:
- # Skip all files that are not binary libraries
- if not self.IsLibrary:
- continue
- RuleObject = self.BuildRules[TAB_DEFAULT_BINARY_FILE]
- elif FileType in self.BuildRules:
- RuleObject = self.BuildRules[FileType]
- elif Source.Ext in self.BuildRules:
- RuleObject = self.BuildRules[Source.Ext]
- else:
- # No more rule to apply: Source is a final target.
- FinalTargetName.add(Source)
- continue
-
- FileType = RuleObject.SourceFileType
- self._FileTypes[FileType].add(Source)
-
- # stop at STATIC_LIBRARY for library
- if self.IsLibrary and FileType == TAB_STATIC_LIBRARY:
- FinalTargetName.add(Source)
- continue
-
- Target = RuleObject.Apply(Source, self.BuildRuleOrder)
- if not Target:
- # No Target: Source is a final target.
- FinalTargetName.add(Source)
- continue
-
- TargetList.add(Target)
- self._BuildTargets[FileType].add(Target)
-
- if not Source.IsBinary and Source == File:
- self._IntroBuildTargetList.add(Target)
-
- # to avoid cyclic rule
- if FileType in RuleChain:
- EdkLogger.error("build", ERROR_STATEMENT, "Cyclic dependency detected while generating rule for %s" % str(Source))
-
- RuleChain.add(FileType)
- SourceList.extend(Target.Outputs)
-
- # For each final target name, retrieve the corresponding TargetDescBlock instance.
- for FTargetName in FinalTargetName:
- for Target in TargetList:
- if FTargetName == Target.Target:
- self._FinalBuildTargetList.add(Target)
-
- @cached_property
- def Targets(self):
- if self._BuildTargets is None:
- self._IntroBuildTargetList = set()
- self._FinalBuildTargetList = set()
- self._BuildTargets = defaultdict(set)
- self._FileTypes = defaultdict(set)
-
- #TRICK: call SourceFileList property to apply build rule for source files
- self.SourceFileList
-
- #TRICK: call _GetBinaryFileList to apply build rule for binary files
- self.BinaryFileList
-
- return self._BuildTargets
-
- @cached_property
- def IntroTargetList(self):
- self.Targets
- return self._IntroBuildTargetList
-
- @cached_property
- def CodaTargetList(self):
- self.Targets
- return self._FinalBuildTargetList
-
- @cached_property
- def FileTypes(self):
- self.Targets
- return self._FileTypes
-
- ## Get the list of package object the module depends on and the Platform depends on
- #
- # @retval list The package object list
- #
- @cached_property
- def DependentPackageList(self):
- return self.PackageList
-
- ## Return the list of auto-generated code file
- #
- # @retval list The list of auto-generated file
- #
- @cached_property
- def AutoGenFileList(self):
- AutoGenUniIdf = self.BuildType != 'UEFI_HII'
- UniStringBinBuffer = BytesIO()
- IdfGenBinBuffer = BytesIO()
- RetVal = {}
- AutoGenC = TemplateString()
- AutoGenH = TemplateString()
- StringH = TemplateString()
- StringIdf = TemplateString()
- GenC.CreateCode(self, AutoGenC, AutoGenH, StringH, AutoGenUniIdf, UniStringBinBuffer, StringIdf, AutoGenUniIdf, IdfGenBinBuffer)
- #
- # AutoGen.c is generated if there are library classes in inf, or there are object files
- #
- if str(AutoGenC) != "" and (len(self.Module.LibraryClasses) > 0
- or TAB_OBJECT_FILE in self.FileTypes):
- AutoFile = PathClass(gAutoGenCodeFileName, self.DebugDir)
- RetVal[AutoFile] = str(AutoGenC)
- self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE)
- if str(AutoGenH) != "":
- AutoFile = PathClass(gAutoGenHeaderFileName, self.DebugDir)
- RetVal[AutoFile] = str(AutoGenH)
- self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE)
- if str(StringH) != "":
- AutoFile = PathClass(gAutoGenStringFileName % {"module_name":self.Name}, self.DebugDir)
- RetVal[AutoFile] = str(StringH)
- self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE)
- if UniStringBinBuffer is not None and UniStringBinBuffer.getvalue() != b"":
- AutoFile = PathClass(gAutoGenStringFormFileName % {"module_name":self.Name}, self.OutputDir)
- RetVal[AutoFile] = UniStringBinBuffer.getvalue()
- AutoFile.IsBinary = True
- self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE)
- if UniStringBinBuffer is not None:
- UniStringBinBuffer.close()
- if str(StringIdf) != "":
- AutoFile = PathClass(gAutoGenImageDefFileName % {"module_name":self.Name}, self.DebugDir)
- RetVal[AutoFile] = str(StringIdf)
- self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE)
- if IdfGenBinBuffer is not None and IdfGenBinBuffer.getvalue() != b"":
- AutoFile = PathClass(gAutoGenIdfFileName % {"module_name":self.Name}, self.OutputDir)
- RetVal[AutoFile] = IdfGenBinBuffer.getvalue()
- AutoFile.IsBinary = True
- self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE)
- if IdfGenBinBuffer is not None:
- IdfGenBinBuffer.close()
- return RetVal
-
- ## Return the list of library modules explicitly or implicitly used by this module
- @cached_property
- def DependentLibraryList(self):
- # only merge library classes and PCD for non-library module
- if self.IsLibrary:
- return []
- return self.PlatformInfo.ApplyLibraryInstance(self.Module)
-
- ## Get the list of PCDs from current module
- #
- # @retval list The list of PCD
- #
- @cached_property
- def ModulePcdList(self):
- # apply PCD settings from platform
- RetVal = self.PlatformInfo.ApplyPcdSetting(self, self.Module.Pcds)
-
- return RetVal
- @cached_property
- def _PcdComments(self):
- ReVal = OrderedListDict()
- ExtendCopyDictionaryLists(ReVal, self.Module.PcdComments)
- if not self.IsLibrary:
- for Library in self.DependentLibraryList:
- ExtendCopyDictionaryLists(ReVal, Library.PcdComments)
- return ReVal
-
- ## Get the list of PCDs from dependent libraries
- #
- # @retval list The list of PCD
- #
- @cached_property
- def LibraryPcdList(self):
- if self.IsLibrary:
- return []
- RetVal = []
- Pcds = set()
- # get PCDs from dependent libraries
- for Library in self.DependentLibraryList:
- PcdsInLibrary = OrderedDict()
- for Key in Library.Pcds:
- # skip duplicated PCDs
- if Key in self.Module.Pcds or Key in Pcds:
- continue
- Pcds.add(Key)
- PcdsInLibrary[Key] = copy.copy(Library.Pcds[Key])
- RetVal.extend(self.PlatformInfo.ApplyPcdSetting(self, PcdsInLibrary, Library=Library))
- return RetVal
-
- ## Get the GUID value mapping
- #
- # @retval dict The mapping between GUID cname and its value
- #
- @cached_property
- def GuidList(self):
- RetVal = self.Module.Guids
- for Library in self.DependentLibraryList:
- RetVal.update(Library.Guids)
- ExtendCopyDictionaryLists(self._GuidComments, Library.GuidComments)
- ExtendCopyDictionaryLists(self._GuidComments, self.Module.GuidComments)
- return RetVal
-
- @cached_property
- def GetGuidsUsedByPcd(self):
- RetVal = OrderedDict(self.Module.GetGuidsUsedByPcd())
- for Library in self.DependentLibraryList:
- RetVal.update(Library.GetGuidsUsedByPcd())
- return RetVal
- ## Get the protocol value mapping
- #
- # @retval dict The mapping between protocol cname and its value
- #
- @cached_property
- def ProtocolList(self):
- RetVal = OrderedDict(self.Module.Protocols)
- for Library in self.DependentLibraryList:
- RetVal.update(Library.Protocols)
- ExtendCopyDictionaryLists(self._ProtocolComments, Library.ProtocolComments)
- ExtendCopyDictionaryLists(self._ProtocolComments, self.Module.ProtocolComments)
- return RetVal
-
- ## Get the PPI value mapping
- #
- # @retval dict The mapping between PPI cname and its value
- #
- @cached_property
- def PpiList(self):
- RetVal = OrderedDict(self.Module.Ppis)
- for Library in self.DependentLibraryList:
- RetVal.update(Library.Ppis)
- ExtendCopyDictionaryLists(self._PpiComments, Library.PpiComments)
- ExtendCopyDictionaryLists(self._PpiComments, self.Module.PpiComments)
- return RetVal
-
- ## Get the list of include search path
- #
- # @retval list The list path
- #
- @cached_property
- def IncludePathList(self):
- RetVal = []
- RetVal.append(self.MetaFile.Dir)
- RetVal.append(self.DebugDir)
-
- for Package in self.PackageList:
- PackageDir = mws.join(self.WorkspaceDir, Package.MetaFile.Dir)
- if PackageDir not in RetVal:
- RetVal.append(PackageDir)
- IncludesList = Package.Includes
- if Package._PrivateIncludes:
- if not self.MetaFile.OriginalPath.Path.startswith(PackageDir):
- IncludesList = list(set(Package.Includes).difference(set(Package._PrivateIncludes)))
- for Inc in IncludesList:
- if Inc not in RetVal:
- RetVal.append(str(Inc))
- RetVal.extend(self.IncPathFromBuildOptions)
- return RetVal
-
- @cached_property
- def IncPathFromBuildOptions(self):
- IncPathList = []
- for tool in self.BuildOption:
- if 'FLAGS' in self.BuildOption[tool]:
- flags = self.BuildOption[tool]['FLAGS']
- whitespace = False
- for flag in flags.split(" "):
- flag = flag.strip()
- if flag.startswith(("/I","-I")):
- if len(flag)>2:
- if os.path.exists(flag[2:]):
- IncPathList.append(flag[2:])
- else:
- whitespace = True
- continue
- if whitespace and flag:
- if os.path.exists(flag):
- IncPathList.append(flag)
- whitespace = False
- return IncPathList
-
- @cached_property
- def IncludePathLength(self):
- return sum(len(inc)+1 for inc in self.IncludePathList)
-
- ## Get the list of include paths from the packages
- #
- # @IncludesList list The list path
- #
- @cached_property
- def PackageIncludePathList(self):
- IncludesList = []
- for Package in self.PackageList:
- PackageDir = mws.join(self.WorkspaceDir, Package.MetaFile.Dir)
- IncludesList = Package.Includes
- if Package._PrivateIncludes:
- if not self.MetaFile.Path.startswith(PackageDir):
- IncludesList = list(set(Package.Includes).difference(set(Package._PrivateIncludes)))
- return IncludesList
-
- ## Get HII EX PCDs which maybe used by VFR
- #
- # efivarstore used by VFR may relate with HII EX PCDs
- # Get the variable name and GUID from efivarstore and HII EX PCD
- # List the HII EX PCDs in As Built INF if both name and GUID match.
- #
- # @retval list HII EX PCDs
- #
- def _GetPcdsMaybeUsedByVfr(self):
- if not self.SourceFileList:
- return []
-
- NameGuids = set()
- for SrcFile in self.SourceFileList:
- if SrcFile.Ext.lower() != '.vfr':
- continue
- Vfri = os.path.join(self.OutputDir, SrcFile.BaseName + '.i')
- if not os.path.exists(Vfri):
- continue
- VfriFile = open(Vfri, 'r')
- Content = VfriFile.read()
- VfriFile.close()
- Pos = Content.find('efivarstore')
- while Pos != -1:
- #
- # Make sure 'efivarstore' is the start of efivarstore statement
- # In case of the value of 'name' (name = efivarstore) is equal to 'efivarstore'
- #
- Index = Pos - 1
- while Index >= 0 and Content[Index] in ' \t\r\n':
- Index -= 1
- if Index >= 0 and Content[Index] != ';':
- Pos = Content.find('efivarstore', Pos + len('efivarstore'))
- continue
- #
- # 'efivarstore' must be followed by name and guid
- #
- Name = gEfiVarStoreNamePattern.search(Content, Pos)
- if not Name:
- break
- Guid = gEfiVarStoreGuidPattern.search(Content, Pos)
- if not Guid:
- break
- NameArray = _ConvertStringToByteArray('L"' + Name.group(1) + '"')
- NameGuids.add((NameArray, GuidStructureStringToGuidString(Guid.group(1))))
- Pos = Content.find('efivarstore', Name.end())
- if not NameGuids:
- return []
- HiiExPcds = []
- for Pcd in self.PlatformInfo.Pcds.values():
- if Pcd.Type != TAB_PCDS_DYNAMIC_EX_HII:
- continue
- for SkuInfo in Pcd.SkuInfoList.values():
- Value = GuidValue(SkuInfo.VariableGuid, self.PlatformInfo.PackageList, self.MetaFile.Path)
- if not Value:
- continue
- Name = _ConvertStringToByteArray(SkuInfo.VariableName)
- Guid = GuidStructureStringToGuidString(Value)
- if (Name, Guid) in NameGuids and Pcd not in HiiExPcds:
- HiiExPcds.append(Pcd)
- break
-
- return HiiExPcds
-
- def _GenOffsetBin(self):
- VfrUniBaseName = {}
- for SourceFile in self.Module.Sources:
- if SourceFile.Type.upper() == ".VFR" :
- #
- # search the .map file to find the offset of vfr binary in the PE32+/TE file.
- #
- VfrUniBaseName[SourceFile.BaseName] = (SourceFile.BaseName + "Bin")
- elif SourceFile.Type.upper() == ".UNI" :
- #
- # search the .map file to find the offset of Uni strings binary in the PE32+/TE file.
- #
- VfrUniBaseName["UniOffsetName"] = (self.Name + "Strings")
-
- if not VfrUniBaseName:
- return None
- MapFileName = os.path.join(self.OutputDir, self.Name + ".map")
- EfiFileName = os.path.join(self.OutputDir, self.Name + ".efi")
- VfrUniOffsetList = GetVariableOffset(MapFileName, EfiFileName, list(VfrUniBaseName.values()))
- if not VfrUniOffsetList:
- return None
-
- OutputName = '%sOffset.bin' % self.Name
- UniVfrOffsetFileName = os.path.join( self.OutputDir, OutputName)
-
- try:
- fInputfile = open(UniVfrOffsetFileName, "wb+", 0)
- except:
- EdkLogger.error("build", FILE_OPEN_FAILURE, "File open failed for %s" % UniVfrOffsetFileName, None)
-
- # Use a instance of BytesIO to cache data
- fStringIO = BytesIO()
-
- for Item in VfrUniOffsetList:
- if (Item[0].find("Strings") != -1):
- #
- # UNI offset in image.
- # GUID + Offset
- # { 0x8913c5e0, 0x33f6, 0x4d86, { 0x9b, 0xf1, 0x43, 0xef, 0x89, 0xfc, 0x6, 0x66 } }
- #
- UniGuid = b'\xe0\xc5\x13\x89\xf63\x86M\x9b\xf1C\xef\x89\xfc\x06f'
- fStringIO.write(UniGuid)
- UniValue = pack ('Q', int (Item[1], 16))
- fStringIO.write (UniValue)
- else:
- #
- # VFR binary offset in image.
- # GUID + Offset
- # { 0xd0bc7cb4, 0x6a47, 0x495f, { 0xaa, 0x11, 0x71, 0x7, 0x46, 0xda, 0x6, 0xa2 } };
- #
- VfrGuid = b'\xb4|\xbc\xd0Gj_I\xaa\x11q\x07F\xda\x06\xa2'
- fStringIO.write(VfrGuid)
- VfrValue = pack ('Q', int (Item[1], 16))
- fStringIO.write (VfrValue)
- #
- # write data into file.
- #
- try :
- fInputfile.write (fStringIO.getvalue())
- except:
- EdkLogger.error("build", FILE_WRITE_FAILURE, "Write data to file %s failed, please check whether the "
- "file been locked or using by other applications." %UniVfrOffsetFileName, None)
-
- fStringIO.close ()
- fInputfile.close ()
- return OutputName
-
- @cached_property
- def OutputFile(self):
- retVal = set()
-
- for Root, Dirs, Files in os.walk(self.BuildDir):
- for File in Files:
- # lib file is already added through above CodaTargetList, skip it here
- if not (File.lower().endswith('.obj') or File.lower().endswith('.debug')):
- NewFile = path.join(Root, File)
- retVal.add(NewFile)
-
- for Root, Dirs, Files in os.walk(self.FfsOutputDir):
- for File in Files:
- NewFile = path.join(Root, File)
- retVal.add(NewFile)
-
- return retVal
-
- ## Create AsBuilt INF file the module
- #
- def CreateAsBuiltInf(self):
-
- if self.IsAsBuiltInfCreated:
- return
-
- # Skip INF file generation for libraries
- if self.IsLibrary:
- return
-
- # Skip the following code for modules with no source files
- if not self.SourceFileList:
- return
-
- # Skip the following code for modules without any binary files
- if self.BinaryFileList:
- return
-
- ### TODO: How to handles mixed source and binary modules
-
- # Find all DynamicEx and PatchableInModule PCDs used by this module and dependent libraries
- # Also find all packages that the DynamicEx PCDs depend on
- Pcds = []
- PatchablePcds = []
- Packages = []
- PcdCheckList = []
- PcdTokenSpaceList = []
- for Pcd in self.ModulePcdList + self.LibraryPcdList:
- if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE:
- PatchablePcds.append(Pcd)
- PcdCheckList.append((Pcd.TokenCName, Pcd.TokenSpaceGuidCName, TAB_PCDS_PATCHABLE_IN_MODULE))
- elif Pcd.Type in PCD_DYNAMIC_EX_TYPE_SET:
- if Pcd not in Pcds:
- Pcds.append(Pcd)
- PcdCheckList.append((Pcd.TokenCName, Pcd.TokenSpaceGuidCName, TAB_PCDS_DYNAMIC_EX))
- PcdCheckList.append((Pcd.TokenCName, Pcd.TokenSpaceGuidCName, TAB_PCDS_DYNAMIC))
- PcdTokenSpaceList.append(Pcd.TokenSpaceGuidCName)
- GuidList = OrderedDict(self.GuidList)
- for TokenSpace in self.GetGuidsUsedByPcd:
- # If token space is not referred by patch PCD or Ex PCD, remove the GUID from GUID list
- # The GUIDs in GUIDs section should really be the GUIDs in source INF or referred by Ex an patch PCDs
- if TokenSpace not in PcdTokenSpaceList and TokenSpace in GuidList:
- GuidList.pop(TokenSpace)
- CheckList = (GuidList, self.PpiList, self.ProtocolList, PcdCheckList)
- for Package in self.DerivedPackageList:
- if Package in Packages:
- continue
- BeChecked = (Package.Guids, Package.Ppis, Package.Protocols, Package.Pcds)
- Found = False
- for Index in range(len(BeChecked)):
- for Item in CheckList[Index]:
- if Item in BeChecked[Index]:
- Packages.append(Package)
- Found = True
- break
- if Found:
- break
-
- VfrPcds = self._GetPcdsMaybeUsedByVfr()
- for Pkg in self.PlatformInfo.PackageList:
- if Pkg in Packages:
- continue
- for VfrPcd in VfrPcds:
- if ((VfrPcd.TokenCName, VfrPcd.TokenSpaceGuidCName, TAB_PCDS_DYNAMIC_EX) in Pkg.Pcds or
- (VfrPcd.TokenCName, VfrPcd.TokenSpaceGuidCName, TAB_PCDS_DYNAMIC) in Pkg.Pcds):
- Packages.append(Pkg)
- break
-
- ModuleType = SUP_MODULE_DXE_DRIVER if self.ModuleType == SUP_MODULE_UEFI_DRIVER and self.DepexGenerated else self.ModuleType
- DriverType = self.PcdIsDriver if self.PcdIsDriver else ''
- Guid = self.Guid
- MDefs = self.Module.Defines
-
- AsBuiltInfDict = {
- 'module_name' : self.Name,
- 'module_guid' : Guid,
- 'module_module_type' : ModuleType,
- 'module_version_string' : [MDefs['VERSION_STRING']] if 'VERSION_STRING' in MDefs else [],
- 'pcd_is_driver_string' : [],
- 'module_uefi_specification_version' : [],
- 'module_pi_specification_version' : [],
- 'module_entry_point' : self.Module.ModuleEntryPointList,
- 'module_unload_image' : self.Module.ModuleUnloadImageList,
- 'module_constructor' : self.Module.ConstructorList,
- 'module_destructor' : self.Module.DestructorList,
- 'module_shadow' : [MDefs['SHADOW']] if 'SHADOW' in MDefs else [],
- 'module_pci_vendor_id' : [MDefs['PCI_VENDOR_ID']] if 'PCI_VENDOR_ID' in MDefs else [],
- 'module_pci_device_id' : [MDefs['PCI_DEVICE_ID']] if 'PCI_DEVICE_ID' in MDefs else [],
- 'module_pci_class_code' : [MDefs['PCI_CLASS_CODE']] if 'PCI_CLASS_CODE' in MDefs else [],
- 'module_pci_revision' : [MDefs['PCI_REVISION']] if 'PCI_REVISION' in MDefs else [],
- 'module_build_number' : [MDefs['BUILD_NUMBER']] if 'BUILD_NUMBER' in MDefs else [],
- 'module_spec' : [MDefs['SPEC']] if 'SPEC' in MDefs else [],
- 'module_uefi_hii_resource_section' : [MDefs['UEFI_HII_RESOURCE_SECTION']] if 'UEFI_HII_RESOURCE_SECTION' in MDefs else [],
- 'module_uni_file' : [MDefs['MODULE_UNI_FILE']] if 'MODULE_UNI_FILE' in MDefs else [],
- 'module_arch' : self.Arch,
- 'package_item' : [Package.MetaFile.File.replace('\\', '/') for Package in Packages],
- 'binary_item' : [],
- 'patchablepcd_item' : [],
- 'pcd_item' : [],
- 'protocol_item' : [],
- 'ppi_item' : [],
- 'guid_item' : [],
- 'flags_item' : [],
- 'libraryclasses_item' : []
- }
-
- if 'MODULE_UNI_FILE' in MDefs:
- UNIFile = os.path.join(self.MetaFile.Dir, MDefs['MODULE_UNI_FILE'])
- if os.path.isfile(UNIFile):
- shutil.copy2(UNIFile, self.OutputDir)
-
- if self.AutoGenVersion > int(gInfSpecVersion, 0):
- AsBuiltInfDict['module_inf_version'] = '0x%08x' % self.AutoGenVersion
- else:
- AsBuiltInfDict['module_inf_version'] = gInfSpecVersion
-
- if DriverType:
- AsBuiltInfDict['pcd_is_driver_string'].append(DriverType)
-
- if 'UEFI_SPECIFICATION_VERSION' in self.Specification:
- AsBuiltInfDict['module_uefi_specification_version'].append(self.Specification['UEFI_SPECIFICATION_VERSION'])
- if 'PI_SPECIFICATION_VERSION' in self.Specification:
- AsBuiltInfDict['module_pi_specification_version'].append(self.Specification['PI_SPECIFICATION_VERSION'])
-
- OutputDir = self.OutputDir.replace('\\', '/').strip('/')
- DebugDir = self.DebugDir.replace('\\', '/').strip('/')
- for Item in self.CodaTargetList:
- File = Item.Target.Path.replace('\\', '/').strip('/').replace(DebugDir, '').replace(OutputDir, '').strip('/')
- if os.path.isabs(File):
- File = File.replace('\\', '/').strip('/').replace(OutputDir, '').strip('/')
- if Item.Target.Ext.lower() == '.aml':
- AsBuiltInfDict['binary_item'].append('ASL|' + File)
- elif Item.Target.Ext.lower() == '.acpi':
- AsBuiltInfDict['binary_item'].append('ACPI|' + File)
- elif Item.Target.Ext.lower() == '.efi':
- AsBuiltInfDict['binary_item'].append('PE32|' + self.Name + '.efi')
- else:
- AsBuiltInfDict['binary_item'].append('BIN|' + File)
- if not self.DepexGenerated:
- DepexFile = os.path.join(self.OutputDir, self.Name + '.depex')
- if os.path.exists(DepexFile):
- self.DepexGenerated = True
- if self.DepexGenerated:
- if self.ModuleType in [SUP_MODULE_PEIM]:
- AsBuiltInfDict['binary_item'].append('PEI_DEPEX|' + self.Name + '.depex')
- elif self.ModuleType in [SUP_MODULE_DXE_DRIVER, SUP_MODULE_DXE_RUNTIME_DRIVER, SUP_MODULE_DXE_SAL_DRIVER, SUP_MODULE_UEFI_DRIVER]:
- AsBuiltInfDict['binary_item'].append('DXE_DEPEX|' + self.Name + '.depex')
- elif self.ModuleType in [SUP_MODULE_DXE_SMM_DRIVER]:
- AsBuiltInfDict['binary_item'].append('SMM_DEPEX|' + self.Name + '.depex')
-
- Bin = self._GenOffsetBin()
- if Bin:
- AsBuiltInfDict['binary_item'].append('BIN|%s' % Bin)
-
- for Root, Dirs, Files in os.walk(OutputDir):
- for File in Files:
- if File.lower().endswith('.pdb'):
- AsBuiltInfDict['binary_item'].append('DISPOSABLE|' + File)
- HeaderComments = self.Module.HeaderComments
- StartPos = 0
- for Index in range(len(HeaderComments)):
- if HeaderComments[Index].find('@BinaryHeader') != -1:
- HeaderComments[Index] = HeaderComments[Index].replace('@BinaryHeader', '@file')
- StartPos = Index
- break
- AsBuiltInfDict['header_comments'] = '\n'.join(HeaderComments[StartPos:]).replace(':#', '://')
- AsBuiltInfDict['tail_comments'] = '\n'.join(self.Module.TailComments)
-
- GenList = [
- (self.ProtocolList, self._ProtocolComments, 'protocol_item'),
- (self.PpiList, self._PpiComments, 'ppi_item'),
- (GuidList, self._GuidComments, 'guid_item')
- ]
- for Item in GenList:
- for CName in Item[0]:
- Comments = '\n '.join(Item[1][CName]) if CName in Item[1] else ''
- Entry = Comments + '\n ' + CName if Comments else CName
- AsBuiltInfDict[Item[2]].append(Entry)
- PatchList = parsePcdInfoFromMapFile(
- os.path.join(self.OutputDir, self.Name + '.map'),
- os.path.join(self.OutputDir, self.Name + '.efi')
- )
- if PatchList:
- for Pcd in PatchablePcds:
- TokenCName = Pcd.TokenCName
- for PcdItem in GlobalData.MixedPcd:
- if (Pcd.TokenCName, Pcd.TokenSpaceGuidCName) in GlobalData.MixedPcd[PcdItem]:
- TokenCName = PcdItem[0]
- break
- for PatchPcd in PatchList:
- if TokenCName == PatchPcd[0]:
- break
- else:
- continue
- PcdValue = ''
- if Pcd.DatumType == 'BOOLEAN':
- BoolValue = Pcd.DefaultValue.upper()
- if BoolValue == 'TRUE':
- Pcd.DefaultValue = '1'
- elif BoolValue == 'FALSE':
- Pcd.DefaultValue = '0'
-
- if Pcd.DatumType in TAB_PCD_NUMERIC_TYPES:
- HexFormat = '0x%02x'
- if Pcd.DatumType == TAB_UINT16:
- HexFormat = '0x%04x'
- elif Pcd.DatumType == TAB_UINT32:
- HexFormat = '0x%08x'
- elif Pcd.DatumType == TAB_UINT64:
- HexFormat = '0x%016x'
- PcdValue = HexFormat % int(Pcd.DefaultValue, 0)
- else:
- if Pcd.MaxDatumSize is None or Pcd.MaxDatumSize == '':
- EdkLogger.error("build", AUTOGEN_ERROR,
- "Unknown [MaxDatumSize] of PCD [%s.%s]" % (Pcd.TokenSpaceGuidCName, TokenCName)
- )
- ArraySize = int(Pcd.MaxDatumSize, 0)
- PcdValue = Pcd.DefaultValue
- if PcdValue[0] != '{':
- Unicode = False
- if PcdValue[0] == 'L':
- Unicode = True
- PcdValue = PcdValue.lstrip('L')
- PcdValue = eval(PcdValue)
- NewValue = '{'
- for Index in range(0, len(PcdValue)):
- if Unicode:
- CharVal = ord(PcdValue[Index])
- NewValue = NewValue + '0x%02x' % (CharVal & 0x00FF) + ', ' \
- + '0x%02x' % (CharVal >> 8) + ', '
- else:
- NewValue = NewValue + '0x%02x' % (ord(PcdValue[Index]) % 0x100) + ', '
- Padding = '0x00, '
- if Unicode:
- Padding = Padding * 2
- ArraySize = ArraySize // 2
- if ArraySize < (len(PcdValue) + 1):
- if Pcd.MaxSizeUserSet:
- EdkLogger.error("build", AUTOGEN_ERROR,
- "The maximum size of VOID* type PCD '%s.%s' is less than its actual size occupied." % (Pcd.TokenSpaceGuidCName, TokenCName)
- )
- else:
- ArraySize = len(PcdValue) + 1
- if ArraySize > len(PcdValue) + 1:
- NewValue = NewValue + Padding * (ArraySize - len(PcdValue) - 1)
- PcdValue = NewValue + Padding.strip().rstrip(',') + '}'
- elif len(PcdValue.split(',')) <= ArraySize:
- PcdValue = PcdValue.rstrip('}') + ', 0x00' * (ArraySize - len(PcdValue.split(',')))
- PcdValue += '}'
- else:
- if Pcd.MaxSizeUserSet:
- EdkLogger.error("build", AUTOGEN_ERROR,
- "The maximum size of VOID* type PCD '%s.%s' is less than its actual size occupied." % (Pcd.TokenSpaceGuidCName, TokenCName)
- )
- else:
- ArraySize = len(PcdValue) + 1
- PcdItem = '%s.%s|%s|0x%X' % \
- (Pcd.TokenSpaceGuidCName, TokenCName, PcdValue, PatchPcd[1])
- PcdComments = ''
- if (Pcd.TokenSpaceGuidCName, Pcd.TokenCName) in self._PcdComments:
- PcdComments = '\n '.join(self._PcdComments[Pcd.TokenSpaceGuidCName, Pcd.TokenCName])
- if PcdComments:
- PcdItem = PcdComments + '\n ' + PcdItem
- AsBuiltInfDict['patchablepcd_item'].append(PcdItem)
-
- for Pcd in Pcds + VfrPcds:
- PcdCommentList = []
- HiiInfo = ''
- TokenCName = Pcd.TokenCName
- for PcdItem in GlobalData.MixedPcd:
- if (Pcd.TokenCName, Pcd.TokenSpaceGuidCName) in GlobalData.MixedPcd[PcdItem]:
- TokenCName = PcdItem[0]
- break
- if Pcd.Type == TAB_PCDS_DYNAMIC_EX_HII:
- for SkuName in Pcd.SkuInfoList:
- SkuInfo = Pcd.SkuInfoList[SkuName]
- HiiInfo = '## %s|%s|%s' % (SkuInfo.VariableName, SkuInfo.VariableGuid, SkuInfo.VariableOffset)
- break
- if (Pcd.TokenSpaceGuidCName, Pcd.TokenCName) in self._PcdComments:
- PcdCommentList = self._PcdComments[Pcd.TokenSpaceGuidCName, Pcd.TokenCName][:]
- if HiiInfo:
- UsageIndex = -1
- UsageStr = ''
- for Index, Comment in enumerate(PcdCommentList):
- for Usage in UsageList:
- if Comment.find(Usage) != -1:
- UsageStr = Usage
- UsageIndex = Index
- break
- if UsageIndex != -1:
- PcdCommentList[UsageIndex] = '## %s %s %s' % (UsageStr, HiiInfo, PcdCommentList[UsageIndex].replace(UsageStr, ''))
- else:
- PcdCommentList.append('## UNDEFINED ' + HiiInfo)
- PcdComments = '\n '.join(PcdCommentList)
- PcdEntry = Pcd.TokenSpaceGuidCName + '.' + TokenCName
- if PcdComments:
- PcdEntry = PcdComments + '\n ' + PcdEntry
- AsBuiltInfDict['pcd_item'].append(PcdEntry)
- for Item in self.BuildOption:
- if 'FLAGS' in self.BuildOption[Item]:
- AsBuiltInfDict['flags_item'].append('%s:%s_%s_%s_%s_FLAGS = %s' % (self.ToolChainFamily, self.BuildTarget, self.ToolChain, self.Arch, Item, self.BuildOption[Item]['FLAGS'].strip()))
-
- # Generated LibraryClasses section in comments.
- for Library in self.LibraryAutoGenList:
- AsBuiltInfDict['libraryclasses_item'].append(Library.MetaFile.File.replace('\\', '/'))
-
- # Generated UserExtensions TianoCore section.
- # All tianocore user extensions are copied.
- UserExtStr = ''
- for TianoCore in self._GetTianoCoreUserExtensionList():
- UserExtStr += '\n'.join(TianoCore)
- ExtensionFile = os.path.join(self.MetaFile.Dir, TianoCore[1])
- if os.path.isfile(ExtensionFile):
- shutil.copy2(ExtensionFile, self.OutputDir)
- AsBuiltInfDict['userextension_tianocore_item'] = UserExtStr
-
- # Generated depex expression section in comments.
- DepexExpression = self._GetDepexExpresionString()
- AsBuiltInfDict['depexsection_item'] = DepexExpression if DepexExpression else ''
-
- AsBuiltInf = TemplateString()
- AsBuiltInf.Append(gAsBuiltInfHeaderString.Replace(AsBuiltInfDict))
-
- SaveFileOnChange(os.path.join(self.OutputDir, self.Name + '.inf'), str(AsBuiltInf), False)
-
- self.IsAsBuiltInfCreated = True
-
- def CacheCopyFile(self, DestDir, SourceDir, File):
- if os.path.isdir(File):
- return
-
- sub_dir = os.path.relpath(File, SourceDir)
- destination_file = os.path.join(DestDir, sub_dir)
- destination_dir = os.path.dirname(destination_file)
- CreateDirectory(destination_dir)
- try:
- CopyFileOnChange(File, destination_dir)
- except:
- EdkLogger.quiet("[cache warning]: fail to copy file:%s to folder:%s" % (File, destination_dir))
- return
-
- def CopyModuleToCache(self):
- # Find the MakeHashStr and PreMakeHashStr from latest MakeHashFileList
- # and PreMakeHashFileList files
- MakeHashStr = None
- PreMakeHashStr = None
- MakeTimeStamp = 0
- PreMakeTimeStamp = 0
- Files = [f for f in os.listdir(LongFilePath(self.BuildDir)) if path.isfile(LongFilePath(path.join(self.BuildDir, f)))]
- for File in Files:
- if ".MakeHashFileList." in File:
- #find lastest file through time stamp
- FileTimeStamp = os.stat(LongFilePath(path.join(self.BuildDir, File)))[8]
- if FileTimeStamp > MakeTimeStamp:
- MakeTimeStamp = FileTimeStamp
- MakeHashStr = File.split('.')[-1]
- if len(MakeHashStr) != 32:
- EdkLogger.quiet("[cache error]: wrong MakeHashFileList file:%s" % (File))
- if ".PreMakeHashFileList." in File:
- FileTimeStamp = os.stat(LongFilePath(path.join(self.BuildDir, File)))[8]
- if FileTimeStamp > PreMakeTimeStamp:
- PreMakeTimeStamp = FileTimeStamp
- PreMakeHashStr = File.split('.')[-1]
- if len(PreMakeHashStr) != 32:
- EdkLogger.quiet("[cache error]: wrong PreMakeHashFileList file:%s" % (File))
-
- if not MakeHashStr:
- EdkLogger.quiet("[cache error]: No MakeHashFileList file for module:%s[%s]" % (self.MetaFile.Path, self.Arch))
- return
- if not PreMakeHashStr:
- EdkLogger.quiet("[cache error]: No PreMakeHashFileList file for module:%s[%s]" % (self.MetaFile.Path, self.Arch))
- return
-
- # Create Cache destination dirs
- FileDir = path.join(GlobalData.gBinCacheDest, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName)
- FfsDir = path.join(GlobalData.gBinCacheDest, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name)
- CacheFileDir = path.join(FileDir, MakeHashStr)
- CacheFfsDir = path.join(FfsDir, MakeHashStr)
- CreateDirectory (CacheFileDir)
- CreateDirectory (CacheFfsDir)
-
- # Create ModuleHashPair file to support multiple version cache together
- ModuleHashPair = path.join(FileDir, self.Name + ".ModuleHashPair")
- ModuleHashPairList = [] # tuple list: [tuple(PreMakefileHash, MakeHash)]
- if os.path.exists(ModuleHashPair):
- with open(ModuleHashPair, 'r') as f:
- ModuleHashPairList = json.load(f)
- if not (PreMakeHashStr, MakeHashStr) in set(map(tuple, ModuleHashPairList)):
- ModuleHashPairList.insert(0, (PreMakeHashStr, MakeHashStr))
- with open(ModuleHashPair, 'w') as f:
- json.dump(ModuleHashPairList, f, indent=2)
-
- # Copy files to Cache destination dirs
- if not self.OutputFile:
- Ma = self.BuildDatabase[self.MetaFile, self.Arch, self.BuildTarget, self.ToolChain]
- self.OutputFile = Ma.Binaries
- for File in self.OutputFile:
- if File.startswith(os.path.abspath(self.FfsOutputDir)+os.sep):
- self.CacheCopyFile(CacheFfsDir, self.FfsOutputDir, File)
- else:
- if self.Name + ".autogen.hash." in File or \
- self.Name + ".autogen.hashchain." in File or \
- self.Name + ".hash." in File or \
- self.Name + ".hashchain." in File or \
- self.Name + ".PreMakeHashFileList." in File or \
- self.Name + ".MakeHashFileList." in File:
- self.CacheCopyFile(FileDir, self.BuildDir, File)
- else:
- self.CacheCopyFile(CacheFileDir, self.BuildDir, File)
- ## Create makefile for the module and its dependent libraries
- #
- # @param CreateLibraryMakeFile Flag indicating if or not the makefiles of
- # dependent libraries will be created
- #
- @cached_class_function
- def CreateMakeFile(self, CreateLibraryMakeFile=True, GenFfsList = []):
-
- # nest this function inside it's only caller.
- def CreateTimeStamp():
- FileSet = {self.MetaFile.Path}
-
- for SourceFile in self.Module.Sources:
- FileSet.add (SourceFile.Path)
-
- for Lib in self.DependentLibraryList:
- FileSet.add (Lib.MetaFile.Path)
-
- for f in self.AutoGenDepSet:
- FileSet.add (f.Path)
-
- if os.path.exists (self.TimeStampPath):
- os.remove (self.TimeStampPath)
-
- SaveFileOnChange(self.TimeStampPath, "\n".join(FileSet), False)
-
- # Ignore generating makefile when it is a binary module
- if self.IsBinaryModule:
- return
-
- self.GenFfsList = GenFfsList
-
- if not self.IsLibrary and CreateLibraryMakeFile:
- for LibraryAutoGen in self.LibraryAutoGenList:
- LibraryAutoGen.CreateMakeFile()
-
- # CanSkip uses timestamps to determine build skipping
- if self.CanSkip():
- return
-
- if len(self.CustomMakefile) == 0:
- Makefile = GenMake.ModuleMakefile(self)
- else:
- Makefile = GenMake.CustomMakefile(self)
- if Makefile.Generate():
- EdkLogger.debug(EdkLogger.DEBUG_9, "Generated makefile for module %s [%s]" %
- (self.Name, self.Arch))
- else:
- EdkLogger.debug(EdkLogger.DEBUG_9, "Skipped the generation of makefile for module %s [%s]" %
- (self.Name, self.Arch))
-
- CreateTimeStamp()
-
- MakefileType = Makefile._FileType
- MakefileName = Makefile._FILE_NAME_[MakefileType]
- MakefilePath = os.path.join(self.MakeFileDir, MakefileName)
- FilePath = path.join(self.BuildDir, self.Name + ".makefile")
- SaveFileOnChange(FilePath, MakefilePath, False)
-
- def CopyBinaryFiles(self):
- for File in self.Module.Binaries:
- SrcPath = File.Path
- DstPath = os.path.join(self.OutputDir, os.path.basename(SrcPath))
- CopyLongFilePath(SrcPath, DstPath)
- ## Create autogen code for the module and its dependent libraries
- #
- # @param CreateLibraryCodeFile Flag indicating if or not the code of
- # dependent libraries will be created
- #
- def CreateCodeFile(self, CreateLibraryCodeFile=True):
-
- if self.IsCodeFileCreated:
- return
-
- # Need to generate PcdDatabase even PcdDriver is binarymodule
- if self.IsBinaryModule and self.PcdIsDriver != '':
- CreatePcdDatabaseCode(self, TemplateString(), TemplateString())
- return
- if self.IsBinaryModule:
- if self.IsLibrary:
- self.CopyBinaryFiles()
- return
-
- if not self.IsLibrary and CreateLibraryCodeFile:
- for LibraryAutoGen in self.LibraryAutoGenList:
- LibraryAutoGen.CreateCodeFile()
-
- self.LibraryAutoGenList
- AutoGenList = []
- IgoredAutoGenList = []
-
- for File in self.AutoGenFileList:
- if GenC.Generate(File.Path, self.AutoGenFileList[File], File.IsBinary):
- AutoGenList.append(str(File))
- else:
- IgoredAutoGenList.append(str(File))
-
-
- for ModuleType in self.DepexList:
- # Ignore empty [depex] section or [depex] section for SUP_MODULE_USER_DEFINED module
- if len(self.DepexList[ModuleType]) == 0 or ModuleType == SUP_MODULE_USER_DEFINED or ModuleType == SUP_MODULE_HOST_APPLICATION:
- continue
-
- Dpx = GenDepex.DependencyExpression(self.DepexList[ModuleType], ModuleType, True)
- DpxFile = gAutoGenDepexFileName % {"module_name" : self.Name}
-
- if len(Dpx.PostfixNotation) != 0:
- self.DepexGenerated = True
-
- if Dpx.Generate(path.join(self.OutputDir, DpxFile)):
- AutoGenList.append(str(DpxFile))
- else:
- IgoredAutoGenList.append(str(DpxFile))
-
- if IgoredAutoGenList == []:
- EdkLogger.debug(EdkLogger.DEBUG_9, "Generated [%s] files for module %s [%s]" %
- (" ".join(AutoGenList), self.Name, self.Arch))
- elif AutoGenList == []:
- EdkLogger.debug(EdkLogger.DEBUG_9, "Skipped the generation of [%s] files for module %s [%s]" %
- (" ".join(IgoredAutoGenList), self.Name, self.Arch))
- else:
- EdkLogger.debug(EdkLogger.DEBUG_9, "Generated [%s] (skipped %s) files for module %s [%s]" %
- (" ".join(AutoGenList), " ".join(IgoredAutoGenList), self.Name, self.Arch))
-
- self.IsCodeFileCreated = True
-
- return AutoGenList
-
- ## Summarize the ModuleAutoGen objects of all libraries used by this module
- @cached_property
- def LibraryAutoGenList(self):
- RetVal = []
- for Library in self.DependentLibraryList:
- La = ModuleAutoGen(
- self.Workspace,
- Library.MetaFile,
- self.BuildTarget,
- self.ToolChain,
- self.Arch,
- self.PlatformInfo.MetaFile,
- self.DataPipe
- )
- La.IsLibrary = True
- if La not in RetVal:
- RetVal.append(La)
- for Lib in La.CodaTargetList:
- self._ApplyBuildRule(Lib.Target, TAB_UNKNOWN_FILE)
- return RetVal
-
- def GenCMakeHash(self):
- # GenCMakeHash can only be called in --binary-destination
- # Never called in multiprocessing and always directly save result in main process,
- # so no need remote dict to share the gCMakeHashFile result with main process
-
- DependencyFileSet = set()
- # Add AutoGen files
- if self.AutoGenFileList:
- for File in set(self.AutoGenFileList):
- DependencyFileSet.add(File)
-
- # Add Makefile
- abspath = path.join(self.BuildDir, self.Name + ".makefile")
- try:
- with open(LongFilePath(abspath),"r") as fd:
- lines = fd.readlines()
- except Exception as e:
- EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
- if lines:
- DependencyFileSet.update(lines)
-
- # Caculate all above dependency files hash
- # Initialze hash object
- FileList = []
- m = hashlib.md5()
- for File in sorted(DependencyFileSet, key=lambda x: str(x)):
- if not path.exists(LongFilePath(str(File))):
- EdkLogger.quiet("[cache warning]: header file %s is missing for module: %s[%s]" % (File, self.MetaFile.Path, self.Arch))
- continue
- with open(LongFilePath(str(File)), 'rb') as f:
- Content = f.read()
- m.update(Content)
- FileList.append((str(File), hashlib.md5(Content).hexdigest()))
-
- HashChainFile = path.join(self.BuildDir, self.Name + ".autogen.hashchain." + m.hexdigest())
- GlobalData.gCMakeHashFile[(self.MetaFile.Path, self.Arch)] = HashChainFile
- try:
- with open(LongFilePath(HashChainFile), 'w') as f:
- json.dump(FileList, f, indent=2)
- except:
- EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile)
- return False
-
- def GenModuleHash(self):
- # GenModuleHash only called after autogen phase
- # Never called in multiprocessing and always directly save result in main process,
- # so no need remote dict to share the gModuleHashFile result with main process
- #
- # GenPreMakefileHashList consume no dict.
- # GenPreMakefileHashList produce local gModuleHashFile dict.
-
- DependencyFileSet = set()
- # Add Module Meta file
- DependencyFileSet.add(self.MetaFile.Path)
-
- # Add Module's source files
- if self.SourceFileList:
- for File in set(self.SourceFileList):
- DependencyFileSet.add(File.Path)
-
- # Add modules's include header files
- # Directly use the deps.txt file in the module BuildDir
- abspath = path.join(self.BuildDir, "deps.txt")
- rt = None
- try:
- with open(LongFilePath(abspath),"r") as fd:
- lines = fd.readlines()
- if lines:
- rt = set([item.lstrip().strip("\n") for item in lines if item.strip("\n").endswith(".h")])
- except Exception as e:
- EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False)
-
- if rt:
- DependencyFileSet.update(rt)
-
-
- # Caculate all above dependency files hash
- # Initialze hash object
- FileList = []
- m = hashlib.md5()
- BuildDirStr = path.abspath(self.BuildDir).lower()
- for File in sorted(DependencyFileSet, key=lambda x: str(x)):
- # Skip the AutoGen files in BuildDir which already been
- # included in .autogen.hash. file
- if BuildDirStr in path.abspath(File).lower():
- continue
- if not path.exists(LongFilePath(File)):
- EdkLogger.quiet("[cache warning]: header file %s is missing for module: %s[%s]" % (File, self.MetaFile.Path, self.Arch))
- continue
- with open(LongFilePath(File), 'rb') as f:
- Content = f.read()
- m.update(Content)
- FileList.append((File, hashlib.md5(Content).hexdigest()))
-
- HashChainFile = path.join(self.BuildDir, self.Name + ".hashchain." + m.hexdigest())
- GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)] = HashChainFile
- try:
- with open(LongFilePath(HashChainFile), 'w') as f:
- json.dump(FileList, f, indent=2)
- except:
- EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile)
- return False
-
- def GenPreMakefileHashList(self):
- # GenPreMakefileHashList consume below dicts:
- # gPlatformHashFile
- # gPackageHashFile
- # gModuleHashFile
- # GenPreMakefileHashList produce no dict.
- # gModuleHashFile items might be produced in multiprocessing, so
- # need check gModuleHashFile remote dict
-
- # skip binary module
- if self.IsBinaryModule:
- return
-
- FileList = []
- m = hashlib.md5()
- # Add Platform level hash
- HashFile = GlobalData.gPlatformHashFile
- if path.exists(LongFilePath(HashFile)):
- FileList.append(HashFile)
- m.update(HashFile.encode('utf-8'))
- else:
- EdkLogger.quiet("[cache warning]: No Platform HashFile: %s" % HashFile)
-
- # Add Package level hash
- if self.DependentPackageList:
- for Pkg in sorted(self.DependentPackageList, key=lambda x: x.PackageName):
- if not (Pkg.PackageName, Pkg.Arch) in GlobalData.gPackageHashFile:
- EdkLogger.quiet("[cache warning]:No Package %s for module %s[%s]" % (Pkg.PackageName, self.MetaFile.Path, self.Arch))
- continue
- HashFile = GlobalData.gPackageHashFile[(Pkg.PackageName, Pkg.Arch)]
- if path.exists(LongFilePath(HashFile)):
- FileList.append(HashFile)
- m.update(HashFile.encode('utf-8'))
- else:
- EdkLogger.quiet("[cache warning]:No Package HashFile: %s" % HashFile)
-
- # Add Module self
- # GenPreMakefileHashList needed in both --binary-destination
- # and --hash. And --hash might save ModuleHashFile in remote dict
- # during multiprocessing.
- if (self.MetaFile.Path, self.Arch) in GlobalData.gModuleHashFile:
- HashFile = GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)]
- else:
- EdkLogger.quiet("[cache error]:No ModuleHashFile for module: %s[%s]" % (self.MetaFile.Path, self.Arch))
- if path.exists(LongFilePath(HashFile)):
- FileList.append(HashFile)
- m.update(HashFile.encode('utf-8'))
- else:
- EdkLogger.quiet("[cache warning]:No Module HashFile: %s" % HashFile)
-
- # Add Library hash
- if self.LibraryAutoGenList:
- for Lib in sorted(self.LibraryAutoGenList, key=lambda x: x.MetaFile.Path):
-
- if (Lib.MetaFile.Path, Lib.Arch) in GlobalData.gModuleHashFile:
- HashFile = GlobalData.gModuleHashFile[(Lib.MetaFile.Path, Lib.Arch)]
- else:
- EdkLogger.quiet("[cache error]:No ModuleHashFile for lib: %s[%s]" % (Lib.MetaFile.Path, Lib.Arch))
- if path.exists(LongFilePath(HashFile)):
- FileList.append(HashFile)
- m.update(HashFile.encode('utf-8'))
- else:
- EdkLogger.quiet("[cache warning]:No Lib HashFile: %s" % HashFile)
-
- # Save PreMakeHashFileList
- FilePath = path.join(self.BuildDir, self.Name + ".PreMakeHashFileList." + m.hexdigest())
- try:
- with open(LongFilePath(FilePath), 'w') as f:
- json.dump(FileList, f, indent=0)
- except:
- EdkLogger.quiet("[cache warning]: fail to save PreMake HashFileList: %s" % FilePath)
-
- def GenMakefileHashList(self):
- # GenMakefileHashList only need in --binary-destination which will
- # everything in local dict. So don't need check remote dict.
-
- # skip binary module
- if self.IsBinaryModule:
- return
-
- FileList = []
- m = hashlib.md5()
- # Add AutoGen hash
- HashFile = GlobalData.gCMakeHashFile[(self.MetaFile.Path, self.Arch)]
- if path.exists(LongFilePath(HashFile)):
- FileList.append(HashFile)
- m.update(HashFile.encode('utf-8'))
- else:
- EdkLogger.quiet("[cache warning]:No AutoGen HashFile: %s" % HashFile)
-
- # Add Module self
- if (self.MetaFile.Path, self.Arch) in GlobalData.gModuleHashFile:
- HashFile = GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)]
- else:
- EdkLogger.quiet("[cache error]:No ModuleHashFile for module: %s[%s]" % (self.MetaFile.Path, self.Arch))
- if path.exists(LongFilePath(HashFile)):
- FileList.append(HashFile)
- m.update(HashFile.encode('utf-8'))
- else:
- EdkLogger.quiet("[cache warning]:No Module HashFile: %s" % HashFile)
-
- # Add Library hash
- if self.LibraryAutoGenList:
- for Lib in sorted(self.LibraryAutoGenList, key=lambda x: x.MetaFile.Path):
- if (Lib.MetaFile.Path, Lib.Arch) in GlobalData.gModuleHashFile:
- HashFile = GlobalData.gModuleHashFile[(Lib.MetaFile.Path, Lib.Arch)]
- else:
- EdkLogger.quiet("[cache error]:No ModuleHashFile for lib: %s[%s]" % (Lib.MetaFile.Path, Lib.Arch))
- if path.exists(LongFilePath(HashFile)):
- FileList.append(HashFile)
- m.update(HashFile.encode('utf-8'))
- else:
- EdkLogger.quiet("[cache warning]:No Lib HashFile: %s" % HashFile)
-
- # Save MakeHashFileList
- FilePath = path.join(self.BuildDir, self.Name + ".MakeHashFileList." + m.hexdigest())
- try:
- with open(LongFilePath(FilePath), 'w') as f:
- json.dump(FileList, f, indent=0)
- except:
- EdkLogger.quiet("[cache warning]: fail to save Make HashFileList: %s" % FilePath)
-
- def CheckHashChainFile(self, HashChainFile):
- # Assume the HashChainFile basename format is the 'x.hashchain.16BytesHexStr'
- # The x is module name and the 16BytesHexStr is md5 hexdigest of
- # all hashchain files content
- HashStr = HashChainFile.split('.')[-1]
- if len(HashStr) != 32:
- EdkLogger.quiet("[cache error]: wrong format HashChainFile:%s" % (File))
- return False
-
- try:
- with open(LongFilePath(HashChainFile), 'r') as f:
- HashChainList = json.load(f)
- except:
- EdkLogger.quiet("[cache error]: fail to load HashChainFile: %s" % HashChainFile)
- return False
-
- # Print the different file info
- # print(HashChainFile)
- for idx, (SrcFile, SrcHash) in enumerate (HashChainList):
- if SrcFile in GlobalData.gFileHashDict:
- DestHash = GlobalData.gFileHashDict[SrcFile]
- else:
- try:
- with open(LongFilePath(SrcFile), 'rb') as f:
- Content = f.read()
- DestHash = hashlib.md5(Content).hexdigest()
- GlobalData.gFileHashDict[SrcFile] = DestHash
- except IOError as X:
- # cache miss if SrcFile is removed in new version code
- GlobalData.gFileHashDict[SrcFile] = 0
- EdkLogger.quiet("[cache insight]: first cache miss file in %s is %s" % (HashChainFile, SrcFile))
- return False
- if SrcHash != DestHash:
- EdkLogger.quiet("[cache insight]: first cache miss file in %s is %s" % (HashChainFile, SrcFile))
- return False
-
- return True
-
- ## Decide whether we can skip the left autogen and make process
- def CanSkipbyMakeCache(self):
- # For --binary-source only
- # CanSkipbyMakeCache consume below dicts:
- # gModuleMakeCacheStatus
- # gHashChainStatus
- # GenPreMakefileHashList produce gModuleMakeCacheStatus, gModuleHashFile dict.
- # all these dicts might be produced in multiprocessing, so
- # need check these remote dict
-
- if not GlobalData.gBinCacheSource:
- return False
-
- if (self.MetaFile.Path, self.Arch) in GlobalData.gModuleMakeCacheStatus:
- return GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)]
-
- # If Module is binary, which has special build rule, do not skip by cache.
- if self.IsBinaryModule:
- print("[cache miss]: MakeCache: Skip BinaryModule:", self.MetaFile.Path, self.Arch)
- GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False
- return False
-
- # see .inc as binary file, do not skip by hash
- for f_ext in self.SourceFileList:
- if '.inc' in str(f_ext):
- print("[cache miss]: MakeCache: Skip '.inc' File:", self.MetaFile.Path, self.Arch)
- GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False
- return False
-
- ModuleCacheDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName)
- FfsDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name)
-
- ModuleHashPairList = [] # tuple list: [tuple(PreMakefileHash, MakeHash)]
- ModuleHashPair = path.join(ModuleCacheDir, self.Name + ".ModuleHashPair")
- try:
- with open(LongFilePath(ModuleHashPair), 'r') as f:
- ModuleHashPairList = json.load(f)
- except:
- # ModuleHashPair might not exist for new added module
- GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False
- EdkLogger.quiet("[cache warning]: fail to load ModuleHashPair file: %s" % ModuleHashPair)
- print("[cache miss]: MakeCache:", self.MetaFile.Path, self.Arch)
- return False
-
- # Check the PreMakeHash in ModuleHashPairList one by one
- for idx, (PreMakefileHash, MakeHash) in enumerate (ModuleHashPairList):
- SourceHashDir = path.join(ModuleCacheDir, MakeHash)
- SourceFfsHashDir = path.join(FfsDir, MakeHash)
- PreMakeHashFileList_FilePah = path.join(ModuleCacheDir, self.Name + ".PreMakeHashFileList." + PreMakefileHash)
- MakeHashFileList_FilePah = path.join(ModuleCacheDir, self.Name + ".MakeHashFileList." + MakeHash)
-
- try:
- with open(LongFilePath(MakeHashFileList_FilePah), 'r') as f:
- MakeHashFileList = json.load(f)
- except:
- EdkLogger.quiet("[cache error]: fail to load MakeHashFileList file: %s" % MakeHashFileList_FilePah)
- continue
-
- HashMiss = False
- for HashChainFile in MakeHashFileList:
- HashChainStatus = None
- if HashChainFile in GlobalData.gHashChainStatus:
- HashChainStatus = GlobalData.gHashChainStatus[HashChainFile]
- if HashChainStatus == False:
- HashMiss = True
- break
- elif HashChainStatus == True:
- continue
- # Convert to path start with cache source dir
- RelativePath = os.path.relpath(HashChainFile, self.WorkspaceDir)
- NewFilePath = os.path.join(GlobalData.gBinCacheSource, RelativePath)
- if self.CheckHashChainFile(NewFilePath):
- GlobalData.gHashChainStatus[HashChainFile] = True
- # Save the module self HashFile for GenPreMakefileHashList later usage
- if self.Name + ".hashchain." in HashChainFile:
- GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)] = HashChainFile
- else:
- GlobalData.gHashChainStatus[HashChainFile] = False
- HashMiss = True
- break
-
- if HashMiss:
- continue
-
- # PreMakefile cache hit, restore the module build result
- for root, dir, files in os.walk(SourceHashDir):
- for f in files:
- File = path.join(root, f)
- self.CacheCopyFile(self.BuildDir, SourceHashDir, File)
- if os.path.exists(SourceFfsHashDir):
- for root, dir, files in os.walk(SourceFfsHashDir):
- for f in files:
- File = path.join(root, f)
- self.CacheCopyFile(self.FfsOutputDir, SourceFfsHashDir, File)
-
- if self.Name == "PcdPeim" or self.Name == "PcdDxe":
- CreatePcdDatabaseCode(self, TemplateString(), TemplateString())
-
- print("[cache hit]: MakeCache:", self.MetaFile.Path, self.Arch)
- GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = True
- return True
-
- print("[cache miss]: MakeCache:", self.MetaFile.Path, self.Arch)
- GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False
- return False
-
- ## Decide whether we can skip the left autogen and make process
- def CanSkipbyPreMakeCache(self):
- # CanSkipbyPreMakeCache consume below dicts:
- # gModulePreMakeCacheStatus
- # gHashChainStatus
- # gModuleHashFile
- # GenPreMakefileHashList produce gModulePreMakeCacheStatus dict.
- # all these dicts might be produced in multiprocessing, so
- # need check these remote dicts
-
- if not GlobalData.gUseHashCache or GlobalData.gBinCacheDest:
- return False
-
- if (self.MetaFile.Path, self.Arch) in GlobalData.gModulePreMakeCacheStatus:
- return GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)]
-
- # If Module is binary, which has special build rule, do not skip by cache.
- if self.IsBinaryModule:
- print("[cache miss]: PreMakeCache: Skip BinaryModule:", self.MetaFile.Path, self.Arch)
- GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False
- return False
-
- # see .inc as binary file, do not skip by hash
- for f_ext in self.SourceFileList:
- if '.inc' in str(f_ext):
- print("[cache miss]: PreMakeCache: Skip '.inc' File:", self.MetaFile.Path, self.Arch)
- GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False
- return False
-
- # For --hash only in the incremental build
- if not GlobalData.gBinCacheSource:
- Files = [path.join(self.BuildDir, f) for f in os.listdir(self.BuildDir) if path.isfile(path.join(self.BuildDir, f))]
- PreMakeHashFileList_FilePah = None
- MakeTimeStamp = 0
- # Find latest PreMakeHashFileList file in self.BuildDir folder
- for File in Files:
- if ".PreMakeHashFileList." in File:
- FileTimeStamp = os.stat(path.join(self.BuildDir, File))[8]
- if FileTimeStamp > MakeTimeStamp:
- MakeTimeStamp = FileTimeStamp
- PreMakeHashFileList_FilePah = File
- if not PreMakeHashFileList_FilePah:
- GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False
- return False
-
- try:
- with open(LongFilePath(PreMakeHashFileList_FilePah), 'r') as f:
- PreMakeHashFileList = json.load(f)
- except:
- EdkLogger.quiet("[cache error]: fail to load PreMakeHashFileList file: %s" % PreMakeHashFileList_FilePah)
- print("[cache miss]: PreMakeCache:", self.MetaFile.Path, self.Arch)
- GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False
- return False
-
- HashMiss = False
- for HashChainFile in PreMakeHashFileList:
- HashChainStatus = None
- if HashChainFile in GlobalData.gHashChainStatus:
- HashChainStatus = GlobalData.gHashChainStatus[HashChainFile]
- if HashChainStatus == False:
- HashMiss = True
- break
- elif HashChainStatus == True:
- continue
- if self.CheckHashChainFile(HashChainFile):
- GlobalData.gHashChainStatus[HashChainFile] = True
- # Save the module self HashFile for GenPreMakefileHashList later usage
- if self.Name + ".hashchain." in HashChainFile:
- GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)] = HashChainFile
- else:
- GlobalData.gHashChainStatus[HashChainFile] = False
- HashMiss = True
- break
-
- if HashMiss:
- print("[cache miss]: PreMakeCache:", self.MetaFile.Path, self.Arch)
- GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False
- return False
- else:
- print("[cache hit]: PreMakeCache:", self.MetaFile.Path, self.Arch)
- GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = True
- return True
-
- ModuleCacheDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName)
- FfsDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name)
-
- ModuleHashPairList = [] # tuple list: [tuple(PreMakefileHash, MakeHash)]
- ModuleHashPair = path.join(ModuleCacheDir, self.Name + ".ModuleHashPair")
- try:
- with open(LongFilePath(ModuleHashPair), 'r') as f:
- ModuleHashPairList = json.load(f)
- except:
- # ModuleHashPair might not exist for new added module
- GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False
- EdkLogger.quiet("[cache warning]: fail to load ModuleHashPair file: %s" % ModuleHashPair)
- print("[cache miss]: PreMakeCache:", self.MetaFile.Path, self.Arch)
- return False
-
- # Check the PreMakeHash in ModuleHashPairList one by one
- for idx, (PreMakefileHash, MakeHash) in enumerate (ModuleHashPairList):
- SourceHashDir = path.join(ModuleCacheDir, MakeHash)
- SourceFfsHashDir = path.join(FfsDir, MakeHash)
- PreMakeHashFileList_FilePah = path.join(ModuleCacheDir, self.Name + ".PreMakeHashFileList." + PreMakefileHash)
- MakeHashFileList_FilePah = path.join(ModuleCacheDir, self.Name + ".MakeHashFileList." + MakeHash)
-
- try:
- with open(LongFilePath(PreMakeHashFileList_FilePah), 'r') as f:
- PreMakeHashFileList = json.load(f)
- except:
- EdkLogger.quiet("[cache error]: fail to load PreMakeHashFileList file: %s" % PreMakeHashFileList_FilePah)
- continue
-
- HashMiss = False
- for HashChainFile in PreMakeHashFileList:
- HashChainStatus = None
- if HashChainFile in GlobalData.gHashChainStatus:
- HashChainStatus = GlobalData.gHashChainStatus[HashChainFile]
- if HashChainStatus == False:
- HashMiss = True
- break
- elif HashChainStatus == True:
- continue
- # Convert to path start with cache source dir
- RelativePath = os.path.relpath(HashChainFile, self.WorkspaceDir)
- NewFilePath = os.path.join(GlobalData.gBinCacheSource, RelativePath)
- if self.CheckHashChainFile(NewFilePath):
- GlobalData.gHashChainStatus[HashChainFile] = True
- else:
- GlobalData.gHashChainStatus[HashChainFile] = False
- HashMiss = True
- break
-
- if HashMiss:
- continue
-
- # PreMakefile cache hit, restore the module build result
- for root, dir, files in os.walk(SourceHashDir):
- for f in files:
- File = path.join(root, f)
- self.CacheCopyFile(self.BuildDir, SourceHashDir, File)
- if os.path.exists(SourceFfsHashDir):
- for root, dir, files in os.walk(SourceFfsHashDir):
- for f in files:
- File = path.join(root, f)
- self.CacheCopyFile(self.FfsOutputDir, SourceFfsHashDir, File)
-
- if self.Name == "PcdPeim" or self.Name == "PcdDxe":
- CreatePcdDatabaseCode(self, TemplateString(), TemplateString())
-
- print("[cache hit]: PreMakeCache:", self.MetaFile.Path, self.Arch)
- GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = True
- return True
-
- print("[cache miss]: PreMakeCache:", self.MetaFile.Path, self.Arch)
- GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False
- return False
-
- ## Decide whether we can skip the Module build
- def CanSkipbyCache(self, gHitSet):
- # Hashing feature is off
- if not GlobalData.gBinCacheSource:
- return False
-
- if self in gHitSet:
- return True
-
- return False
-
- ## Decide whether we can skip the ModuleAutoGen process
- # If any source file is newer than the module than we cannot skip
- #
- def CanSkip(self):
- # Don't skip if cache feature enabled
- if GlobalData.gUseHashCache or GlobalData.gBinCacheDest or GlobalData.gBinCacheSource:
- return False
- if self.MakeFileDir in GlobalData.gSikpAutoGenCache:
- return True
- if not os.path.exists(self.TimeStampPath):
- return False
- #last creation time of the module
- DstTimeStamp = os.stat(self.TimeStampPath)[8]
-
- SrcTimeStamp = self.Workspace._SrcTimeStamp
- if SrcTimeStamp > DstTimeStamp:
- return False
-
- with open(self.TimeStampPath,'r') as f:
- for source in f:
- source = source.rstrip('\n')
- if not os.path.exists(source):
- return False
- if source not in ModuleAutoGen.TimeDict :
- ModuleAutoGen.TimeDict[source] = os.stat(source)[8]
- if ModuleAutoGen.TimeDict[source] > DstTimeStamp:
- return False
- GlobalData.gSikpAutoGenCache.add(self.MakeFileDir)
- return True
-
- @cached_property
- def TimeStampPath(self):
- return os.path.join(self.MakeFileDir, 'AutoGenTimeStamp')
+## @file +# Create makefile for MS nmake and GNU make +# +# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR> +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +from __future__ import absolute_import +from AutoGen.AutoGen import AutoGen +from Common.LongFilePathSupport import LongFilePath, CopyLongFilePath +from Common.BuildToolError import * +from Common.DataType import * +from Common.Misc import * +from Common.StringUtils import NormPath,GetSplitList +from collections import defaultdict +from Workspace.WorkspaceCommon import OrderedListDict +import os.path as path +import copy +import hashlib +from . import InfSectionParser +from . import GenC +from . import GenMake +from . import GenDepex +from io import BytesIO +from GenPatchPcdTable.GenPatchPcdTable import parsePcdInfoFromMapFile +from Workspace.MetaFileCommentParser import UsageList +from .GenPcdDb import CreatePcdDatabaseCode +from Common.caching import cached_class_function +from AutoGen.ModuleAutoGenHelper import PlatformInfo,WorkSpaceInfo +import json +import tempfile + +## Mapping Makefile type +gMakeTypeMap = {TAB_COMPILER_MSFT:"nmake", "GCC":"gmake"} +# +# Regular expression for finding Include Directories, the difference between MSFT and INTEL/GCC +# is the former use /I , the Latter used -I to specify include directories +# +gBuildOptIncludePatternMsft = re.compile(r"(?:.*?)/I[ \t]*([^ ]*)", re.MULTILINE | re.DOTALL) +gBuildOptIncludePatternOther = re.compile(r"(?:.*?)-I[ \t]*([^ ]*)", re.MULTILINE | re.DOTALL) + +## default file name for AutoGen +gAutoGenCodeFileName = "AutoGen.c" +gAutoGenHeaderFileName = "AutoGen.h" +gAutoGenStringFileName = "%(module_name)sStrDefs.h" +gAutoGenStringFormFileName = "%(module_name)sStrDefs.hpk" +gAutoGenDepexFileName = "%(module_name)s.depex" +gAutoGenImageDefFileName = "%(module_name)sImgDefs.h" +gAutoGenIdfFileName = "%(module_name)sIdf.hpk" +gInfSpecVersion = "0x00010017" + +# +# Match name = variable +# +gEfiVarStoreNamePattern = re.compile(r"\s*name\s*=\s*(\w+)") +# +# The format of guid in efivarstore statement likes following and must be correct: +# guid = {0xA04A27f4, 0xDF00, 0x4D42, {0xB5, 0x52, 0x39, 0x51, 0x13, 0x02, 0x11, 0x3D}} +# +gEfiVarStoreGuidPattern = re.compile(r"\s*guid\s*=\s*({.*?{.*?}\s*})") + +# +# Template string to generic AsBuilt INF +# +gAsBuiltInfHeaderString = TemplateString("""${header_comments} + +# DO NOT EDIT +# FILE auto-generated + +[Defines] + INF_VERSION = ${module_inf_version} + BASE_NAME = ${module_name} + FILE_GUID = ${module_guid} + MODULE_TYPE = ${module_module_type}${BEGIN} + VERSION_STRING = ${module_version_string}${END}${BEGIN} + PCD_IS_DRIVER = ${pcd_is_driver_string}${END}${BEGIN} + UEFI_SPECIFICATION_VERSION = ${module_uefi_specification_version}${END}${BEGIN} + PI_SPECIFICATION_VERSION = ${module_pi_specification_version}${END}${BEGIN} + ENTRY_POINT = ${module_entry_point}${END}${BEGIN} + UNLOAD_IMAGE = ${module_unload_image}${END}${BEGIN} + CONSTRUCTOR = ${module_constructor}${END}${BEGIN} + DESTRUCTOR = ${module_destructor}${END}${BEGIN} + SHADOW = ${module_shadow}${END}${BEGIN} + PCI_VENDOR_ID = ${module_pci_vendor_id}${END}${BEGIN} + PCI_DEVICE_ID = ${module_pci_device_id}${END}${BEGIN} + PCI_CLASS_CODE = ${module_pci_class_code}${END}${BEGIN} + PCI_REVISION = ${module_pci_revision}${END}${BEGIN} + BUILD_NUMBER = ${module_build_number}${END}${BEGIN} + SPEC = ${module_spec}${END}${BEGIN} + UEFI_HII_RESOURCE_SECTION = ${module_uefi_hii_resource_section}${END}${BEGIN} + MODULE_UNI_FILE = ${module_uni_file}${END} + +[Packages.${module_arch}]${BEGIN} + ${package_item}${END} + +[Binaries.${module_arch}]${BEGIN} + ${binary_item}${END} + +[PatchPcd.${module_arch}]${BEGIN} + ${patchablepcd_item} +${END} + +[Protocols.${module_arch}]${BEGIN} + ${protocol_item} +${END} + +[Ppis.${module_arch}]${BEGIN} + ${ppi_item} +${END} + +[Guids.${module_arch}]${BEGIN} + ${guid_item} +${END} + +[PcdEx.${module_arch}]${BEGIN} + ${pcd_item} +${END} + +[LibraryClasses.${module_arch}] +## @LIB_INSTANCES${BEGIN} +# ${libraryclasses_item}${END} + +${depexsection_item} + +${userextension_tianocore_item} + +${tail_comments} + +[BuildOptions.${module_arch}] +## @AsBuilt${BEGIN} +## ${flags_item}${END} +""") +# +# extend lists contained in a dictionary with lists stored in another dictionary +# if CopyToDict is not derived from DefaultDict(list) then this may raise exception +# +def ExtendCopyDictionaryLists(CopyToDict, CopyFromDict): + for Key in CopyFromDict: + CopyToDict[Key].extend(CopyFromDict[Key]) + +# Create a directory specified by a set of path elements and return the full path +def _MakeDir(PathList): + RetVal = path.join(*PathList) + CreateDirectory(RetVal) + return RetVal + +# +# Convert string to C format array +# +def _ConvertStringToByteArray(Value): + Value = Value.strip() + if not Value: + return None + if Value[0] == '{': + if not Value.endswith('}'): + return None + Value = Value.replace(' ', '').replace('{', '').replace('}', '') + ValFields = Value.split(',') + try: + for Index in range(len(ValFields)): + ValFields[Index] = str(int(ValFields[Index], 0)) + except ValueError: + return None + Value = '{' + ','.join(ValFields) + '}' + return Value + + Unicode = False + if Value.startswith('L"'): + if not Value.endswith('"'): + return None + Value = Value[1:] + Unicode = True + elif not Value.startswith('"') or not Value.endswith('"'): + return None + + Value = eval(Value) # translate escape character + NewValue = '{' + for Index in range(0, len(Value)): + if Unicode: + NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ',' + else: + NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ',' + Value = NewValue + '0}' + return Value + +## ModuleAutoGen class +# +# This class encapsules the AutoGen behaviors for the build tools. In addition to +# the generation of AutoGen.h and AutoGen.c, it will generate *.depex file according +# to the [depex] section in module's inf file. +# +class ModuleAutoGen(AutoGen): + # call super().__init__ then call the worker function with different parameter count + def __init__(self, Workspace, MetaFile, Target, Toolchain, Arch, *args, **kwargs): + if not hasattr(self, "_Init"): + self._InitWorker(Workspace, MetaFile, Target, Toolchain, Arch, *args) + self._Init = True + + ## Cache the timestamps of metafiles of every module in a class attribute + # + TimeDict = {} + + def __new__(cls, Workspace, MetaFile, Target, Toolchain, Arch, *args, **kwargs): +# check if this module is employed by active platform + if not PlatformInfo(Workspace, args[0], Target, Toolchain, Arch,args[-1]).ValidModule(MetaFile): + EdkLogger.verbose("Module [%s] for [%s] is not employed by active platform\n" \ + % (MetaFile, Arch)) + return None + return super(ModuleAutoGen, cls).__new__(cls, Workspace, MetaFile, Target, Toolchain, Arch, *args, **kwargs) + + ## Initialize ModuleAutoGen + # + # @param Workspace EdkIIWorkspaceBuild object + # @param ModuleFile The path of module file + # @param Target Build target (DEBUG, RELEASE) + # @param Toolchain Name of tool chain + # @param Arch The arch the module supports + # @param PlatformFile Platform meta-file + # + def _InitWorker(self, Workspace, ModuleFile, Target, Toolchain, Arch, PlatformFile,DataPipe): + EdkLogger.debug(EdkLogger.DEBUG_9, "AutoGen module [%s] [%s]" % (ModuleFile, Arch)) + GlobalData.gProcessingFile = "%s [%s, %s, %s]" % (ModuleFile, Arch, Toolchain, Target) + + self.Workspace = Workspace + self.WorkspaceDir = "" + self.PlatformInfo = None + self.DataPipe = DataPipe + self.__init_platform_info__() + self.MetaFile = ModuleFile + self.SourceDir = self.MetaFile.SubDir + self.SourceDir = mws.relpath(self.SourceDir, self.WorkspaceDir) + + self.ToolChain = Toolchain + self.BuildTarget = Target + self.Arch = Arch + self.ToolChainFamily = self.PlatformInfo.ToolChainFamily + self.BuildRuleFamily = self.PlatformInfo.BuildRuleFamily + + self.IsCodeFileCreated = False + self.IsAsBuiltInfCreated = False + self.DepexGenerated = False + + self.BuildDatabase = self.Workspace.BuildDatabase + self.BuildRuleOrder = None + self.BuildTime = 0 + + self._GuidComments = OrderedListDict() + self._ProtocolComments = OrderedListDict() + self._PpiComments = OrderedListDict() + self._BuildTargets = None + self._IntroBuildTargetList = None + self._FinalBuildTargetList = None + self._FileTypes = None + + self.AutoGenDepSet = set() + self.ReferenceModules = [] + self.ConstPcd = {} + self.FileDependCache = {} + + def __init_platform_info__(self): + pinfo = self.DataPipe.Get("P_Info") + self.WorkspaceDir = pinfo.get("WorkspaceDir") + self.PlatformInfo = PlatformInfo(self.Workspace,pinfo.get("ActivePlatform"),pinfo.get("Target"),pinfo.get("ToolChain"),pinfo.get("Arch"),self.DataPipe) + ## hash() operator of ModuleAutoGen + # + # The module file path and arch string will be used to represent + # hash value of this object + # + # @retval int Hash value of the module file path and arch + # + @cached_class_function + def __hash__(self): + return hash((self.MetaFile, self.Arch, self.ToolChain,self.BuildTarget)) + def __repr__(self): + return "%s [%s]" % (self.MetaFile, self.Arch) + + # Get FixedAtBuild Pcds of this Module + @cached_property + def FixedAtBuildPcds(self): + RetVal = [] + for Pcd in self.ModulePcdList: + if Pcd.Type != TAB_PCDS_FIXED_AT_BUILD: + continue + if Pcd not in RetVal: + RetVal.append(Pcd) + return RetVal + + @cached_property + def FixedVoidTypePcds(self): + RetVal = {} + for Pcd in self.FixedAtBuildPcds: + if Pcd.DatumType == TAB_VOID: + if '.'.join((Pcd.TokenSpaceGuidCName, Pcd.TokenCName)) not in RetVal: + RetVal['.'.join((Pcd.TokenSpaceGuidCName, Pcd.TokenCName))] = Pcd.DefaultValue + return RetVal + + @property + def UniqueBaseName(self): + ModuleNames = self.DataPipe.Get("M_Name") + if not ModuleNames: + return self.Name + return ModuleNames.get((self.Name,self.MetaFile),self.Name) + + # Macros could be used in build_rule.txt (also Makefile) + @cached_property + def Macros(self): + return OrderedDict(( + ("WORKSPACE" ,self.WorkspaceDir), + ("MODULE_NAME" ,self.Name), + ("MODULE_NAME_GUID" ,self.UniqueBaseName), + ("MODULE_GUID" ,self.Guid), + ("MODULE_VERSION" ,self.Version), + ("MODULE_TYPE" ,self.ModuleType), + ("MODULE_FILE" ,str(self.MetaFile)), + ("MODULE_FILE_BASE_NAME" ,self.MetaFile.BaseName), + ("MODULE_RELATIVE_DIR" ,self.SourceDir), + ("MODULE_DIR" ,self.SourceDir), + ("BASE_NAME" ,self.Name), + ("ARCH" ,self.Arch), + ("TOOLCHAIN" ,self.ToolChain), + ("TOOLCHAIN_TAG" ,self.ToolChain), + ("TOOL_CHAIN_TAG" ,self.ToolChain), + ("TARGET" ,self.BuildTarget), + ("BUILD_DIR" ,self.PlatformInfo.BuildDir), + ("BIN_DIR" ,os.path.join(self.PlatformInfo.BuildDir, self.Arch)), + ("LIB_DIR" ,os.path.join(self.PlatformInfo.BuildDir, self.Arch)), + ("MODULE_BUILD_DIR" ,self.BuildDir), + ("OUTPUT_DIR" ,self.OutputDir), + ("DEBUG_DIR" ,self.DebugDir), + ("DEST_DIR_OUTPUT" ,self.OutputDir), + ("DEST_DIR_DEBUG" ,self.DebugDir), + ("PLATFORM_NAME" ,self.PlatformInfo.Name), + ("PLATFORM_GUID" ,self.PlatformInfo.Guid), + ("PLATFORM_VERSION" ,self.PlatformInfo.Version), + ("PLATFORM_RELATIVE_DIR" ,self.PlatformInfo.SourceDir), + ("PLATFORM_DIR" ,mws.join(self.WorkspaceDir, self.PlatformInfo.SourceDir)), + ("PLATFORM_OUTPUT_DIR" ,self.PlatformInfo.OutputDir), + ("FFS_OUTPUT_DIR" ,self.FfsOutputDir) + )) + + ## Return the module build data object + @cached_property + def Module(self): + return self.BuildDatabase[self.MetaFile, self.Arch, self.BuildTarget, self.ToolChain] + + ## Return the module name + @cached_property + def Name(self): + return self.Module.BaseName + + ## Return the module DxsFile if exist + @cached_property + def DxsFile(self): + return self.Module.DxsFile + + ## Return the module meta-file GUID + @cached_property + def Guid(self): + # + # To build same module more than once, the module path with FILE_GUID overridden has + # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path + # in DSC. The overridden GUID can be retrieved from file name + # + if os.path.basename(self.MetaFile.File) != os.path.basename(self.MetaFile.Path): + # + # Length of GUID is 36 + # + return os.path.basename(self.MetaFile.Path)[:36] + return self.Module.Guid + + ## Return the module version + @cached_property + def Version(self): + return self.Module.Version + + ## Return the module type + @cached_property + def ModuleType(self): + return self.Module.ModuleType + + ## Return the component type (for Edk.x style of module) + @cached_property + def ComponentType(self): + return self.Module.ComponentType + + ## Return the build type + @cached_property + def BuildType(self): + return self.Module.BuildType + + ## Return the PCD_IS_DRIVER setting + @cached_property + def PcdIsDriver(self): + return self.Module.PcdIsDriver + + ## Return the autogen version, i.e. module meta-file version + @cached_property + def AutoGenVersion(self): + return self.Module.AutoGenVersion + + ## Check if the module is library or not + @cached_property + def IsLibrary(self): + return bool(self.Module.LibraryClass) + + ## Check if the module is binary module or not + @cached_property + def IsBinaryModule(self): + return self.Module.IsBinaryModule + + ## Return the directory to store intermediate files of the module + @cached_property + def BuildDir(self): + return _MakeDir(( + self.PlatformInfo.BuildDir, + self.Arch, + self.SourceDir, + self.MetaFile.BaseName + )) + + ## Return the directory to store the intermediate object files of the module + @cached_property + def OutputDir(self): + return _MakeDir((self.BuildDir, "OUTPUT")) + + ## Return the directory path to store ffs file + @cached_property + def FfsOutputDir(self): + if GlobalData.gFdfParser: + return path.join(self.PlatformInfo.BuildDir, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name) + return '' + + ## Return the directory to store auto-gened source files of the module + @cached_property + def DebugDir(self): + return _MakeDir((self.BuildDir, "DEBUG")) + + ## Return the path of custom file + @cached_property + def CustomMakefile(self): + RetVal = {} + for Type in self.Module.CustomMakefile: + MakeType = gMakeTypeMap[Type] if Type in gMakeTypeMap else 'nmake' + File = os.path.join(self.SourceDir, self.Module.CustomMakefile[Type]) + RetVal[MakeType] = File + return RetVal + + ## Return the directory of the makefile + # + # @retval string The directory string of module's makefile + # + @cached_property + def MakeFileDir(self): + return self.BuildDir + + ## Return build command string + # + # @retval string Build command string + # + @cached_property + def BuildCommand(self): + return self.PlatformInfo.BuildCommand + + ## Get Module package and Platform package + # + # @retval list The list of package object + # + @cached_property + def PackageList(self): + PkagList = [] + if self.Module.Packages: + PkagList.extend(self.Module.Packages) + Platform = self.BuildDatabase[self.PlatformInfo.MetaFile, self.Arch, self.BuildTarget, self.ToolChain] + for Package in Platform.Packages: + if Package in PkagList: + continue + PkagList.append(Package) + return PkagList + + ## Get object list of all packages the module and its dependent libraries belong to and the Platform depends on + # + # @retval list The list of package object + # + @cached_property + def DerivedPackageList(self): + PackageList = [] + PackageList.extend(self.PackageList) + for M in self.DependentLibraryList: + for Package in M.Packages: + if Package in PackageList: + continue + PackageList.append(Package) + return PackageList + + ## Get the depex string + # + # @return : a string contain all depex expression. + def _GetDepexExpresionString(self): + DepexStr = '' + DepexList = [] + ## DPX_SOURCE IN Define section. + if self.Module.DxsFile: + return DepexStr + for M in [self.Module] + self.DependentLibraryList: + Filename = M.MetaFile.Path + InfObj = InfSectionParser.InfSectionParser(Filename) + DepexExpressionList = InfObj.GetDepexExpresionList() + for DepexExpression in DepexExpressionList: + for key in DepexExpression: + Arch, ModuleType = key + DepexExpr = [x for x in DepexExpression[key] if not str(x).startswith('#')] + # the type of build module is USER_DEFINED. + # All different DEPEX section tags would be copied into the As Built INF file + # and there would be separate DEPEX section tags + if self.ModuleType.upper() == SUP_MODULE_USER_DEFINED or self.ModuleType.upper() == SUP_MODULE_HOST_APPLICATION: + if (Arch.upper() == self.Arch.upper()) and (ModuleType.upper() != TAB_ARCH_COMMON): + DepexList.append({(Arch, ModuleType): DepexExpr}) + else: + if Arch.upper() == TAB_ARCH_COMMON or \ + (Arch.upper() == self.Arch.upper() and \ + ModuleType.upper() in [TAB_ARCH_COMMON, self.ModuleType.upper()]): + DepexList.append({(Arch, ModuleType): DepexExpr}) + + #the type of build module is USER_DEFINED. + if self.ModuleType.upper() == SUP_MODULE_USER_DEFINED or self.ModuleType.upper() == SUP_MODULE_HOST_APPLICATION: + for Depex in DepexList: + for key in Depex: + DepexStr += '[Depex.%s.%s]\n' % key + DepexStr += '\n'.join('# '+ val for val in Depex[key]) + DepexStr += '\n\n' + if not DepexStr: + return '[Depex.%s]\n' % self.Arch + return DepexStr + + #the type of build module not is USER_DEFINED. + Count = 0 + for Depex in DepexList: + Count += 1 + if DepexStr != '': + DepexStr += ' AND ' + DepexStr += '(' + for D in Depex.values(): + DepexStr += ' '.join(val for val in D) + Index = DepexStr.find('END') + if Index > -1 and Index == len(DepexStr) - 3: + DepexStr = DepexStr[:-3] + DepexStr = DepexStr.strip() + DepexStr += ')' + if Count == 1: + DepexStr = DepexStr.lstrip('(').rstrip(')').strip() + if not DepexStr: + return '[Depex.%s]\n' % self.Arch + return '[Depex.%s]\n# ' % self.Arch + DepexStr + + ## Merge dependency expression + # + # @retval list The token list of the dependency expression after parsed + # + @cached_property + def DepexList(self): + if self.DxsFile or self.IsLibrary or TAB_DEPENDENCY_EXPRESSION_FILE in self.FileTypes: + return {} + + DepexList = [] + # + # Append depex from dependent libraries, if not "BEFORE", "AFTER" expression + # + FixedVoidTypePcds = {} + for M in [self] + self.LibraryAutoGenList: + FixedVoidTypePcds.update(M.FixedVoidTypePcds) + for M in [self] + self.LibraryAutoGenList: + Inherited = False + for D in M.Module.Depex[self.Arch, self.ModuleType]: + if DepexList != []: + DepexList.append('AND') + DepexList.append('(') + #replace D with value if D is FixedAtBuild PCD + NewList = [] + for item in D: + if '.' not in item: + NewList.append(item) + else: + try: + Value = FixedVoidTypePcds[item] + if len(Value.split(',')) != 16: + EdkLogger.error("build", FORMAT_INVALID, + "{} used in [Depex] section should be used as FixedAtBuild type and VOID* datum type and 16 bytes in the module.".format(item)) + NewList.append(Value) + except: + EdkLogger.error("build", FORMAT_INVALID, "{} used in [Depex] section should be used as FixedAtBuild type and VOID* datum type in the module.".format(item)) + + DepexList.extend(NewList) + if DepexList[-1] == 'END': # no need of a END at this time + DepexList.pop() + DepexList.append(')') + Inherited = True + if Inherited: + EdkLogger.verbose("DEPEX[%s] (+%s) = %s" % (self.Name, M.Module.BaseName, DepexList)) + if 'BEFORE' in DepexList or 'AFTER' in DepexList: + break + if len(DepexList) > 0: + EdkLogger.verbose('') + return {self.ModuleType:DepexList} + + ## Merge dependency expression + # + # @retval list The token list of the dependency expression after parsed + # + @cached_property + def DepexExpressionDict(self): + if self.DxsFile or self.IsLibrary or TAB_DEPENDENCY_EXPRESSION_FILE in self.FileTypes: + return {} + + DepexExpressionString = '' + # + # Append depex from dependent libraries, if not "BEFORE", "AFTER" expresion + # + for M in [self.Module] + self.DependentLibraryList: + Inherited = False + for D in M.DepexExpression[self.Arch, self.ModuleType]: + if DepexExpressionString != '': + DepexExpressionString += ' AND ' + DepexExpressionString += '(' + DepexExpressionString += D + DepexExpressionString = DepexExpressionString.rstrip('END').strip() + DepexExpressionString += ')' + Inherited = True + if Inherited: + EdkLogger.verbose("DEPEX[%s] (+%s) = %s" % (self.Name, M.BaseName, DepexExpressionString)) + if 'BEFORE' in DepexExpressionString or 'AFTER' in DepexExpressionString: + break + if len(DepexExpressionString) > 0: + EdkLogger.verbose('') + + return {self.ModuleType:DepexExpressionString} + + # Get the tiano core user extension, it is contain dependent library. + # @retval: a list contain tiano core userextension. + # + def _GetTianoCoreUserExtensionList(self): + TianoCoreUserExtentionList = [] + for M in [self.Module] + self.DependentLibraryList: + Filename = M.MetaFile.Path + InfObj = InfSectionParser.InfSectionParser(Filename) + TianoCoreUserExtenList = InfObj.GetUserExtensionTianoCore() + for TianoCoreUserExtent in TianoCoreUserExtenList: + for Section in TianoCoreUserExtent: + ItemList = Section.split(TAB_SPLIT) + Arch = self.Arch + if len(ItemList) == 4: + Arch = ItemList[3] + if Arch.upper() == TAB_ARCH_COMMON or Arch.upper() == self.Arch.upper(): + TianoCoreList = [] + TianoCoreList.extend([TAB_SECTION_START + Section + TAB_SECTION_END]) + TianoCoreList.extend(TianoCoreUserExtent[Section][:]) + TianoCoreList.append('\n') + TianoCoreUserExtentionList.append(TianoCoreList) + + return TianoCoreUserExtentionList + + ## Return the list of specification version required for the module + # + # @retval list The list of specification defined in module file + # + @cached_property + def Specification(self): + return self.Module.Specification + + ## Tool option for the module build + # + # @param PlatformInfo The object of PlatformBuildInfo + # @retval dict The dict containing valid options + # + @cached_property + def BuildOption(self): + RetVal, self.BuildRuleOrder = self.PlatformInfo.ApplyBuildOption(self.Module) + if self.BuildRuleOrder: + self.BuildRuleOrder = ['.%s' % Ext for Ext in self.BuildRuleOrder.split()] + return RetVal + + ## Get include path list from tool option for the module build + # + # @retval list The include path list + # + @cached_property + def BuildOptionIncPathList(self): + # + # Regular expression for finding Include Directories, the difference between MSFT and INTEL/GCC + # is the former use /I , the Latter used -I to specify include directories + # + if self.PlatformInfo.ToolChainFamily in (TAB_COMPILER_MSFT): + BuildOptIncludeRegEx = gBuildOptIncludePatternMsft + elif self.PlatformInfo.ToolChainFamily in ('INTEL', 'GCC'): + BuildOptIncludeRegEx = gBuildOptIncludePatternOther + else: + # + # New ToolChainFamily, don't known whether there is option to specify include directories + # + return [] + + RetVal = [] + for Tool in ('CC', 'PP', 'VFRPP', 'ASLPP', 'ASLCC', 'APP', 'ASM'): + try: + FlagOption = self.BuildOption[Tool]['FLAGS'] + except KeyError: + FlagOption = '' + + IncPathList = [NormPath(Path, self.Macros) for Path in BuildOptIncludeRegEx.findall(FlagOption)] + + # + # EDK II modules must not reference header files outside of the packages they depend on or + # within the module's directory tree. Report error if violation. + # + if GlobalData.gDisableIncludePathCheck == False: + for Path in IncPathList: + if (Path not in self.IncludePathList) and (CommonPath([Path, self.MetaFile.Dir]) != self.MetaFile.Dir): + ErrMsg = "The include directory for the EDK II module in this line is invalid %s specified in %s FLAGS '%s'" % (Path, Tool, FlagOption) + EdkLogger.error("build", + PARAMETER_INVALID, + ExtraData=ErrMsg, + File=str(self.MetaFile)) + RetVal += IncPathList + return RetVal + + ## Return a list of files which can be built from source + # + # What kind of files can be built is determined by build rules in + # $(CONF_DIRECTORY)/build_rule.txt and toolchain family. + # + @cached_property + def SourceFileList(self): + RetVal = [] + ToolChainTagSet = {"", TAB_STAR, self.ToolChain} + ToolChainFamilySet = {"", TAB_STAR, self.ToolChainFamily, self.BuildRuleFamily} + for F in self.Module.Sources: + # match tool chain + if F.TagName not in ToolChainTagSet: + EdkLogger.debug(EdkLogger.DEBUG_9, "The toolchain [%s] for processing file [%s] is found, " + "but [%s] is currently used" % (F.TagName, str(F), self.ToolChain)) + continue + # match tool chain family or build rule family + if F.ToolChainFamily not in ToolChainFamilySet: + EdkLogger.debug( + EdkLogger.DEBUG_0, + "The file [%s] must be built by tools of [%s], " \ + "but current toolchain family is [%s], buildrule family is [%s]" \ + % (str(F), F.ToolChainFamily, self.ToolChainFamily, self.BuildRuleFamily)) + continue + + # add the file path into search path list for file including + if F.Dir not in self.IncludePathList: + self.IncludePathList.insert(0, F.Dir) + RetVal.append(F) + + self._MatchBuildRuleOrder(RetVal) + + for F in RetVal: + self._ApplyBuildRule(F, TAB_UNKNOWN_FILE) + return RetVal + + def _MatchBuildRuleOrder(self, FileList): + Order_Dict = {} + self.BuildOption + for SingleFile in FileList: + if self.BuildRuleOrder and SingleFile.Ext in self.BuildRuleOrder and SingleFile.Ext in self.BuildRules: + key = SingleFile.Path.rsplit(SingleFile.Ext,1)[0] + if key in Order_Dict: + Order_Dict[key].append(SingleFile.Ext) + else: + Order_Dict[key] = [SingleFile.Ext] + + RemoveList = [] + for F in Order_Dict: + if len(Order_Dict[F]) > 1: + Order_Dict[F].sort(key=lambda i: self.BuildRuleOrder.index(i)) + for Ext in Order_Dict[F][1:]: + RemoveList.append(F + Ext) + + for item in RemoveList: + FileList.remove(item) + + return FileList + + ## Return the list of unicode files + @cached_property + def UnicodeFileList(self): + return self.FileTypes.get(TAB_UNICODE_FILE,[]) + + ## Return the list of vfr files + @cached_property + def VfrFileList(self): + return self.FileTypes.get(TAB_VFR_FILE, []) + + ## Return the list of Image Definition files + @cached_property + def IdfFileList(self): + return self.FileTypes.get(TAB_IMAGE_FILE,[]) + + ## Return a list of files which can be built from binary + # + # "Build" binary files are just to copy them to build directory. + # + # @retval list The list of files which can be built later + # + @cached_property + def BinaryFileList(self): + RetVal = [] + for F in self.Module.Binaries: + if F.Target not in [TAB_ARCH_COMMON, TAB_STAR] and F.Target != self.BuildTarget: + continue + RetVal.append(F) + self._ApplyBuildRule(F, F.Type, BinaryFileList=RetVal) + return RetVal + + @cached_property + def BuildRules(self): + RetVal = {} + BuildRuleDatabase = self.PlatformInfo.BuildRule + for Type in BuildRuleDatabase.FileTypeList: + #first try getting build rule by BuildRuleFamily + RuleObject = BuildRuleDatabase[Type, self.BuildType, self.Arch, self.BuildRuleFamily] + if not RuleObject: + # build type is always module type, but ... + if self.ModuleType != self.BuildType: + RuleObject = BuildRuleDatabase[Type, self.ModuleType, self.Arch, self.BuildRuleFamily] + #second try getting build rule by ToolChainFamily + if not RuleObject: + RuleObject = BuildRuleDatabase[Type, self.BuildType, self.Arch, self.ToolChainFamily] + if not RuleObject: + # build type is always module type, but ... + if self.ModuleType != self.BuildType: + RuleObject = BuildRuleDatabase[Type, self.ModuleType, self.Arch, self.ToolChainFamily] + if not RuleObject: + continue + RuleObject = RuleObject.Instantiate(self.Macros) + RetVal[Type] = RuleObject + for Ext in RuleObject.SourceFileExtList: + RetVal[Ext] = RuleObject + return RetVal + + def _ApplyBuildRule(self, File, FileType, BinaryFileList=None): + if self._BuildTargets is None: + self._IntroBuildTargetList = set() + self._FinalBuildTargetList = set() + self._BuildTargets = defaultdict(set) + self._FileTypes = defaultdict(set) + + if not BinaryFileList: + BinaryFileList = self.BinaryFileList + + SubDirectory = os.path.join(self.OutputDir, File.SubDir) + if not os.path.exists(SubDirectory): + CreateDirectory(SubDirectory) + TargetList = set() + FinalTargetName = set() + RuleChain = set() + SourceList = [File] + Index = 0 + # + # Make sure to get build rule order value + # + self.BuildOption + + while Index < len(SourceList): + # Reset the FileType if not the first iteration. + if Index > 0: + FileType = TAB_UNKNOWN_FILE + Source = SourceList[Index] + Index = Index + 1 + + if Source != File: + CreateDirectory(Source.Dir) + + if File.IsBinary and File == Source and File in BinaryFileList: + # Skip all files that are not binary libraries + if not self.IsLibrary: + continue + RuleObject = self.BuildRules[TAB_DEFAULT_BINARY_FILE] + elif FileType in self.BuildRules: + RuleObject = self.BuildRules[FileType] + elif Source.Ext in self.BuildRules: + RuleObject = self.BuildRules[Source.Ext] + else: + # No more rule to apply: Source is a final target. + FinalTargetName.add(Source) + continue + + FileType = RuleObject.SourceFileType + self._FileTypes[FileType].add(Source) + + # stop at STATIC_LIBRARY for library + if self.IsLibrary and FileType == TAB_STATIC_LIBRARY: + FinalTargetName.add(Source) + continue + + Target = RuleObject.Apply(Source, self.BuildRuleOrder) + if not Target: + # No Target: Source is a final target. + FinalTargetName.add(Source) + continue + + TargetList.add(Target) + self._BuildTargets[FileType].add(Target) + + if not Source.IsBinary and Source == File: + self._IntroBuildTargetList.add(Target) + + # to avoid cyclic rule + if FileType in RuleChain: + EdkLogger.error("build", ERROR_STATEMENT, "Cyclic dependency detected while generating rule for %s" % str(Source)) + + RuleChain.add(FileType) + SourceList.extend(Target.Outputs) + + # For each final target name, retrieve the corresponding TargetDescBlock instance. + for FTargetName in FinalTargetName: + for Target in TargetList: + if FTargetName == Target.Target: + self._FinalBuildTargetList.add(Target) + + @cached_property + def Targets(self): + if self._BuildTargets is None: + self._IntroBuildTargetList = set() + self._FinalBuildTargetList = set() + self._BuildTargets = defaultdict(set) + self._FileTypes = defaultdict(set) + + #TRICK: call SourceFileList property to apply build rule for source files + self.SourceFileList + + #TRICK: call _GetBinaryFileList to apply build rule for binary files + self.BinaryFileList + + return self._BuildTargets + + @cached_property + def IntroTargetList(self): + self.Targets + return self._IntroBuildTargetList + + @cached_property + def CodaTargetList(self): + self.Targets + return self._FinalBuildTargetList + + @cached_property + def FileTypes(self): + self.Targets + return self._FileTypes + + ## Get the list of package object the module depends on and the Platform depends on + # + # @retval list The package object list + # + @cached_property + def DependentPackageList(self): + return self.PackageList + + ## Return the list of auto-generated code file + # + # @retval list The list of auto-generated file + # + @cached_property + def AutoGenFileList(self): + AutoGenUniIdf = self.BuildType != 'UEFI_HII' + UniStringBinBuffer = BytesIO() + IdfGenBinBuffer = BytesIO() + RetVal = {} + AutoGenC = TemplateString() + AutoGenH = TemplateString() + StringH = TemplateString() + StringIdf = TemplateString() + GenC.CreateCode(self, AutoGenC, AutoGenH, StringH, AutoGenUniIdf, UniStringBinBuffer, StringIdf, AutoGenUniIdf, IdfGenBinBuffer) + # + # AutoGen.c is generated if there are library classes in inf, or there are object files + # + if str(AutoGenC) != "" and (len(self.Module.LibraryClasses) > 0 + or TAB_OBJECT_FILE in self.FileTypes): + AutoFile = PathClass(gAutoGenCodeFileName, self.DebugDir) + RetVal[AutoFile] = str(AutoGenC) + self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE) + if str(AutoGenH) != "": + AutoFile = PathClass(gAutoGenHeaderFileName, self.DebugDir) + RetVal[AutoFile] = str(AutoGenH) + self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE) + if str(StringH) != "": + AutoFile = PathClass(gAutoGenStringFileName % {"module_name":self.Name}, self.DebugDir) + RetVal[AutoFile] = str(StringH) + self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE) + if UniStringBinBuffer is not None and UniStringBinBuffer.getvalue() != b"": + AutoFile = PathClass(gAutoGenStringFormFileName % {"module_name":self.Name}, self.OutputDir) + RetVal[AutoFile] = UniStringBinBuffer.getvalue() + AutoFile.IsBinary = True + self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE) + if UniStringBinBuffer is not None: + UniStringBinBuffer.close() + if str(StringIdf) != "": + AutoFile = PathClass(gAutoGenImageDefFileName % {"module_name":self.Name}, self.DebugDir) + RetVal[AutoFile] = str(StringIdf) + self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE) + if IdfGenBinBuffer is not None and IdfGenBinBuffer.getvalue() != b"": + AutoFile = PathClass(gAutoGenIdfFileName % {"module_name":self.Name}, self.OutputDir) + RetVal[AutoFile] = IdfGenBinBuffer.getvalue() + AutoFile.IsBinary = True + self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE) + if IdfGenBinBuffer is not None: + IdfGenBinBuffer.close() + return RetVal + + ## Return the list of library modules explicitly or implicitly used by this module + @cached_property + def DependentLibraryList(self): + # only merge library classes and PCD for non-library module + if self.IsLibrary: + return [] + return self.PlatformInfo.ApplyLibraryInstance(self.Module) + + ## Get the list of PCDs from current module + # + # @retval list The list of PCD + # + @cached_property + def ModulePcdList(self): + # apply PCD settings from platform + RetVal = self.PlatformInfo.ApplyPcdSetting(self, self.Module.Pcds) + + return RetVal + @cached_property + def _PcdComments(self): + ReVal = OrderedListDict() + ExtendCopyDictionaryLists(ReVal, self.Module.PcdComments) + if not self.IsLibrary: + for Library in self.DependentLibraryList: + ExtendCopyDictionaryLists(ReVal, Library.PcdComments) + return ReVal + + ## Get the list of PCDs from dependent libraries + # + # @retval list The list of PCD + # + @cached_property + def LibraryPcdList(self): + if self.IsLibrary: + return [] + RetVal = [] + Pcds = set() + # get PCDs from dependent libraries + for Library in self.DependentLibraryList: + PcdsInLibrary = OrderedDict() + for Key in Library.Pcds: + # skip duplicated PCDs + if Key in self.Module.Pcds or Key in Pcds: + continue + Pcds.add(Key) + PcdsInLibrary[Key] = copy.copy(Library.Pcds[Key]) + RetVal.extend(self.PlatformInfo.ApplyPcdSetting(self, PcdsInLibrary, Library=Library)) + return RetVal + + ## Get the GUID value mapping + # + # @retval dict The mapping between GUID cname and its value + # + @cached_property + def GuidList(self): + RetVal = self.Module.Guids + for Library in self.DependentLibraryList: + RetVal.update(Library.Guids) + ExtendCopyDictionaryLists(self._GuidComments, Library.GuidComments) + ExtendCopyDictionaryLists(self._GuidComments, self.Module.GuidComments) + return RetVal + + @cached_property + def GetGuidsUsedByPcd(self): + RetVal = OrderedDict(self.Module.GetGuidsUsedByPcd()) + for Library in self.DependentLibraryList: + RetVal.update(Library.GetGuidsUsedByPcd()) + return RetVal + ## Get the protocol value mapping + # + # @retval dict The mapping between protocol cname and its value + # + @cached_property + def ProtocolList(self): + RetVal = OrderedDict(self.Module.Protocols) + for Library in self.DependentLibraryList: + RetVal.update(Library.Protocols) + ExtendCopyDictionaryLists(self._ProtocolComments, Library.ProtocolComments) + ExtendCopyDictionaryLists(self._ProtocolComments, self.Module.ProtocolComments) + return RetVal + + ## Get the PPI value mapping + # + # @retval dict The mapping between PPI cname and its value + # + @cached_property + def PpiList(self): + RetVal = OrderedDict(self.Module.Ppis) + for Library in self.DependentLibraryList: + RetVal.update(Library.Ppis) + ExtendCopyDictionaryLists(self._PpiComments, Library.PpiComments) + ExtendCopyDictionaryLists(self._PpiComments, self.Module.PpiComments) + return RetVal + + ## Get the list of include search path + # + # @retval list The list path + # + @cached_property + def IncludePathList(self): + RetVal = [] + RetVal.append(self.MetaFile.Dir) + RetVal.append(self.DebugDir) + + for Package in self.PackageList: + PackageDir = mws.join(self.WorkspaceDir, Package.MetaFile.Dir) + if PackageDir not in RetVal: + RetVal.append(PackageDir) + IncludesList = Package.Includes + if Package._PrivateIncludes: + if not self.MetaFile.OriginalPath.Path.startswith(PackageDir): + IncludesList = list(set(Package.Includes).difference(set(Package._PrivateIncludes))) + for Inc in IncludesList: + if Inc not in RetVal: + RetVal.append(str(Inc)) + RetVal.extend(self.IncPathFromBuildOptions) + return RetVal + + @cached_property + def IncPathFromBuildOptions(self): + IncPathList = [] + for tool in self.BuildOption: + if 'FLAGS' in self.BuildOption[tool]: + flags = self.BuildOption[tool]['FLAGS'] + whitespace = False + for flag in flags.split(" "): + flag = flag.strip() + if flag.startswith(("/I","-I")): + if len(flag)>2: + if os.path.exists(flag[2:]): + IncPathList.append(flag[2:]) + else: + whitespace = True + continue + if whitespace and flag: + if os.path.exists(flag): + IncPathList.append(flag) + whitespace = False + return IncPathList + + @cached_property + def IncludePathLength(self): + return sum(len(inc)+1 for inc in self.IncludePathList) + + ## Get the list of include paths from the packages + # + # @IncludesList list The list path + # + @cached_property + def PackageIncludePathList(self): + IncludesList = [] + for Package in self.PackageList: + PackageDir = mws.join(self.WorkspaceDir, Package.MetaFile.Dir) + IncludesList = Package.Includes + if Package._PrivateIncludes: + if not self.MetaFile.Path.startswith(PackageDir): + IncludesList = list(set(Package.Includes).difference(set(Package._PrivateIncludes))) + return IncludesList + + ## Get HII EX PCDs which maybe used by VFR + # + # efivarstore used by VFR may relate with HII EX PCDs + # Get the variable name and GUID from efivarstore and HII EX PCD + # List the HII EX PCDs in As Built INF if both name and GUID match. + # + # @retval list HII EX PCDs + # + def _GetPcdsMaybeUsedByVfr(self): + if not self.SourceFileList: + return [] + + NameGuids = set() + for SrcFile in self.SourceFileList: + if SrcFile.Ext.lower() != '.vfr': + continue + Vfri = os.path.join(self.OutputDir, SrcFile.BaseName + '.i') + if not os.path.exists(Vfri): + continue + VfriFile = open(Vfri, 'r') + Content = VfriFile.read() + VfriFile.close() + Pos = Content.find('efivarstore') + while Pos != -1: + # + # Make sure 'efivarstore' is the start of efivarstore statement + # In case of the value of 'name' (name = efivarstore) is equal to 'efivarstore' + # + Index = Pos - 1 + while Index >= 0 and Content[Index] in ' \t\r\n': + Index -= 1 + if Index >= 0 and Content[Index] != ';': + Pos = Content.find('efivarstore', Pos + len('efivarstore')) + continue + # + # 'efivarstore' must be followed by name and guid + # + Name = gEfiVarStoreNamePattern.search(Content, Pos) + if not Name: + break + Guid = gEfiVarStoreGuidPattern.search(Content, Pos) + if not Guid: + break + NameArray = _ConvertStringToByteArray('L"' + Name.group(1) + '"') + NameGuids.add((NameArray, GuidStructureStringToGuidString(Guid.group(1)))) + Pos = Content.find('efivarstore', Name.end()) + if not NameGuids: + return [] + HiiExPcds = [] + for Pcd in self.PlatformInfo.Pcds.values(): + if Pcd.Type != TAB_PCDS_DYNAMIC_EX_HII: + continue + for SkuInfo in Pcd.SkuInfoList.values(): + Value = GuidValue(SkuInfo.VariableGuid, self.PlatformInfo.PackageList, self.MetaFile.Path) + if not Value: + continue + Name = _ConvertStringToByteArray(SkuInfo.VariableName) + Guid = GuidStructureStringToGuidString(Value) + if (Name, Guid) in NameGuids and Pcd not in HiiExPcds: + HiiExPcds.append(Pcd) + break + + return HiiExPcds + + def _GenOffsetBin(self): + VfrUniBaseName = {} + for SourceFile in self.Module.Sources: + if SourceFile.Type.upper() == ".VFR" : + # + # search the .map file to find the offset of vfr binary in the PE32+/TE file. + # + VfrUniBaseName[SourceFile.BaseName] = (SourceFile.BaseName + "Bin") + elif SourceFile.Type.upper() == ".UNI" : + # + # search the .map file to find the offset of Uni strings binary in the PE32+/TE file. + # + VfrUniBaseName["UniOffsetName"] = (self.Name + "Strings") + + if not VfrUniBaseName: + return None + MapFileName = os.path.join(self.OutputDir, self.Name + ".map") + EfiFileName = os.path.join(self.OutputDir, self.Name + ".efi") + VfrUniOffsetList = GetVariableOffset(MapFileName, EfiFileName, list(VfrUniBaseName.values())) + if not VfrUniOffsetList: + return None + + OutputName = '%sOffset.bin' % self.Name + UniVfrOffsetFileName = os.path.join( self.OutputDir, OutputName) + + try: + fInputfile = open(UniVfrOffsetFileName, "wb+", 0) + except: + EdkLogger.error("build", FILE_OPEN_FAILURE, "File open failed for %s" % UniVfrOffsetFileName, None) + + # Use a instance of BytesIO to cache data + fStringIO = BytesIO() + + for Item in VfrUniOffsetList: + if (Item[0].find("Strings") != -1): + # + # UNI offset in image. + # GUID + Offset + # { 0x8913c5e0, 0x33f6, 0x4d86, { 0x9b, 0xf1, 0x43, 0xef, 0x89, 0xfc, 0x6, 0x66 } } + # + UniGuid = b'\xe0\xc5\x13\x89\xf63\x86M\x9b\xf1C\xef\x89\xfc\x06f' + fStringIO.write(UniGuid) + UniValue = pack ('Q', int (Item[1], 16)) + fStringIO.write (UniValue) + else: + # + # VFR binary offset in image. + # GUID + Offset + # { 0xd0bc7cb4, 0x6a47, 0x495f, { 0xaa, 0x11, 0x71, 0x7, 0x46, 0xda, 0x6, 0xa2 } }; + # + VfrGuid = b'\xb4|\xbc\xd0Gj_I\xaa\x11q\x07F\xda\x06\xa2' + fStringIO.write(VfrGuid) + VfrValue = pack ('Q', int (Item[1], 16)) + fStringIO.write (VfrValue) + # + # write data into file. + # + try : + fInputfile.write (fStringIO.getvalue()) + except: + EdkLogger.error("build", FILE_WRITE_FAILURE, "Write data to file %s failed, please check whether the " + "file been locked or using by other applications." %UniVfrOffsetFileName, None) + + fStringIO.close () + fInputfile.close () + return OutputName + + @cached_property + def OutputFile(self): + retVal = set() + + for Root, Dirs, Files in os.walk(self.BuildDir): + for File in Files: + # lib file is already added through above CodaTargetList, skip it here + if not (File.lower().endswith('.obj') or File.lower().endswith('.debug')): + NewFile = path.join(Root, File) + retVal.add(NewFile) + + for Root, Dirs, Files in os.walk(self.FfsOutputDir): + for File in Files: + NewFile = path.join(Root, File) + retVal.add(NewFile) + + return retVal + + ## Create AsBuilt INF file the module + # + def CreateAsBuiltInf(self): + + if self.IsAsBuiltInfCreated: + return + + # Skip INF file generation for libraries + if self.IsLibrary: + return + + # Skip the following code for modules with no source files + if not self.SourceFileList: + return + + # Skip the following code for modules without any binary files + if self.BinaryFileList: + return + + ### TODO: How to handles mixed source and binary modules + + # Find all DynamicEx and PatchableInModule PCDs used by this module and dependent libraries + # Also find all packages that the DynamicEx PCDs depend on + Pcds = [] + PatchablePcds = [] + Packages = [] + PcdCheckList = [] + PcdTokenSpaceList = [] + for Pcd in self.ModulePcdList + self.LibraryPcdList: + if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE: + PatchablePcds.append(Pcd) + PcdCheckList.append((Pcd.TokenCName, Pcd.TokenSpaceGuidCName, TAB_PCDS_PATCHABLE_IN_MODULE)) + elif Pcd.Type in PCD_DYNAMIC_EX_TYPE_SET: + if Pcd not in Pcds: + Pcds.append(Pcd) + PcdCheckList.append((Pcd.TokenCName, Pcd.TokenSpaceGuidCName, TAB_PCDS_DYNAMIC_EX)) + PcdCheckList.append((Pcd.TokenCName, Pcd.TokenSpaceGuidCName, TAB_PCDS_DYNAMIC)) + PcdTokenSpaceList.append(Pcd.TokenSpaceGuidCName) + GuidList = OrderedDict(self.GuidList) + for TokenSpace in self.GetGuidsUsedByPcd: + # If token space is not referred by patch PCD or Ex PCD, remove the GUID from GUID list + # The GUIDs in GUIDs section should really be the GUIDs in source INF or referred by Ex an patch PCDs + if TokenSpace not in PcdTokenSpaceList and TokenSpace in GuidList: + GuidList.pop(TokenSpace) + CheckList = (GuidList, self.PpiList, self.ProtocolList, PcdCheckList) + for Package in self.DerivedPackageList: + if Package in Packages: + continue + BeChecked = (Package.Guids, Package.Ppis, Package.Protocols, Package.Pcds) + Found = False + for Index in range(len(BeChecked)): + for Item in CheckList[Index]: + if Item in BeChecked[Index]: + Packages.append(Package) + Found = True + break + if Found: + break + + VfrPcds = self._GetPcdsMaybeUsedByVfr() + for Pkg in self.PlatformInfo.PackageList: + if Pkg in Packages: + continue + for VfrPcd in VfrPcds: + if ((VfrPcd.TokenCName, VfrPcd.TokenSpaceGuidCName, TAB_PCDS_DYNAMIC_EX) in Pkg.Pcds or + (VfrPcd.TokenCName, VfrPcd.TokenSpaceGuidCName, TAB_PCDS_DYNAMIC) in Pkg.Pcds): + Packages.append(Pkg) + break + + ModuleType = SUP_MODULE_DXE_DRIVER if self.ModuleType == SUP_MODULE_UEFI_DRIVER and self.DepexGenerated else self.ModuleType + DriverType = self.PcdIsDriver if self.PcdIsDriver else '' + Guid = self.Guid + MDefs = self.Module.Defines + + AsBuiltInfDict = { + 'module_name' : self.Name, + 'module_guid' : Guid, + 'module_module_type' : ModuleType, + 'module_version_string' : [MDefs['VERSION_STRING']] if 'VERSION_STRING' in MDefs else [], + 'pcd_is_driver_string' : [], + 'module_uefi_specification_version' : [], + 'module_pi_specification_version' : [], + 'module_entry_point' : self.Module.ModuleEntryPointList, + 'module_unload_image' : self.Module.ModuleUnloadImageList, + 'module_constructor' : self.Module.ConstructorList, + 'module_destructor' : self.Module.DestructorList, + 'module_shadow' : [MDefs['SHADOW']] if 'SHADOW' in MDefs else [], + 'module_pci_vendor_id' : [MDefs['PCI_VENDOR_ID']] if 'PCI_VENDOR_ID' in MDefs else [], + 'module_pci_device_id' : [MDefs['PCI_DEVICE_ID']] if 'PCI_DEVICE_ID' in MDefs else [], + 'module_pci_class_code' : [MDefs['PCI_CLASS_CODE']] if 'PCI_CLASS_CODE' in MDefs else [], + 'module_pci_revision' : [MDefs['PCI_REVISION']] if 'PCI_REVISION' in MDefs else [], + 'module_build_number' : [MDefs['BUILD_NUMBER']] if 'BUILD_NUMBER' in MDefs else [], + 'module_spec' : [MDefs['SPEC']] if 'SPEC' in MDefs else [], + 'module_uefi_hii_resource_section' : [MDefs['UEFI_HII_RESOURCE_SECTION']] if 'UEFI_HII_RESOURCE_SECTION' in MDefs else [], + 'module_uni_file' : [MDefs['MODULE_UNI_FILE']] if 'MODULE_UNI_FILE' in MDefs else [], + 'module_arch' : self.Arch, + 'package_item' : [Package.MetaFile.File.replace('\\', '/') for Package in Packages], + 'binary_item' : [], + 'patchablepcd_item' : [], + 'pcd_item' : [], + 'protocol_item' : [], + 'ppi_item' : [], + 'guid_item' : [], + 'flags_item' : [], + 'libraryclasses_item' : [] + } + + if 'MODULE_UNI_FILE' in MDefs: + UNIFile = os.path.join(self.MetaFile.Dir, MDefs['MODULE_UNI_FILE']) + if os.path.isfile(UNIFile): + shutil.copy2(UNIFile, self.OutputDir) + + if self.AutoGenVersion > int(gInfSpecVersion, 0): + AsBuiltInfDict['module_inf_version'] = '0x%08x' % self.AutoGenVersion + else: + AsBuiltInfDict['module_inf_version'] = gInfSpecVersion + + if DriverType: + AsBuiltInfDict['pcd_is_driver_string'].append(DriverType) + + if 'UEFI_SPECIFICATION_VERSION' in self.Specification: + AsBuiltInfDict['module_uefi_specification_version'].append(self.Specification['UEFI_SPECIFICATION_VERSION']) + if 'PI_SPECIFICATION_VERSION' in self.Specification: + AsBuiltInfDict['module_pi_specification_version'].append(self.Specification['PI_SPECIFICATION_VERSION']) + + OutputDir = self.OutputDir.replace('\\', '/').strip('/') + DebugDir = self.DebugDir.replace('\\', '/').strip('/') + for Item in self.CodaTargetList: + File = Item.Target.Path.replace('\\', '/').strip('/').replace(DebugDir, '').replace(OutputDir, '').strip('/') + if os.path.isabs(File): + File = File.replace('\\', '/').strip('/').replace(OutputDir, '').strip('/') + if Item.Target.Ext.lower() == '.aml': + AsBuiltInfDict['binary_item'].append('ASL|' + File) + elif Item.Target.Ext.lower() == '.acpi': + AsBuiltInfDict['binary_item'].append('ACPI|' + File) + elif Item.Target.Ext.lower() == '.efi': + AsBuiltInfDict['binary_item'].append('PE32|' + self.Name + '.efi') + else: + AsBuiltInfDict['binary_item'].append('BIN|' + File) + if not self.DepexGenerated: + DepexFile = os.path.join(self.OutputDir, self.Name + '.depex') + if os.path.exists(DepexFile): + self.DepexGenerated = True + if self.DepexGenerated: + if self.ModuleType in [SUP_MODULE_PEIM]: + AsBuiltInfDict['binary_item'].append('PEI_DEPEX|' + self.Name + '.depex') + elif self.ModuleType in [SUP_MODULE_DXE_DRIVER, SUP_MODULE_DXE_RUNTIME_DRIVER, SUP_MODULE_DXE_SAL_DRIVER, SUP_MODULE_UEFI_DRIVER]: + AsBuiltInfDict['binary_item'].append('DXE_DEPEX|' + self.Name + '.depex') + elif self.ModuleType in [SUP_MODULE_DXE_SMM_DRIVER]: + AsBuiltInfDict['binary_item'].append('SMM_DEPEX|' + self.Name + '.depex') + + Bin = self._GenOffsetBin() + if Bin: + AsBuiltInfDict['binary_item'].append('BIN|%s' % Bin) + + for Root, Dirs, Files in os.walk(OutputDir): + for File in Files: + if File.lower().endswith('.pdb'): + AsBuiltInfDict['binary_item'].append('DISPOSABLE|' + File) + HeaderComments = self.Module.HeaderComments + StartPos = 0 + for Index in range(len(HeaderComments)): + if HeaderComments[Index].find('@BinaryHeader') != -1: + HeaderComments[Index] = HeaderComments[Index].replace('@BinaryHeader', '@file') + StartPos = Index + break + AsBuiltInfDict['header_comments'] = '\n'.join(HeaderComments[StartPos:]).replace(':#', '://') + AsBuiltInfDict['tail_comments'] = '\n'.join(self.Module.TailComments) + + GenList = [ + (self.ProtocolList, self._ProtocolComments, 'protocol_item'), + (self.PpiList, self._PpiComments, 'ppi_item'), + (GuidList, self._GuidComments, 'guid_item') + ] + for Item in GenList: + for CName in Item[0]: + Comments = '\n '.join(Item[1][CName]) if CName in Item[1] else '' + Entry = Comments + '\n ' + CName if Comments else CName + AsBuiltInfDict[Item[2]].append(Entry) + PatchList = parsePcdInfoFromMapFile( + os.path.join(self.OutputDir, self.Name + '.map'), + os.path.join(self.OutputDir, self.Name + '.efi') + ) + if PatchList: + for Pcd in PatchablePcds: + TokenCName = Pcd.TokenCName + for PcdItem in GlobalData.MixedPcd: + if (Pcd.TokenCName, Pcd.TokenSpaceGuidCName) in GlobalData.MixedPcd[PcdItem]: + TokenCName = PcdItem[0] + break + for PatchPcd in PatchList: + if TokenCName == PatchPcd[0]: + break + else: + continue + PcdValue = '' + if Pcd.DatumType == 'BOOLEAN': + BoolValue = Pcd.DefaultValue.upper() + if BoolValue == 'TRUE': + Pcd.DefaultValue = '1' + elif BoolValue == 'FALSE': + Pcd.DefaultValue = '0' + + if Pcd.DatumType in TAB_PCD_NUMERIC_TYPES: + HexFormat = '0x%02x' + if Pcd.DatumType == TAB_UINT16: + HexFormat = '0x%04x' + elif Pcd.DatumType == TAB_UINT32: + HexFormat = '0x%08x' + elif Pcd.DatumType == TAB_UINT64: + HexFormat = '0x%016x' + PcdValue = HexFormat % int(Pcd.DefaultValue, 0) + else: + if Pcd.MaxDatumSize is None or Pcd.MaxDatumSize == '': + EdkLogger.error("build", AUTOGEN_ERROR, + "Unknown [MaxDatumSize] of PCD [%s.%s]" % (Pcd.TokenSpaceGuidCName, TokenCName) + ) + ArraySize = int(Pcd.MaxDatumSize, 0) + PcdValue = Pcd.DefaultValue + if PcdValue[0] != '{': + Unicode = False + if PcdValue[0] == 'L': + Unicode = True + PcdValue = PcdValue.lstrip('L') + PcdValue = eval(PcdValue) + NewValue = '{' + for Index in range(0, len(PcdValue)): + if Unicode: + CharVal = ord(PcdValue[Index]) + NewValue = NewValue + '0x%02x' % (CharVal & 0x00FF) + ', ' \ + + '0x%02x' % (CharVal >> 8) + ', ' + else: + NewValue = NewValue + '0x%02x' % (ord(PcdValue[Index]) % 0x100) + ', ' + Padding = '0x00, ' + if Unicode: + Padding = Padding * 2 + ArraySize = ArraySize // 2 + if ArraySize < (len(PcdValue) + 1): + if Pcd.MaxSizeUserSet: + EdkLogger.error("build", AUTOGEN_ERROR, + "The maximum size of VOID* type PCD '%s.%s' is less than its actual size occupied." % (Pcd.TokenSpaceGuidCName, TokenCName) + ) + else: + ArraySize = len(PcdValue) + 1 + if ArraySize > len(PcdValue) + 1: + NewValue = NewValue + Padding * (ArraySize - len(PcdValue) - 1) + PcdValue = NewValue + Padding.strip().rstrip(',') + '}' + elif len(PcdValue.split(',')) <= ArraySize: + PcdValue = PcdValue.rstrip('}') + ', 0x00' * (ArraySize - len(PcdValue.split(','))) + PcdValue += '}' + else: + if Pcd.MaxSizeUserSet: + EdkLogger.error("build", AUTOGEN_ERROR, + "The maximum size of VOID* type PCD '%s.%s' is less than its actual size occupied." % (Pcd.TokenSpaceGuidCName, TokenCName) + ) + else: + ArraySize = len(PcdValue) + 1 + PcdItem = '%s.%s|%s|0x%X' % \ + (Pcd.TokenSpaceGuidCName, TokenCName, PcdValue, PatchPcd[1]) + PcdComments = '' + if (Pcd.TokenSpaceGuidCName, Pcd.TokenCName) in self._PcdComments: + PcdComments = '\n '.join(self._PcdComments[Pcd.TokenSpaceGuidCName, Pcd.TokenCName]) + if PcdComments: + PcdItem = PcdComments + '\n ' + PcdItem + AsBuiltInfDict['patchablepcd_item'].append(PcdItem) + + for Pcd in Pcds + VfrPcds: + PcdCommentList = [] + HiiInfo = '' + TokenCName = Pcd.TokenCName + for PcdItem in GlobalData.MixedPcd: + if (Pcd.TokenCName, Pcd.TokenSpaceGuidCName) in GlobalData.MixedPcd[PcdItem]: + TokenCName = PcdItem[0] + break + if Pcd.Type == TAB_PCDS_DYNAMIC_EX_HII: + for SkuName in Pcd.SkuInfoList: + SkuInfo = Pcd.SkuInfoList[SkuName] + HiiInfo = '## %s|%s|%s' % (SkuInfo.VariableName, SkuInfo.VariableGuid, SkuInfo.VariableOffset) + break + if (Pcd.TokenSpaceGuidCName, Pcd.TokenCName) in self._PcdComments: + PcdCommentList = self._PcdComments[Pcd.TokenSpaceGuidCName, Pcd.TokenCName][:] + if HiiInfo: + UsageIndex = -1 + UsageStr = '' + for Index, Comment in enumerate(PcdCommentList): + for Usage in UsageList: + if Comment.find(Usage) != -1: + UsageStr = Usage + UsageIndex = Index + break + if UsageIndex != -1: + PcdCommentList[UsageIndex] = '## %s %s %s' % (UsageStr, HiiInfo, PcdCommentList[UsageIndex].replace(UsageStr, '')) + else: + PcdCommentList.append('## UNDEFINED ' + HiiInfo) + PcdComments = '\n '.join(PcdCommentList) + PcdEntry = Pcd.TokenSpaceGuidCName + '.' + TokenCName + if PcdComments: + PcdEntry = PcdComments + '\n ' + PcdEntry + AsBuiltInfDict['pcd_item'].append(PcdEntry) + for Item in self.BuildOption: + if 'FLAGS' in self.BuildOption[Item]: + AsBuiltInfDict['flags_item'].append('%s:%s_%s_%s_%s_FLAGS = %s' % (self.ToolChainFamily, self.BuildTarget, self.ToolChain, self.Arch, Item, self.BuildOption[Item]['FLAGS'].strip())) + + # Generated LibraryClasses section in comments. + for Library in self.LibraryAutoGenList: + AsBuiltInfDict['libraryclasses_item'].append(Library.MetaFile.File.replace('\\', '/')) + + # Generated UserExtensions TianoCore section. + # All tianocore user extensions are copied. + UserExtStr = '' + for TianoCore in self._GetTianoCoreUserExtensionList(): + UserExtStr += '\n'.join(TianoCore) + ExtensionFile = os.path.join(self.MetaFile.Dir, TianoCore[1]) + if os.path.isfile(ExtensionFile): + shutil.copy2(ExtensionFile, self.OutputDir) + AsBuiltInfDict['userextension_tianocore_item'] = UserExtStr + + # Generated depex expression section in comments. + DepexExpression = self._GetDepexExpresionString() + AsBuiltInfDict['depexsection_item'] = DepexExpression if DepexExpression else '' + + AsBuiltInf = TemplateString() + AsBuiltInf.Append(gAsBuiltInfHeaderString.Replace(AsBuiltInfDict)) + + SaveFileOnChange(os.path.join(self.OutputDir, self.Name + '.inf'), str(AsBuiltInf), False) + + self.IsAsBuiltInfCreated = True + + def CacheCopyFile(self, DestDir, SourceDir, File): + if os.path.isdir(File): + return + + sub_dir = os.path.relpath(File, SourceDir) + destination_file = os.path.join(DestDir, sub_dir) + destination_dir = os.path.dirname(destination_file) + CreateDirectory(destination_dir) + try: + CopyFileOnChange(File, destination_dir) + except: + EdkLogger.quiet("[cache warning]: fail to copy file:%s to folder:%s" % (File, destination_dir)) + return + + def CopyModuleToCache(self): + # Find the MakeHashStr and PreMakeHashStr from latest MakeHashFileList + # and PreMakeHashFileList files + MakeHashStr = None + PreMakeHashStr = None + MakeTimeStamp = 0 + PreMakeTimeStamp = 0 + Files = [f for f in os.listdir(LongFilePath(self.BuildDir)) if path.isfile(LongFilePath(path.join(self.BuildDir, f)))] + for File in Files: + if ".MakeHashFileList." in File: + #find lastest file through time stamp + FileTimeStamp = os.stat(LongFilePath(path.join(self.BuildDir, File)))[8] + if FileTimeStamp > MakeTimeStamp: + MakeTimeStamp = FileTimeStamp + MakeHashStr = File.split('.')[-1] + if len(MakeHashStr) != 32: + EdkLogger.quiet("[cache error]: wrong MakeHashFileList file:%s" % (File)) + if ".PreMakeHashFileList." in File: + FileTimeStamp = os.stat(LongFilePath(path.join(self.BuildDir, File)))[8] + if FileTimeStamp > PreMakeTimeStamp: + PreMakeTimeStamp = FileTimeStamp + PreMakeHashStr = File.split('.')[-1] + if len(PreMakeHashStr) != 32: + EdkLogger.quiet("[cache error]: wrong PreMakeHashFileList file:%s" % (File)) + + if not MakeHashStr: + EdkLogger.quiet("[cache error]: No MakeHashFileList file for module:%s[%s]" % (self.MetaFile.Path, self.Arch)) + return + if not PreMakeHashStr: + EdkLogger.quiet("[cache error]: No PreMakeHashFileList file for module:%s[%s]" % (self.MetaFile.Path, self.Arch)) + return + + # Create Cache destination dirs + FileDir = path.join(GlobalData.gBinCacheDest, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName) + FfsDir = path.join(GlobalData.gBinCacheDest, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name) + CacheFileDir = path.join(FileDir, MakeHashStr) + CacheFfsDir = path.join(FfsDir, MakeHashStr) + CreateDirectory (CacheFileDir) + CreateDirectory (CacheFfsDir) + + # Create ModuleHashPair file to support multiple version cache together + ModuleHashPair = path.join(FileDir, self.Name + ".ModuleHashPair") + ModuleHashPairList = [] # tuple list: [tuple(PreMakefileHash, MakeHash)] + if os.path.exists(ModuleHashPair): + with open(ModuleHashPair, 'r') as f: + ModuleHashPairList = json.load(f) + if not (PreMakeHashStr, MakeHashStr) in set(map(tuple, ModuleHashPairList)): + ModuleHashPairList.insert(0, (PreMakeHashStr, MakeHashStr)) + with open(ModuleHashPair, 'w') as f: + json.dump(ModuleHashPairList, f, indent=2) + + # Copy files to Cache destination dirs + if not self.OutputFile: + Ma = self.BuildDatabase[self.MetaFile, self.Arch, self.BuildTarget, self.ToolChain] + self.OutputFile = Ma.Binaries + for File in self.OutputFile: + if File.startswith(os.path.abspath(self.FfsOutputDir)+os.sep): + self.CacheCopyFile(CacheFfsDir, self.FfsOutputDir, File) + else: + if self.Name + ".autogen.hash." in File or \ + self.Name + ".autogen.hashchain." in File or \ + self.Name + ".hash." in File or \ + self.Name + ".hashchain." in File or \ + self.Name + ".PreMakeHashFileList." in File or \ + self.Name + ".MakeHashFileList." in File: + self.CacheCopyFile(FileDir, self.BuildDir, File) + else: + self.CacheCopyFile(CacheFileDir, self.BuildDir, File) + ## Create makefile for the module and its dependent libraries + # + # @param CreateLibraryMakeFile Flag indicating if or not the makefiles of + # dependent libraries will be created + # + @cached_class_function + def CreateMakeFile(self, CreateLibraryMakeFile=True, GenFfsList = []): + + # nest this function inside it's only caller. + def CreateTimeStamp(): + FileSet = {self.MetaFile.Path} + + for SourceFile in self.Module.Sources: + FileSet.add (SourceFile.Path) + + for Lib in self.DependentLibraryList: + FileSet.add (Lib.MetaFile.Path) + + for f in self.AutoGenDepSet: + FileSet.add (f.Path) + + if os.path.exists (self.TimeStampPath): + os.remove (self.TimeStampPath) + + SaveFileOnChange(self.TimeStampPath, "\n".join(FileSet), False) + + # Ignore generating makefile when it is a binary module + if self.IsBinaryModule: + return + + self.GenFfsList = GenFfsList + + if not self.IsLibrary and CreateLibraryMakeFile: + for LibraryAutoGen in self.LibraryAutoGenList: + LibraryAutoGen.CreateMakeFile() + + # CanSkip uses timestamps to determine build skipping + if self.CanSkip(): + return + + if len(self.CustomMakefile) == 0: + Makefile = GenMake.ModuleMakefile(self) + else: + Makefile = GenMake.CustomMakefile(self) + if Makefile.Generate(): + EdkLogger.debug(EdkLogger.DEBUG_9, "Generated makefile for module %s [%s]" % + (self.Name, self.Arch)) + else: + EdkLogger.debug(EdkLogger.DEBUG_9, "Skipped the generation of makefile for module %s [%s]" % + (self.Name, self.Arch)) + + CreateTimeStamp() + + MakefileType = Makefile._FileType + MakefileName = Makefile._FILE_NAME_[MakefileType] + MakefilePath = os.path.join(self.MakeFileDir, MakefileName) + FilePath = path.join(self.BuildDir, self.Name + ".makefile") + SaveFileOnChange(FilePath, MakefilePath, False) + + def CopyBinaryFiles(self): + for File in self.Module.Binaries: + SrcPath = File.Path + DstPath = os.path.join(self.OutputDir, os.path.basename(SrcPath)) + CopyLongFilePath(SrcPath, DstPath) + ## Create autogen code for the module and its dependent libraries + # + # @param CreateLibraryCodeFile Flag indicating if or not the code of + # dependent libraries will be created + # + def CreateCodeFile(self, CreateLibraryCodeFile=True): + + if self.IsCodeFileCreated: + return + + # Need to generate PcdDatabase even PcdDriver is binarymodule + if self.IsBinaryModule and self.PcdIsDriver != '': + CreatePcdDatabaseCode(self, TemplateString(), TemplateString()) + return + if self.IsBinaryModule: + if self.IsLibrary: + self.CopyBinaryFiles() + return + + if not self.IsLibrary and CreateLibraryCodeFile: + for LibraryAutoGen in self.LibraryAutoGenList: + LibraryAutoGen.CreateCodeFile() + + self.LibraryAutoGenList + AutoGenList = [] + IgoredAutoGenList = [] + + for File in self.AutoGenFileList: + if GenC.Generate(File.Path, self.AutoGenFileList[File], File.IsBinary): + AutoGenList.append(str(File)) + else: + IgoredAutoGenList.append(str(File)) + + + for ModuleType in self.DepexList: + # Ignore empty [depex] section or [depex] section for SUP_MODULE_USER_DEFINED module + if len(self.DepexList[ModuleType]) == 0 or ModuleType == SUP_MODULE_USER_DEFINED or ModuleType == SUP_MODULE_HOST_APPLICATION: + continue + + Dpx = GenDepex.DependencyExpression(self.DepexList[ModuleType], ModuleType, True) + DpxFile = gAutoGenDepexFileName % {"module_name" : self.Name} + + if len(Dpx.PostfixNotation) != 0: + self.DepexGenerated = True + + if Dpx.Generate(path.join(self.OutputDir, DpxFile)): + AutoGenList.append(str(DpxFile)) + else: + IgoredAutoGenList.append(str(DpxFile)) + + if IgoredAutoGenList == []: + EdkLogger.debug(EdkLogger.DEBUG_9, "Generated [%s] files for module %s [%s]" % + (" ".join(AutoGenList), self.Name, self.Arch)) + elif AutoGenList == []: + EdkLogger.debug(EdkLogger.DEBUG_9, "Skipped the generation of [%s] files for module %s [%s]" % + (" ".join(IgoredAutoGenList), self.Name, self.Arch)) + else: + EdkLogger.debug(EdkLogger.DEBUG_9, "Generated [%s] (skipped %s) files for module %s [%s]" % + (" ".join(AutoGenList), " ".join(IgoredAutoGenList), self.Name, self.Arch)) + + self.IsCodeFileCreated = True + + return AutoGenList + + ## Summarize the ModuleAutoGen objects of all libraries used by this module + @cached_property + def LibraryAutoGenList(self): + RetVal = [] + for Library in self.DependentLibraryList: + La = ModuleAutoGen( + self.Workspace, + Library.MetaFile, + self.BuildTarget, + self.ToolChain, + self.Arch, + self.PlatformInfo.MetaFile, + self.DataPipe + ) + La.IsLibrary = True + if La not in RetVal: + RetVal.append(La) + for Lib in La.CodaTargetList: + self._ApplyBuildRule(Lib.Target, TAB_UNKNOWN_FILE) + return RetVal + + def GenCMakeHash(self): + # GenCMakeHash can only be called in --binary-destination + # Never called in multiprocessing and always directly save result in main process, + # so no need remote dict to share the gCMakeHashFile result with main process + + DependencyFileSet = set() + # Add AutoGen files + if self.AutoGenFileList: + for File in set(self.AutoGenFileList): + DependencyFileSet.add(File) + + # Add Makefile + abspath = path.join(self.BuildDir, self.Name + ".makefile") + try: + with open(LongFilePath(abspath),"r") as fd: + lines = fd.readlines() + except Exception as e: + EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False) + if lines: + DependencyFileSet.update(lines) + + # Caculate all above dependency files hash + # Initialze hash object + FileList = [] + m = hashlib.md5() + for File in sorted(DependencyFileSet, key=lambda x: str(x)): + if not path.exists(LongFilePath(str(File))): + EdkLogger.quiet("[cache warning]: header file %s is missing for module: %s[%s]" % (File, self.MetaFile.Path, self.Arch)) + continue + with open(LongFilePath(str(File)), 'rb') as f: + Content = f.read() + m.update(Content) + FileList.append((str(File), hashlib.md5(Content).hexdigest())) + + HashChainFile = path.join(self.BuildDir, self.Name + ".autogen.hashchain." + m.hexdigest()) + GlobalData.gCMakeHashFile[(self.MetaFile.Path, self.Arch)] = HashChainFile + try: + with open(LongFilePath(HashChainFile), 'w') as f: + json.dump(FileList, f, indent=2) + except: + EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile) + return False + + def GenModuleHash(self): + # GenModuleHash only called after autogen phase + # Never called in multiprocessing and always directly save result in main process, + # so no need remote dict to share the gModuleHashFile result with main process + # + # GenPreMakefileHashList consume no dict. + # GenPreMakefileHashList produce local gModuleHashFile dict. + + DependencyFileSet = set() + # Add Module Meta file + DependencyFileSet.add(self.MetaFile.Path) + + # Add Module's source files + if self.SourceFileList: + for File in set(self.SourceFileList): + DependencyFileSet.add(File.Path) + + # Add modules's include header files + # Directly use the deps.txt file in the module BuildDir + abspath = path.join(self.BuildDir, "deps.txt") + rt = None + try: + with open(LongFilePath(abspath),"r") as fd: + lines = fd.readlines() + if lines: + rt = set([item.lstrip().strip("\n") for item in lines if item.strip("\n").endswith(".h")]) + except Exception as e: + EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False) + + if rt: + DependencyFileSet.update(rt) + + + # Caculate all above dependency files hash + # Initialze hash object + FileList = [] + m = hashlib.md5() + BuildDirStr = path.abspath(self.BuildDir).lower() + for File in sorted(DependencyFileSet, key=lambda x: str(x)): + # Skip the AutoGen files in BuildDir which already been + # included in .autogen.hash. file + if BuildDirStr in path.abspath(File).lower(): + continue + if not path.exists(LongFilePath(File)): + EdkLogger.quiet("[cache warning]: header file %s is missing for module: %s[%s]" % (File, self.MetaFile.Path, self.Arch)) + continue + with open(LongFilePath(File), 'rb') as f: + Content = f.read() + m.update(Content) + FileList.append((File, hashlib.md5(Content).hexdigest())) + + HashChainFile = path.join(self.BuildDir, self.Name + ".hashchain." + m.hexdigest()) + GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)] = HashChainFile + try: + with open(LongFilePath(HashChainFile), 'w') as f: + json.dump(FileList, f, indent=2) + except: + EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile) + return False + + def GenPreMakefileHashList(self): + # GenPreMakefileHashList consume below dicts: + # gPlatformHashFile + # gPackageHashFile + # gModuleHashFile + # GenPreMakefileHashList produce no dict. + # gModuleHashFile items might be produced in multiprocessing, so + # need check gModuleHashFile remote dict + + # skip binary module + if self.IsBinaryModule: + return + + FileList = [] + m = hashlib.md5() + # Add Platform level hash + HashFile = GlobalData.gPlatformHashFile + if path.exists(LongFilePath(HashFile)): + FileList.append(HashFile) + m.update(HashFile.encode('utf-8')) + else: + EdkLogger.quiet("[cache warning]: No Platform HashFile: %s" % HashFile) + + # Add Package level hash + if self.DependentPackageList: + for Pkg in sorted(self.DependentPackageList, key=lambda x: x.PackageName): + if not (Pkg.PackageName, Pkg.Arch) in GlobalData.gPackageHashFile: + EdkLogger.quiet("[cache warning]:No Package %s for module %s[%s]" % (Pkg.PackageName, self.MetaFile.Path, self.Arch)) + continue + HashFile = GlobalData.gPackageHashFile[(Pkg.PackageName, Pkg.Arch)] + if path.exists(LongFilePath(HashFile)): + FileList.append(HashFile) + m.update(HashFile.encode('utf-8')) + else: + EdkLogger.quiet("[cache warning]:No Package HashFile: %s" % HashFile) + + # Add Module self + # GenPreMakefileHashList needed in both --binary-destination + # and --hash. And --hash might save ModuleHashFile in remote dict + # during multiprocessing. + if (self.MetaFile.Path, self.Arch) in GlobalData.gModuleHashFile: + HashFile = GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)] + else: + EdkLogger.quiet("[cache error]:No ModuleHashFile for module: %s[%s]" % (self.MetaFile.Path, self.Arch)) + if path.exists(LongFilePath(HashFile)): + FileList.append(HashFile) + m.update(HashFile.encode('utf-8')) + else: + EdkLogger.quiet("[cache warning]:No Module HashFile: %s" % HashFile) + + # Add Library hash + if self.LibraryAutoGenList: + for Lib in sorted(self.LibraryAutoGenList, key=lambda x: x.MetaFile.Path): + + if (Lib.MetaFile.Path, Lib.Arch) in GlobalData.gModuleHashFile: + HashFile = GlobalData.gModuleHashFile[(Lib.MetaFile.Path, Lib.Arch)] + else: + EdkLogger.quiet("[cache error]:No ModuleHashFile for lib: %s[%s]" % (Lib.MetaFile.Path, Lib.Arch)) + if path.exists(LongFilePath(HashFile)): + FileList.append(HashFile) + m.update(HashFile.encode('utf-8')) + else: + EdkLogger.quiet("[cache warning]:No Lib HashFile: %s" % HashFile) + + # Save PreMakeHashFileList + FilePath = path.join(self.BuildDir, self.Name + ".PreMakeHashFileList." + m.hexdigest()) + try: + with open(LongFilePath(FilePath), 'w') as f: + json.dump(FileList, f, indent=0) + except: + EdkLogger.quiet("[cache warning]: fail to save PreMake HashFileList: %s" % FilePath) + + def GenMakefileHashList(self): + # GenMakefileHashList only need in --binary-destination which will + # everything in local dict. So don't need check remote dict. + + # skip binary module + if self.IsBinaryModule: + return + + FileList = [] + m = hashlib.md5() + # Add AutoGen hash + HashFile = GlobalData.gCMakeHashFile[(self.MetaFile.Path, self.Arch)] + if path.exists(LongFilePath(HashFile)): + FileList.append(HashFile) + m.update(HashFile.encode('utf-8')) + else: + EdkLogger.quiet("[cache warning]:No AutoGen HashFile: %s" % HashFile) + + # Add Module self + if (self.MetaFile.Path, self.Arch) in GlobalData.gModuleHashFile: + HashFile = GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)] + else: + EdkLogger.quiet("[cache error]:No ModuleHashFile for module: %s[%s]" % (self.MetaFile.Path, self.Arch)) + if path.exists(LongFilePath(HashFile)): + FileList.append(HashFile) + m.update(HashFile.encode('utf-8')) + else: + EdkLogger.quiet("[cache warning]:No Module HashFile: %s" % HashFile) + + # Add Library hash + if self.LibraryAutoGenList: + for Lib in sorted(self.LibraryAutoGenList, key=lambda x: x.MetaFile.Path): + if (Lib.MetaFile.Path, Lib.Arch) in GlobalData.gModuleHashFile: + HashFile = GlobalData.gModuleHashFile[(Lib.MetaFile.Path, Lib.Arch)] + else: + EdkLogger.quiet("[cache error]:No ModuleHashFile for lib: %s[%s]" % (Lib.MetaFile.Path, Lib.Arch)) + if path.exists(LongFilePath(HashFile)): + FileList.append(HashFile) + m.update(HashFile.encode('utf-8')) + else: + EdkLogger.quiet("[cache warning]:No Lib HashFile: %s" % HashFile) + + # Save MakeHashFileList + FilePath = path.join(self.BuildDir, self.Name + ".MakeHashFileList." + m.hexdigest()) + try: + with open(LongFilePath(FilePath), 'w') as f: + json.dump(FileList, f, indent=0) + except: + EdkLogger.quiet("[cache warning]: fail to save Make HashFileList: %s" % FilePath) + + def CheckHashChainFile(self, HashChainFile): + # Assume the HashChainFile basename format is the 'x.hashchain.16BytesHexStr' + # The x is module name and the 16BytesHexStr is md5 hexdigest of + # all hashchain files content + HashStr = HashChainFile.split('.')[-1] + if len(HashStr) != 32: + EdkLogger.quiet("[cache error]: wrong format HashChainFile:%s" % (File)) + return False + + try: + with open(LongFilePath(HashChainFile), 'r') as f: + HashChainList = json.load(f) + except: + EdkLogger.quiet("[cache error]: fail to load HashChainFile: %s" % HashChainFile) + return False + + # Print the different file info + # print(HashChainFile) + for idx, (SrcFile, SrcHash) in enumerate (HashChainList): + if SrcFile in GlobalData.gFileHashDict: + DestHash = GlobalData.gFileHashDict[SrcFile] + else: + try: + with open(LongFilePath(SrcFile), 'rb') as f: + Content = f.read() + DestHash = hashlib.md5(Content).hexdigest() + GlobalData.gFileHashDict[SrcFile] = DestHash + except IOError as X: + # cache miss if SrcFile is removed in new version code + GlobalData.gFileHashDict[SrcFile] = 0 + EdkLogger.quiet("[cache insight]: first cache miss file in %s is %s" % (HashChainFile, SrcFile)) + return False + if SrcHash != DestHash: + EdkLogger.quiet("[cache insight]: first cache miss file in %s is %s" % (HashChainFile, SrcFile)) + return False + + return True + + ## Decide whether we can skip the left autogen and make process + def CanSkipbyMakeCache(self): + # For --binary-source only + # CanSkipbyMakeCache consume below dicts: + # gModuleMakeCacheStatus + # gHashChainStatus + # GenPreMakefileHashList produce gModuleMakeCacheStatus, gModuleHashFile dict. + # all these dicts might be produced in multiprocessing, so + # need check these remote dict + + if not GlobalData.gBinCacheSource: + return False + + if (self.MetaFile.Path, self.Arch) in GlobalData.gModuleMakeCacheStatus: + return GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] + + # If Module is binary, which has special build rule, do not skip by cache. + if self.IsBinaryModule: + print("[cache miss]: MakeCache: Skip BinaryModule:", self.MetaFile.Path, self.Arch) + GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False + return False + + # see .inc as binary file, do not skip by hash + for f_ext in self.SourceFileList: + if '.inc' in str(f_ext): + print("[cache miss]: MakeCache: Skip '.inc' File:", self.MetaFile.Path, self.Arch) + GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False + return False + + ModuleCacheDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName) + FfsDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name) + + ModuleHashPairList = [] # tuple list: [tuple(PreMakefileHash, MakeHash)] + ModuleHashPair = path.join(ModuleCacheDir, self.Name + ".ModuleHashPair") + try: + with open(LongFilePath(ModuleHashPair), 'r') as f: + ModuleHashPairList = json.load(f) + except: + # ModuleHashPair might not exist for new added module + GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False + EdkLogger.quiet("[cache warning]: fail to load ModuleHashPair file: %s" % ModuleHashPair) + print("[cache miss]: MakeCache:", self.MetaFile.Path, self.Arch) + return False + + # Check the PreMakeHash in ModuleHashPairList one by one + for idx, (PreMakefileHash, MakeHash) in enumerate (ModuleHashPairList): + SourceHashDir = path.join(ModuleCacheDir, MakeHash) + SourceFfsHashDir = path.join(FfsDir, MakeHash) + PreMakeHashFileList_FilePah = path.join(ModuleCacheDir, self.Name + ".PreMakeHashFileList." + PreMakefileHash) + MakeHashFileList_FilePah = path.join(ModuleCacheDir, self.Name + ".MakeHashFileList." + MakeHash) + + try: + with open(LongFilePath(MakeHashFileList_FilePah), 'r') as f: + MakeHashFileList = json.load(f) + except: + EdkLogger.quiet("[cache error]: fail to load MakeHashFileList file: %s" % MakeHashFileList_FilePah) + continue + + HashMiss = False + for HashChainFile in MakeHashFileList: + HashChainStatus = None + if HashChainFile in GlobalData.gHashChainStatus: + HashChainStatus = GlobalData.gHashChainStatus[HashChainFile] + if HashChainStatus == False: + HashMiss = True + break + elif HashChainStatus == True: + continue + # Convert to path start with cache source dir + RelativePath = os.path.relpath(HashChainFile, self.WorkspaceDir) + NewFilePath = os.path.join(GlobalData.gBinCacheSource, RelativePath) + if self.CheckHashChainFile(NewFilePath): + GlobalData.gHashChainStatus[HashChainFile] = True + # Save the module self HashFile for GenPreMakefileHashList later usage + if self.Name + ".hashchain." in HashChainFile: + GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)] = HashChainFile + else: + GlobalData.gHashChainStatus[HashChainFile] = False + HashMiss = True + break + + if HashMiss: + continue + + # PreMakefile cache hit, restore the module build result + for root, dir, files in os.walk(SourceHashDir): + for f in files: + File = path.join(root, f) + self.CacheCopyFile(self.BuildDir, SourceHashDir, File) + if os.path.exists(SourceFfsHashDir): + for root, dir, files in os.walk(SourceFfsHashDir): + for f in files: + File = path.join(root, f) + self.CacheCopyFile(self.FfsOutputDir, SourceFfsHashDir, File) + + if self.Name == "PcdPeim" or self.Name == "PcdDxe": + CreatePcdDatabaseCode(self, TemplateString(), TemplateString()) + + print("[cache hit]: MakeCache:", self.MetaFile.Path, self.Arch) + GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = True + return True + + print("[cache miss]: MakeCache:", self.MetaFile.Path, self.Arch) + GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False + return False + + ## Decide whether we can skip the left autogen and make process + def CanSkipbyPreMakeCache(self): + # CanSkipbyPreMakeCache consume below dicts: + # gModulePreMakeCacheStatus + # gHashChainStatus + # gModuleHashFile + # GenPreMakefileHashList produce gModulePreMakeCacheStatus dict. + # all these dicts might be produced in multiprocessing, so + # need check these remote dicts + + if not GlobalData.gUseHashCache or GlobalData.gBinCacheDest: + return False + + if (self.MetaFile.Path, self.Arch) in GlobalData.gModulePreMakeCacheStatus: + return GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] + + # If Module is binary, which has special build rule, do not skip by cache. + if self.IsBinaryModule: + print("[cache miss]: PreMakeCache: Skip BinaryModule:", self.MetaFile.Path, self.Arch) + GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False + return False + + # see .inc as binary file, do not skip by hash + for f_ext in self.SourceFileList: + if '.inc' in str(f_ext): + print("[cache miss]: PreMakeCache: Skip '.inc' File:", self.MetaFile.Path, self.Arch) + GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False + return False + + # For --hash only in the incremental build + if not GlobalData.gBinCacheSource: + Files = [path.join(self.BuildDir, f) for f in os.listdir(self.BuildDir) if path.isfile(path.join(self.BuildDir, f))] + PreMakeHashFileList_FilePah = None + MakeTimeStamp = 0 + # Find latest PreMakeHashFileList file in self.BuildDir folder + for File in Files: + if ".PreMakeHashFileList." in File: + FileTimeStamp = os.stat(path.join(self.BuildDir, File))[8] + if FileTimeStamp > MakeTimeStamp: + MakeTimeStamp = FileTimeStamp + PreMakeHashFileList_FilePah = File + if not PreMakeHashFileList_FilePah: + GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False + return False + + try: + with open(LongFilePath(PreMakeHashFileList_FilePah), 'r') as f: + PreMakeHashFileList = json.load(f) + except: + EdkLogger.quiet("[cache error]: fail to load PreMakeHashFileList file: %s" % PreMakeHashFileList_FilePah) + print("[cache miss]: PreMakeCache:", self.MetaFile.Path, self.Arch) + GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False + return False + + HashMiss = False + for HashChainFile in PreMakeHashFileList: + HashChainStatus = None + if HashChainFile in GlobalData.gHashChainStatus: + HashChainStatus = GlobalData.gHashChainStatus[HashChainFile] + if HashChainStatus == False: + HashMiss = True + break + elif HashChainStatus == True: + continue + if self.CheckHashChainFile(HashChainFile): + GlobalData.gHashChainStatus[HashChainFile] = True + # Save the module self HashFile for GenPreMakefileHashList later usage + if self.Name + ".hashchain." in HashChainFile: + GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)] = HashChainFile + else: + GlobalData.gHashChainStatus[HashChainFile] = False + HashMiss = True + break + + if HashMiss: + print("[cache miss]: PreMakeCache:", self.MetaFile.Path, self.Arch) + GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False + return False + else: + print("[cache hit]: PreMakeCache:", self.MetaFile.Path, self.Arch) + GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = True + return True + + ModuleCacheDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName) + FfsDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name) + + ModuleHashPairList = [] # tuple list: [tuple(PreMakefileHash, MakeHash)] + ModuleHashPair = path.join(ModuleCacheDir, self.Name + ".ModuleHashPair") + try: + with open(LongFilePath(ModuleHashPair), 'r') as f: + ModuleHashPairList = json.load(f) + except: + # ModuleHashPair might not exist for new added module + GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False + EdkLogger.quiet("[cache warning]: fail to load ModuleHashPair file: %s" % ModuleHashPair) + print("[cache miss]: PreMakeCache:", self.MetaFile.Path, self.Arch) + return False + + # Check the PreMakeHash in ModuleHashPairList one by one + for idx, (PreMakefileHash, MakeHash) in enumerate (ModuleHashPairList): + SourceHashDir = path.join(ModuleCacheDir, MakeHash) + SourceFfsHashDir = path.join(FfsDir, MakeHash) + PreMakeHashFileList_FilePah = path.join(ModuleCacheDir, self.Name + ".PreMakeHashFileList." + PreMakefileHash) + MakeHashFileList_FilePah = path.join(ModuleCacheDir, self.Name + ".MakeHashFileList." + MakeHash) + + try: + with open(LongFilePath(PreMakeHashFileList_FilePah), 'r') as f: + PreMakeHashFileList = json.load(f) + except: + EdkLogger.quiet("[cache error]: fail to load PreMakeHashFileList file: %s" % PreMakeHashFileList_FilePah) + continue + + HashMiss = False + for HashChainFile in PreMakeHashFileList: + HashChainStatus = None + if HashChainFile in GlobalData.gHashChainStatus: + HashChainStatus = GlobalData.gHashChainStatus[HashChainFile] + if HashChainStatus == False: + HashMiss = True + break + elif HashChainStatus == True: + continue + # Convert to path start with cache source dir + RelativePath = os.path.relpath(HashChainFile, self.WorkspaceDir) + NewFilePath = os.path.join(GlobalData.gBinCacheSource, RelativePath) + if self.CheckHashChainFile(NewFilePath): + GlobalData.gHashChainStatus[HashChainFile] = True + else: + GlobalData.gHashChainStatus[HashChainFile] = False + HashMiss = True + break + + if HashMiss: + continue + + # PreMakefile cache hit, restore the module build result + for root, dir, files in os.walk(SourceHashDir): + for f in files: + File = path.join(root, f) + self.CacheCopyFile(self.BuildDir, SourceHashDir, File) + if os.path.exists(SourceFfsHashDir): + for root, dir, files in os.walk(SourceFfsHashDir): + for f in files: + File = path.join(root, f) + self.CacheCopyFile(self.FfsOutputDir, SourceFfsHashDir, File) + + if self.Name == "PcdPeim" or self.Name == "PcdDxe": + CreatePcdDatabaseCode(self, TemplateString(), TemplateString()) + + print("[cache hit]: PreMakeCache:", self.MetaFile.Path, self.Arch) + GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = True + return True + + print("[cache miss]: PreMakeCache:", self.MetaFile.Path, self.Arch) + GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False + return False + + ## Decide whether we can skip the Module build + def CanSkipbyCache(self, gHitSet): + # Hashing feature is off + if not GlobalData.gBinCacheSource: + return False + + if self in gHitSet: + return True + + return False + + ## Decide whether we can skip the ModuleAutoGen process + # If any source file is newer than the module than we cannot skip + # + def CanSkip(self): + # Don't skip if cache feature enabled + if GlobalData.gUseHashCache or GlobalData.gBinCacheDest or GlobalData.gBinCacheSource: + return False + if self.MakeFileDir in GlobalData.gSikpAutoGenCache: + return True + if not os.path.exists(self.TimeStampPath): + return False + #last creation time of the module + DstTimeStamp = os.stat(self.TimeStampPath)[8] + + SrcTimeStamp = self.Workspace._SrcTimeStamp + if SrcTimeStamp > DstTimeStamp: + return False + + with open(self.TimeStampPath,'r') as f: + for source in f: + source = source.rstrip('\n') + if not os.path.exists(source): + return False + if source not in ModuleAutoGen.TimeDict : + ModuleAutoGen.TimeDict[source] = os.stat(source)[8] + if ModuleAutoGen.TimeDict[source] > DstTimeStamp: + return False + GlobalData.gSikpAutoGenCache.add(self.MakeFileDir) + return True + + @cached_property + def TimeStampPath(self): + return os.path.join(self.MakeFileDir, 'AutoGenTimeStamp') |
