BaseTools: Leverage compiler output to optimize binary cache

Redesign the binary cache and bases on the compiler to
output the dependency header files info for every module.
The binary cache will directly consume the dependency header
files info and doesn't parse the C source code by iteself.
Also redesign the dependency files list format for module
and try to share the common lib hash result as more as
possible in local process. Remove the unnecessary share data
access across multiprocessing.

Signed-off-by: Steven Shi <steven.shi@intel.com>

Cc: Liming Gao <liming.gao@intel.com>
Cc: Bob Feng <bob.c.feng@intel.com>
Reviewed-by: Bob Feng <bob.c.feng@intel.com>

1 files changed, 52 insertions, 12 deletions

diff --git a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
index ec0c25bd14..668126aaac 100644
--- a/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
+++ b/BaseTools/Source/Python/AutoGen/WorkspaceAutoGen.py
@@ -23,6 +23,7 @@ from Common.StringUtils import NormPath
 from Common.BuildToolError import *

 from Common.DataType import *

 from Common.Misc import *

+import json

 

 ## Regular expression for splitting Dependency Expression string into tokens

 gDepexTokenPattern = re.compile("(\(|\)|\w+| \S+\.inf)")

@@ -127,7 +128,7 @@ class WorkspaceAutoGen(AutoGen):
 

         self.CreateBuildOptionsFile()

         self.CreatePcdTokenNumberFile()

-        self.CreateModuleHashInfo()

+        self.GeneratePlatformLevelHash()

 

     #

     # Merge Arch

@@ -528,12 +529,12 @@ class WorkspaceAutoGen(AutoGen):
                     PcdTokenNumber += str(Pa.PcdTokenNumber[Pcd.TokenCName, Pcd.TokenSpaceGuidCName])

         SaveFileOnChange(os.path.join(self.BuildDir, 'PcdTokenNumber'), PcdTokenNumber, False)

 

-    def CreateModuleHashInfo(self):

+    def GeneratePlatformLevelHash(self):

         #

         # Get set of workspace metafiles

         #

         AllWorkSpaceMetaFiles = self._GetMetaFiles(self.BuildTarget, self.ToolChain)

-

+        AllWorkSpaceMetaFileList = sorted(AllWorkSpaceMetaFiles, key=lambda x: str(x))

         #

         # Retrieve latest modified time of all metafiles

         #

@@ -544,16 +545,35 @@ class WorkspaceAutoGen(AutoGen):
         self._SrcTimeStamp = SrcTimeStamp

 

         if GlobalData.gUseHashCache:

+            FileList = []

             m = hashlib.md5()

-            for files in AllWorkSpaceMetaFiles:

-                if files.endswith('.dec'):

+            for file in AllWorkSpaceMetaFileList:

+                if file.endswith('.dec'):

                     continue

-                f = open(files, 'rb')

+                f = open(file, 'rb')

                 Content = f.read()

                 f.close()

                 m.update(Content)

-            SaveFileOnChange(os.path.join(self.BuildDir, 'AutoGen.hash'), m.hexdigest(), False)

-            GlobalData.gPlatformHash = m.hexdigest()

+                FileList.append((str(file), hashlib.md5(Content).hexdigest()))

+

+            HashDir = path.join(self.BuildDir, "Hash_Platform")

+            HashFile = path.join(HashDir, 'Platform.hash.' + m.hexdigest())

+            SaveFileOnChange(HashFile, m.hexdigest(), False)

+            HashChainFile = path.join(HashDir, 'Platform.hashchain.' + m.hexdigest())

+            GlobalData.gPlatformHashFile = HashChainFile

+            try:

+                with open(HashChainFile, 'w') as f:

+                    json.dump(FileList, f, indent=2)

+            except:

+                EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile)

+

+            if GlobalData.gBinCacheDest:

+                # Copy platform hash files to cache destination

+                FileDir = path.join(GlobalData.gBinCacheDest, self.OutputDir, self.BuildTarget + "_" + self.ToolChain, "Hash_Platform")

+                CacheFileDir = FileDir

+                CreateDirectory(CacheFileDir)

+                CopyFileOnChange(HashFile, CacheFileDir)

+                CopyFileOnChange(HashChainFile, CacheFileDir)

 

         #

         # Write metafile list to build directory

@@ -564,7 +584,7 @@ class WorkspaceAutoGen(AutoGen):
         if not os.path.exists(self.BuildDir):

             os.makedirs(self.BuildDir)

         with open(os.path.join(self.BuildDir, 'AutoGen'), 'w+') as file:

-            for f in AllWorkSpaceMetaFiles:

+            for f in AllWorkSpaceMetaFileList:

                 print(f, file=file)

         return True

 

@@ -572,15 +592,16 @@ class WorkspaceAutoGen(AutoGen):
         if Pkg.PackageName in GlobalData.gPackageHash:

             return

 

-        PkgDir = os.path.join(self.BuildDir, Pkg.Arch, Pkg.PackageName)

+        PkgDir = os.path.join(self.BuildDir, Pkg.Arch, "Hash_Pkg", Pkg.PackageName)

         CreateDirectory(PkgDir)

-        HashFile = os.path.join(PkgDir, Pkg.PackageName + '.hash')

+        FileList = []

         m = hashlib.md5()

         # Get .dec file's hash value

         f = open(Pkg.MetaFile.Path, 'rb')

         Content = f.read()

         f.close()

         m.update(Content)

+        FileList.append((str(Pkg.MetaFile.Path), hashlib.md5(Content).hexdigest()))

         # Get include files hash value

         if Pkg.Includes:

             for inc in sorted(Pkg.Includes, key=lambda x: str(x)):

@@ -591,9 +612,28 @@ class WorkspaceAutoGen(AutoGen):
                         Content = f.read()

                         f.close()

                         m.update(Content)

-        SaveFileOnChange(HashFile, m.hexdigest(), False)

+                        FileList.append((str(File_Path), hashlib.md5(Content).hexdigest()))

         GlobalData.gPackageHash[Pkg.PackageName] = m.hexdigest()

 

+        HashDir = PkgDir

+        HashFile = path.join(HashDir, Pkg.PackageName + '.hash.' + m.hexdigest())

+        SaveFileOnChange(HashFile, m.hexdigest(), False)

+        HashChainFile = path.join(HashDir, Pkg.PackageName + '.hashchain.' + m.hexdigest())

+        GlobalData.gPackageHashFile[(Pkg.PackageName, Pkg.Arch)] = HashChainFile

+        try:

+            with open(HashChainFile, 'w') as f:

+                json.dump(FileList, f, indent=2)

+        except:

+            EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile)

+

+        if GlobalData.gBinCacheDest:

+            # Copy Pkg hash files to cache destination dir

+            FileDir = path.join(GlobalData.gBinCacheDest, self.OutputDir, self.BuildTarget + "_" + self.ToolChain, Pkg.Arch, "Hash_Pkg", Pkg.PackageName)

+            CacheFileDir = FileDir

+            CreateDirectory(CacheFileDir)

+            CopyFileOnChange(HashFile, CacheFileDir)

+            CopyFileOnChange(HashChainFile, CacheFileDir)

+

     def _GetMetaFiles(self, Target, Toolchain):

         AllWorkSpaceMetaFiles = set()

         #