diff options
Diffstat (limited to 'BaseTools/Source/Python/AutoGen/AutoGenWorker.py')
| -rwxr-xr-x | BaseTools/Source/Python/AutoGen/AutoGenWorker.py | 662 |
1 files changed, 331 insertions, 331 deletions
diff --git a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py index 0ba2339bed..bba7416afd 100755 --- a/BaseTools/Source/Python/AutoGen/AutoGenWorker.py +++ b/BaseTools/Source/Python/AutoGen/AutoGenWorker.py @@ -1,331 +1,331 @@ -## @file
-# Create makefile for MS nmake and GNU make
-#
-# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR>
-# SPDX-License-Identifier: BSD-2-Clause-Patent
-#
-from __future__ import absolute_import
-import multiprocessing as mp
-import threading
-from Common.Misc import PathClass
-from AutoGen.ModuleAutoGen import ModuleAutoGen
-from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo
-import Common.GlobalData as GlobalData
-import Common.EdkLogger as EdkLogger
-import os
-from Common.MultipleWorkspace import MultipleWorkspace as mws
-from AutoGen.AutoGen import AutoGen
-from Workspace.WorkspaceDatabase import BuildDB
-try:
- from queue import Empty
-except:
- from Queue import Empty
-import traceback
-import sys
-from AutoGen.DataPipe import MemoryDataPipe
-import logging
-import time
-
-def clearQ(q):
- try:
- while True:
- q.get_nowait()
- except Empty:
- pass
-
-class LogAgent(threading.Thread):
- def __init__(self,log_q,log_level,log_file=None):
- super(LogAgent,self).__init__()
- self.log_q = log_q
- self.log_level = log_level
- self.log_file = log_file
- def InitLogger(self):
- # For DEBUG level (All DEBUG_0~9 are applicable)
- self._DebugLogger_agent = logging.getLogger("tool_debug_agent")
- _DebugFormatter = logging.Formatter("[%(asctime)s.%(msecs)d]: %(message)s", datefmt="%H:%M:%S")
- self._DebugLogger_agent.setLevel(self.log_level)
- _DebugChannel = logging.StreamHandler(sys.stdout)
- _DebugChannel.setFormatter(_DebugFormatter)
- self._DebugLogger_agent.addHandler(_DebugChannel)
-
- # For VERBOSE, INFO, WARN level
- self._InfoLogger_agent = logging.getLogger("tool_info_agent")
- _InfoFormatter = logging.Formatter("%(message)s")
- self._InfoLogger_agent.setLevel(self.log_level)
- _InfoChannel = logging.StreamHandler(sys.stdout)
- _InfoChannel.setFormatter(_InfoFormatter)
- self._InfoLogger_agent.addHandler(_InfoChannel)
-
- # For ERROR level
- self._ErrorLogger_agent = logging.getLogger("tool_error_agent")
- _ErrorFormatter = logging.Formatter("%(message)s")
- self._ErrorLogger_agent.setLevel(self.log_level)
- _ErrorCh = logging.StreamHandler(sys.stderr)
- _ErrorCh.setFormatter(_ErrorFormatter)
- self._ErrorLogger_agent.addHandler(_ErrorCh)
-
- if self.log_file:
- if os.path.exists(self.log_file):
- os.remove(self.log_file)
- _Ch = logging.FileHandler(self.log_file)
- _Ch.setFormatter(_DebugFormatter)
- self._DebugLogger_agent.addHandler(_Ch)
-
- _Ch= logging.FileHandler(self.log_file)
- _Ch.setFormatter(_InfoFormatter)
- self._InfoLogger_agent.addHandler(_Ch)
-
- _Ch = logging.FileHandler(self.log_file)
- _Ch.setFormatter(_ErrorFormatter)
- self._ErrorLogger_agent.addHandler(_Ch)
-
- def run(self):
- self.InitLogger()
- while True:
- log_message = self.log_q.get()
- if log_message is None:
- break
- if log_message.name == "tool_error":
- self._ErrorLogger_agent.log(log_message.levelno,log_message.getMessage())
- elif log_message.name == "tool_info":
- self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())
- elif log_message.name == "tool_debug":
- self._DebugLogger_agent.log(log_message.levelno,log_message.getMessage())
- else:
- self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage())
-
- def kill(self):
- self.log_q.put(None)
-class AutoGenManager(threading.Thread):
- def __init__(self,autogen_workers, feedback_q,error_event):
- super(AutoGenManager,self).__init__()
- self.autogen_workers = autogen_workers
- self.feedback_q = feedback_q
- self.Status = True
- self.error_event = error_event
- def run(self):
- try:
- fin_num = 0
- while True:
- badnews = self.feedback_q.get()
- if badnews is None:
- break
- if badnews == "Done":
- fin_num += 1
- elif badnews == "QueueEmpty":
- EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews))
- self.TerminateWorkers()
- else:
- EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews))
- self.Status = False
- self.TerminateWorkers()
- if fin_num == len(self.autogen_workers):
- self.clearQueue()
- for w in self.autogen_workers:
- w.join()
- break
- except Exception:
- return
-
- def clearQueue(self):
- taskq = self.autogen_workers[0].module_queue
- logq = self.autogen_workers[0].log_q
- clearQ(taskq)
- clearQ(self.feedback_q)
- clearQ(logq)
- # Copy the cache queue itmes to parent thread before clear
- cacheq = self.autogen_workers[0].cache_q
- try:
- cache_num = 0
- while True:
- item = cacheq.get()
- if item == "CacheDone":
- cache_num += 1
- else:
- GlobalData.gModuleAllCacheStatus.add(item)
- if cache_num == len(self.autogen_workers):
- break
- except:
- print ("cache_q error")
-
- def TerminateWorkers(self):
- self.error_event.set()
- def kill(self):
- self.feedback_q.put(None)
-class AutoGenWorkerInProcess(mp.Process):
- def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_q,log_q,error_event):
- mp.Process.__init__(self)
- self.module_queue = module_queue
- self.data_pipe_file_path =data_pipe_file_path
- self.data_pipe = None
- self.feedback_q = feedback_q
- self.PlatformMetaFileSet = {}
- self.file_lock = file_lock
- self.cache_q = cache_q
- self.log_q = log_q
- self.error_event = error_event
- def GetPlatformMetaFile(self,filepath,root):
- try:
- return self.PlatformMetaFileSet[(filepath,root)]
- except:
- self.PlatformMetaFileSet[(filepath,root)] = filepath
- return self.PlatformMetaFileSet[(filepath,root)]
- def run(self):
- try:
- taskname = "Init"
- with self.file_lock:
- try:
- self.data_pipe = MemoryDataPipe()
- self.data_pipe.load(self.data_pipe_file_path)
- except:
- self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path)
- EdkLogger.LogClientInitialize(self.log_q)
- loglevel = self.data_pipe.Get("LogLevel")
- if not loglevel:
- loglevel = EdkLogger.INFO
- EdkLogger.SetLevel(loglevel)
- target = self.data_pipe.Get("P_Info").get("Target")
- toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
- archlist = self.data_pipe.Get("P_Info").get("ArchList")
-
- active_p = self.data_pipe.Get("P_Info").get("ActivePlatform")
- workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir")
- PackagesPath = os.getenv("PACKAGES_PATH")
- mws.setWs(workspacedir, PackagesPath)
- self.Wa = WorkSpaceInfo(
- workspacedir,active_p,target,toolchain,archlist
- )
- self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp")
- GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines")
- GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines")
- GlobalData.gCommandMaxLength = self.data_pipe.Get('gCommandMaxLength')
- os.environ._data = self.data_pipe.Get("Env_Var")
- GlobalData.gWorkspace = workspacedir
- GlobalData.gDisableIncludePathCheck = False
- GlobalData.gFdfParser = self.data_pipe.Get("FdfParser")
- GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath")
-
- GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache")
- GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource")
- GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest")
- GlobalData.gPlatformHashFile = self.data_pipe.Get("PlatformHashFile")
- GlobalData.gModulePreMakeCacheStatus = dict()
- GlobalData.gModuleMakeCacheStatus = dict()
- GlobalData.gHashChainStatus = dict()
- GlobalData.gCMakeHashFile = dict()
- GlobalData.gModuleHashFile = dict()
- GlobalData.gFileHashDict = dict()
- GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread")
- GlobalData.gPlatformFinalPcds = self.data_pipe.Get("gPlatformFinalPcds")
- GlobalData.file_lock = self.file_lock
- CommandTarget = self.data_pipe.Get("CommandTarget")
- pcd_from_build_option = []
- for pcd_tuple in self.data_pipe.Get("BuildOptPcd"):
- pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1]))
- if pcd_tuple[2].strip():
- pcd_id = ".".join((pcd_id,pcd_tuple[2]))
- pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3])))
- GlobalData.BuildOptionPcd = pcd_from_build_option
- module_count = 0
- FfsCmd = self.data_pipe.Get("FfsCommand")
- if FfsCmd is None:
- FfsCmd = {}
- GlobalData.FfsCmd = FfsCmd
- PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"),
- self.data_pipe.Get("P_Info").get("WorkspaceDir"))
- while True:
- if self.error_event.is_set():
- break
- module_count += 1
- try:
- module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait()
- except Empty:
- EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Fake Empty."))
- time.sleep(0.01)
- continue
- if module_file is None:
- EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Worker get the last item in the queue."))
- self.feedback_q.put("QueueEmpty")
- time.sleep(0.01)
- continue
-
- modulefullpath = os.path.join(module_root,module_file)
- taskname = " : ".join((modulefullpath,module_arch))
- module_metafile = PathClass(module_file,module_root)
- if module_path:
- module_metafile.Path = module_path
- if module_basename:
- module_metafile.BaseName = module_basename
- if module_originalpath:
- module_metafile.OriginalPath = PathClass(module_originalpath,module_root)
- arch = module_arch
- target = self.data_pipe.Get("P_Info").get("Target")
- toolchain = self.data_pipe.Get("P_Info").get("ToolChain")
- Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe)
- Ma.IsLibrary = IsLib
- # SourceFileList calling sequence impact the makefile string sequence.
- # Create cached SourceFileList here to unify its calling sequence for both
- # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile.
- RetVal = Ma.SourceFileList
- if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [None, "", "all"]:
- try:
- CacheResult = Ma.CanSkipbyPreMakeCache()
- except:
- CacheResult = False
- self.feedback_q.put(taskname)
-
- if CacheResult:
- self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True))
- continue
- else:
- self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False))
-
- Ma.CreateCodeFile(False)
- Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[]))
- Ma.CreateAsBuiltInf()
- if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]:
- try:
- CacheResult = Ma.CanSkipbyMakeCache()
- except:
- CacheResult = False
- self.feedback_q.put(taskname)
-
- if CacheResult:
- self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True))
- continue
- else:
- self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False))
-
- except Exception as e:
- EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), str(e)))
- self.feedback_q.put(taskname)
- finally:
- EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Done"))
- self.feedback_q.put("Done")
- self.cache_q.put("CacheDone")
-
- def printStatus(self):
- print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache())))
- print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache())))
- groupobj = {}
- for buildobj in BuildDB.BuildObject.GetCache().values():
- if str(buildobj).lower().endswith("dec"):
- try:
- groupobj['dec'].append(str(buildobj))
- except:
- groupobj['dec'] = [str(buildobj)]
- if str(buildobj).lower().endswith("dsc"):
- try:
- groupobj['dsc'].append(str(buildobj))
- except:
- groupobj['dsc'] = [str(buildobj)]
-
- if str(buildobj).lower().endswith("inf"):
- try:
- groupobj['inf'].append(str(buildobj))
- except:
- groupobj['inf'] = [str(buildobj)]
-
- print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[]))))
- print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[]))))
- print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[]))))
+## @file +# Create makefile for MS nmake and GNU make +# +# Copyright (c) 2019, Intel Corporation. All rights reserved.<BR> +# SPDX-License-Identifier: BSD-2-Clause-Patent +# +from __future__ import absolute_import +import multiprocessing as mp +import threading +from Common.Misc import PathClass +from AutoGen.ModuleAutoGen import ModuleAutoGen +from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo +import Common.GlobalData as GlobalData +import Common.EdkLogger as EdkLogger +import os +from Common.MultipleWorkspace import MultipleWorkspace as mws +from AutoGen.AutoGen import AutoGen +from Workspace.WorkspaceDatabase import BuildDB +try: + from queue import Empty +except: + from Queue import Empty +import traceback +import sys +from AutoGen.DataPipe import MemoryDataPipe +import logging +import time + +def clearQ(q): + try: + while True: + q.get_nowait() + except Empty: + pass + +class LogAgent(threading.Thread): + def __init__(self,log_q,log_level,log_file=None): + super(LogAgent,self).__init__() + self.log_q = log_q + self.log_level = log_level + self.log_file = log_file + def InitLogger(self): + # For DEBUG level (All DEBUG_0~9 are applicable) + self._DebugLogger_agent = logging.getLogger("tool_debug_agent") + _DebugFormatter = logging.Formatter("[%(asctime)s.%(msecs)d]: %(message)s", datefmt="%H:%M:%S") + self._DebugLogger_agent.setLevel(self.log_level) + _DebugChannel = logging.StreamHandler(sys.stdout) + _DebugChannel.setFormatter(_DebugFormatter) + self._DebugLogger_agent.addHandler(_DebugChannel) + + # For VERBOSE, INFO, WARN level + self._InfoLogger_agent = logging.getLogger("tool_info_agent") + _InfoFormatter = logging.Formatter("%(message)s") + self._InfoLogger_agent.setLevel(self.log_level) + _InfoChannel = logging.StreamHandler(sys.stdout) + _InfoChannel.setFormatter(_InfoFormatter) + self._InfoLogger_agent.addHandler(_InfoChannel) + + # For ERROR level + self._ErrorLogger_agent = logging.getLogger("tool_error_agent") + _ErrorFormatter = logging.Formatter("%(message)s") + self._ErrorLogger_agent.setLevel(self.log_level) + _ErrorCh = logging.StreamHandler(sys.stderr) + _ErrorCh.setFormatter(_ErrorFormatter) + self._ErrorLogger_agent.addHandler(_ErrorCh) + + if self.log_file: + if os.path.exists(self.log_file): + os.remove(self.log_file) + _Ch = logging.FileHandler(self.log_file) + _Ch.setFormatter(_DebugFormatter) + self._DebugLogger_agent.addHandler(_Ch) + + _Ch= logging.FileHandler(self.log_file) + _Ch.setFormatter(_InfoFormatter) + self._InfoLogger_agent.addHandler(_Ch) + + _Ch = logging.FileHandler(self.log_file) + _Ch.setFormatter(_ErrorFormatter) + self._ErrorLogger_agent.addHandler(_Ch) + + def run(self): + self.InitLogger() + while True: + log_message = self.log_q.get() + if log_message is None: + break + if log_message.name == "tool_error": + self._ErrorLogger_agent.log(log_message.levelno,log_message.getMessage()) + elif log_message.name == "tool_info": + self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage()) + elif log_message.name == "tool_debug": + self._DebugLogger_agent.log(log_message.levelno,log_message.getMessage()) + else: + self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage()) + + def kill(self): + self.log_q.put(None) +class AutoGenManager(threading.Thread): + def __init__(self,autogen_workers, feedback_q,error_event): + super(AutoGenManager,self).__init__() + self.autogen_workers = autogen_workers + self.feedback_q = feedback_q + self.Status = True + self.error_event = error_event + def run(self): + try: + fin_num = 0 + while True: + badnews = self.feedback_q.get() + if badnews is None: + break + if badnews == "Done": + fin_num += 1 + elif badnews == "QueueEmpty": + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews)) + self.TerminateWorkers() + else: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews)) + self.Status = False + self.TerminateWorkers() + if fin_num == len(self.autogen_workers): + self.clearQueue() + for w in self.autogen_workers: + w.join() + break + except Exception: + return + + def clearQueue(self): + taskq = self.autogen_workers[0].module_queue + logq = self.autogen_workers[0].log_q + clearQ(taskq) + clearQ(self.feedback_q) + clearQ(logq) + # Copy the cache queue itmes to parent thread before clear + cacheq = self.autogen_workers[0].cache_q + try: + cache_num = 0 + while True: + item = cacheq.get() + if item == "CacheDone": + cache_num += 1 + else: + GlobalData.gModuleAllCacheStatus.add(item) + if cache_num == len(self.autogen_workers): + break + except: + print ("cache_q error") + + def TerminateWorkers(self): + self.error_event.set() + def kill(self): + self.feedback_q.put(None) +class AutoGenWorkerInProcess(mp.Process): + def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_q,log_q,error_event): + mp.Process.__init__(self) + self.module_queue = module_queue + self.data_pipe_file_path =data_pipe_file_path + self.data_pipe = None + self.feedback_q = feedback_q + self.PlatformMetaFileSet = {} + self.file_lock = file_lock + self.cache_q = cache_q + self.log_q = log_q + self.error_event = error_event + def GetPlatformMetaFile(self,filepath,root): + try: + return self.PlatformMetaFileSet[(filepath,root)] + except: + self.PlatformMetaFileSet[(filepath,root)] = filepath + return self.PlatformMetaFileSet[(filepath,root)] + def run(self): + try: + taskname = "Init" + with self.file_lock: + try: + self.data_pipe = MemoryDataPipe() + self.data_pipe.load(self.data_pipe_file_path) + except: + self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path) + EdkLogger.LogClientInitialize(self.log_q) + loglevel = self.data_pipe.Get("LogLevel") + if not loglevel: + loglevel = EdkLogger.INFO + EdkLogger.SetLevel(loglevel) + target = self.data_pipe.Get("P_Info").get("Target") + toolchain = self.data_pipe.Get("P_Info").get("ToolChain") + archlist = self.data_pipe.Get("P_Info").get("ArchList") + + active_p = self.data_pipe.Get("P_Info").get("ActivePlatform") + workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir") + PackagesPath = os.getenv("PACKAGES_PATH") + mws.setWs(workspacedir, PackagesPath) + self.Wa = WorkSpaceInfo( + workspacedir,active_p,target,toolchain,archlist + ) + self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp") + GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines") + GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines") + GlobalData.gCommandMaxLength = self.data_pipe.Get('gCommandMaxLength') + os.environ._data = self.data_pipe.Get("Env_Var") + GlobalData.gWorkspace = workspacedir + GlobalData.gDisableIncludePathCheck = False + GlobalData.gFdfParser = self.data_pipe.Get("FdfParser") + GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath") + + GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache") + GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource") + GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest") + GlobalData.gPlatformHashFile = self.data_pipe.Get("PlatformHashFile") + GlobalData.gModulePreMakeCacheStatus = dict() + GlobalData.gModuleMakeCacheStatus = dict() + GlobalData.gHashChainStatus = dict() + GlobalData.gCMakeHashFile = dict() + GlobalData.gModuleHashFile = dict() + GlobalData.gFileHashDict = dict() + GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread") + GlobalData.gPlatformFinalPcds = self.data_pipe.Get("gPlatformFinalPcds") + GlobalData.file_lock = self.file_lock + CommandTarget = self.data_pipe.Get("CommandTarget") + pcd_from_build_option = [] + for pcd_tuple in self.data_pipe.Get("BuildOptPcd"): + pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1])) + if pcd_tuple[2].strip(): + pcd_id = ".".join((pcd_id,pcd_tuple[2])) + pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3]))) + GlobalData.BuildOptionPcd = pcd_from_build_option + module_count = 0 + FfsCmd = self.data_pipe.Get("FfsCommand") + if FfsCmd is None: + FfsCmd = {} + GlobalData.FfsCmd = FfsCmd + PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"), + self.data_pipe.Get("P_Info").get("WorkspaceDir")) + while True: + if self.error_event.is_set(): + break + module_count += 1 + try: + module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait() + except Empty: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Fake Empty.")) + time.sleep(0.01) + continue + if module_file is None: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Worker get the last item in the queue.")) + self.feedback_q.put("QueueEmpty") + time.sleep(0.01) + continue + + modulefullpath = os.path.join(module_root,module_file) + taskname = " : ".join((modulefullpath,module_arch)) + module_metafile = PathClass(module_file,module_root) + if module_path: + module_metafile.Path = module_path + if module_basename: + module_metafile.BaseName = module_basename + if module_originalpath: + module_metafile.OriginalPath = PathClass(module_originalpath,module_root) + arch = module_arch + target = self.data_pipe.Get("P_Info").get("Target") + toolchain = self.data_pipe.Get("P_Info").get("ToolChain") + Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe) + Ma.IsLibrary = IsLib + # SourceFileList calling sequence impact the makefile string sequence. + # Create cached SourceFileList here to unify its calling sequence for both + # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile. + RetVal = Ma.SourceFileList + if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [None, "", "all"]: + try: + CacheResult = Ma.CanSkipbyPreMakeCache() + except: + CacheResult = False + self.feedback_q.put(taskname) + + if CacheResult: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True)) + continue + else: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False)) + + Ma.CreateCodeFile(False) + Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[])) + Ma.CreateAsBuiltInf() + if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: + try: + CacheResult = Ma.CanSkipbyMakeCache() + except: + CacheResult = False + self.feedback_q.put(taskname) + + if CacheResult: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True)) + continue + else: + self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False)) + + except Exception as e: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), str(e))) + self.feedback_q.put(taskname) + finally: + EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Done")) + self.feedback_q.put("Done") + self.cache_q.put("CacheDone") + + def printStatus(self): + print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache()))) + print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache()))) + groupobj = {} + for buildobj in BuildDB.BuildObject.GetCache().values(): + if str(buildobj).lower().endswith("dec"): + try: + groupobj['dec'].append(str(buildobj)) + except: + groupobj['dec'] = [str(buildobj)] + if str(buildobj).lower().endswith("dsc"): + try: + groupobj['dsc'].append(str(buildobj)) + except: + groupobj['dsc'] = [str(buildobj)] + + if str(buildobj).lower().endswith("inf"): + try: + groupobj['inf'].append(str(buildobj)) + except: + groupobj['inf'] = [str(buildobj)] + + print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[])))) + print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[])))) + print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[])))) |