From 1699d3efe111e33e275ca7d4163c8b1470ba79b3 Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Fri, 21 Jul 2023 23:22:28 +0530 Subject: perf scripts python: Add initial script file with usage information Added necessary modules, including the Perf-Trace-Util library, and defines the required functions and variables for using perf script python. The perf_trace_context and Core modules for tracing and processing events has been also imported. Added usage information. Signed-off-by: Anup Sharma Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/f2f1a62f1cc69f44a5414da46a26a4cf124d2744.1689961706.git.anupnewsmail@gmail.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/gecko.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tools/perf/scripts/python/gecko.py (limited to 'tools/perf/scripts/python') diff --git a/tools/perf/scripts/python/gecko.py b/tools/perf/scripts/python/gecko.py new file mode 100644 index 000000000000..7a62c1b411d9 --- /dev/null +++ b/tools/perf/scripts/python/gecko.py @@ -0,0 +1,31 @@ +# firefox-gecko-converter.py - Convert perf record output to Firefox's gecko profile format +# SPDX-License-Identifier: GPL-2.0 +# +# The script converts perf.data to Gecko Profile Format, +# which can be read by https://profiler.firefox.com/. +# +# Usage: +# +# perf record -a -g -F 99 sleep 60 +# perf script report gecko > output.json + +import os +import sys +from typing import Dict + +# Add the Perf-Trace-Util library to the Python path +sys.path.append(os.environ['PERF_EXEC_PATH'] + \ + '/scripts/python/Perf-Trace-Util/lib/Perf/Trace') + +from perf_trace_context import * +from Core import * + +# Uses perf script python interface to parse each +# event and store the data in the thread builder. +def process_event(param_dict: Dict) -> None: + pass + +# Trace_end runs at the end and will be used to aggregate +# the data into the final json object and print it out to stdout. +def trace_end() -> None: + pass -- cgit v1.2.3 From 0a02e44cc2fe1657af1f2740cb9a1dcd8a9338cc Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Fri, 21 Jul 2023 23:22:56 +0530 Subject: perf scripts python: Extact necessary information from process event The script takes in a sample event dictionary(param_dict) and retrieves relevant data such as time stamp, PID, TID, and comm for each event. Also start time is defined as a global variable as it need to be passed to trace_end for gecko meta information field creation. Signed-off-by: Anup Sharma Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/19910fefcfe4be03cd5c2aa3fec11d3f86c0381b.1689961706.git.anupnewsmail@gmail.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/gecko.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'tools/perf/scripts/python') diff --git a/tools/perf/scripts/python/gecko.py b/tools/perf/scripts/python/gecko.py index 7a62c1b411d9..a02b1e04ff52 100644 --- a/tools/perf/scripts/python/gecko.py +++ b/tools/perf/scripts/python/gecko.py @@ -20,10 +20,22 @@ sys.path.append(os.environ['PERF_EXEC_PATH'] + \ from perf_trace_context import * from Core import * +# start_time is intialiazed only once for the all event traces. +start_time = None + # Uses perf script python interface to parse each # event and store the data in the thread builder. def process_event(param_dict: Dict) -> None: - pass + global start_time + global tid_to_thread + time_stamp = (param_dict['sample']['time'] // 1000) / 1000 + pid = param_dict['sample']['pid'] + tid = param_dict['sample']['tid'] + comm = param_dict['comm'] + + # Start time is the time of the first sample + if not start_time: + start_time = time_stamp # Trace_end runs at the end and will be used to aggregate # the data into the final json object and print it out to stdout. -- cgit v1.2.3 From 5aacd7f08a3276f4fad729a600d51a1cc5d5191a Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Fri, 21 Jul 2023 23:23:19 +0530 Subject: perf scripts python: Add classes and conversion functions This commit introduces new classes and conversion functions to facilitate the representation of Gecko profile information. The new classes Frame, Stack, Sample, and Thread are added to handle specific components of the profile data, also link to the origin docs has been commented out. Additionally, Inside the Thread class _to_json_dict() method has been created that converts the current thread data into the corresponding format expected by the GeckoThread JSON schema, as per the Gecko profile format specification. Signed-off-by: Anup Sharma Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/ab7b40bd32df7101a6f8b4a3aa41570b63b831ac.1689961706.git.anupnewsmail@gmail.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/gecko.py | 133 ++++++++++++++++++++++++++++++++++++- 1 file changed, 132 insertions(+), 1 deletion(-) (limited to 'tools/perf/scripts/python') diff --git a/tools/perf/scripts/python/gecko.py b/tools/perf/scripts/python/gecko.py index a02b1e04ff52..2cd5cd6e31e3 100644 --- a/tools/perf/scripts/python/gecko.py +++ b/tools/perf/scripts/python/gecko.py @@ -11,7 +11,8 @@ import os import sys -from typing import Dict +from dataclasses import dataclass, field +from typing import List, Dict, Optional, NamedTuple, Set, Tuple, Any # Add the Perf-Trace-Util library to the Python path sys.path.append(os.environ['PERF_EXEC_PATH'] + \ @@ -20,9 +21,139 @@ sys.path.append(os.environ['PERF_EXEC_PATH'] + \ from perf_trace_context import * from Core import * +StringID = int +StackID = int +FrameID = int +CategoryID = int +Milliseconds = float + # start_time is intialiazed only once for the all event traces. start_time = None +# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 +class Frame(NamedTuple): + string_id: StringID + relevantForJS: bool + innerWindowID: int + implementation: None + optimizations: None + line: None + column: None + category: CategoryID + subcategory: int + +# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216 +class Stack(NamedTuple): + prefix_id: Optional[StackID] + frame_id: FrameID + +# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90 +class Sample(NamedTuple): + stack_id: Optional[StackID] + time_ms: Milliseconds + responsiveness: int + +@dataclass +class Thread: + """A builder for a profile of the thread. + + Attributes: + comm: Thread command-line (name). + pid: process ID of containing process. + tid: thread ID. + samples: Timeline of profile samples. + frameTable: interned stack frame ID -> stack frame. + stringTable: interned string ID -> string. + stringMap: interned string -> string ID. + stackTable: interned stack ID -> stack. + stackMap: (stack prefix ID, leaf stack frame ID) -> interned Stack ID. + frameMap: Stack Frame string -> interned Frame ID. + comm: str + pid: int + tid: int + samples: List[Sample] = field(default_factory=list) + frameTable: List[Frame] = field(default_factory=list) + stringTable: List[str] = field(default_factory=list) + stringMap: Dict[str, int] = field(default_factory=dict) + stackTable: List[Stack] = field(default_factory=list) + stackMap: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict) + frameMap: Dict[str, int] = field(default_factory=dict) + """ + comm: str + pid: int + tid: int + samples: List[Sample] = field(default_factory=list) + frameTable: List[Frame] = field(default_factory=list) + stringTable: List[str] = field(default_factory=list) + stringMap: Dict[str, int] = field(default_factory=dict) + stackTable: List[Stack] = field(default_factory=list) + stackMap: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict) + frameMap: Dict[str, int] = field(default_factory=dict) + + def _to_json_dict(self) -> Dict: + """Converts current Thread to GeckoThread JSON format.""" + # Gecko profile format is row-oriented data as List[List], + # And a schema for interpreting each index. + # Schema: + # https://github.com/firefox-devtools/profiler/blob/main/docs-developer/gecko-profile-format.md + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L230 + return { + "tid": self.tid, + "pid": self.pid, + "name": self.comm, + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L51 + "markers": { + "schema": { + "name": 0, + "startTime": 1, + "endTime": 2, + "phase": 3, + "category": 4, + "data": 5, + }, + "data": [], + }, + + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90 + "samples": { + "schema": { + "stack": 0, + "time": 1, + "responsiveness": 2, + }, + "data": self.samples + }, + + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 + "frameTable": { + "schema": { + "location": 0, + "relevantForJS": 1, + "innerWindowID": 2, + "implementation": 3, + "optimizations": 4, + "line": 5, + "column": 6, + "category": 7, + "subcategory": 8, + }, + "data": self.frameTable, + }, + + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216 + "stackTable": { + "schema": { + "prefix": 0, + "frame": 1, + }, + "data": self.stackTable, + }, + "stringTable": self.stringTable, + "registerTime": 0, + "unregisterTime": None, + "processType": "default", + } + # Uses perf script python interface to parse each # event and store the data in the thread builder. def process_event(param_dict: Dict) -> None: -- cgit v1.2.3 From 833daec7e6cfda0d6c30a94c21b6706ff094cd45 Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Fri, 21 Jul 2023 23:24:42 +0530 Subject: perf scripts python: Add trace end processing and PRODUCT and CATEGORIES information The final output will now be presented in JSON format following the Gecko profile structure. Additionally, the inclusion of PRODUCT allows easy retrieval of header information for UI. Furthermore, CATEGORIES have been introduced to enable customization of kernel and user colors using input arguments. To facilitate this functionality, an argparse-based parser has been implemented. Note: The implementation of threads will be addressed in subsequent commits for now I have commented it out. Signed-off-by: Anup Sharma Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/fa6d027e4134c48e8a2ea45dd8f6b21e6a3418e4.1689961706.git.anupnewsmail@gmail.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/gecko.py | 65 +++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) (limited to 'tools/perf/scripts/python') diff --git a/tools/perf/scripts/python/gecko.py b/tools/perf/scripts/python/gecko.py index 2cd5cd6e31e3..794a91bec464 100644 --- a/tools/perf/scripts/python/gecko.py +++ b/tools/perf/scripts/python/gecko.py @@ -11,6 +11,8 @@ import os import sys +import json +import argparse from dataclasses import dataclass, field from typing import List, Dict, Optional, NamedTuple, Set, Tuple, Any @@ -30,6 +32,13 @@ Milliseconds = float # start_time is intialiazed only once for the all event traces. start_time = None +# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/profile.js#L425 +# Follow Brendan Gregg's Flamegraph convention: orange for kernel and yellow for user space by default. +CATEGORIES = None + +# The product name is used by the profiler UI to show the Operating system and Processor. +PRODUCT = os.popen('uname -op').read().strip() + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 class Frame(NamedTuple): string_id: StringID @@ -171,4 +180,58 @@ def process_event(param_dict: Dict) -> None: # Trace_end runs at the end and will be used to aggregate # the data into the final json object and print it out to stdout. def trace_end() -> None: - pass + # Schema: https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L305 + gecko_profile_with_meta = { + "meta": { + "interval": 1, + "processType": 0, + "product": PRODUCT, + "stackwalk": 1, + "debug": 0, + "gcpoison": 0, + "asyncstack": 1, + "startTime": start_time, + "shutdownTime": None, + "version": 24, + "presymbolicated": True, + "categories": CATEGORIES, + "markerSchema": [], + }, + "libs": [], + # threads will be implemented in later commits. + # "threads": threads, + "processes": [], + "pausedRanges": [], + } + json.dump(gecko_profile_with_meta, sys.stdout, indent=2) + +def main() -> None: + global CATEGORIES + parser = argparse.ArgumentParser(description="Convert perf.data to Firefox\'s Gecko Profile format") + + # Add the command-line options + # Colors must be defined according to this: + # https://github.com/firefox-devtools/profiler/blob/50124adbfa488adba6e2674a8f2618cf34b59cd2/res/css/categories.css + parser.add_argument('--user-color', default='yellow', help='Color for the User category') + parser.add_argument('--kernel-color', default='orange', help='Color for the Kernel category') + # Parse the command-line arguments + args = parser.parse_args() + # Access the values provided by the user + user_color = args.user_color + kernel_color = args.kernel_color + + CATEGORIES = [ + { + "name": 'User', + "color": user_color, + "subcategories": ['Other'] + }, + { + "name": 'Kernel', + "color": kernel_color, + "subcategories": ['Other'] + }, + ] + +if __name__ == '__main__': + main() -- cgit v1.2.3 From 258dfd41c1df2030772ceab378f7bd6f0f78b938 Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Fri, 21 Jul 2023 23:25:38 +0530 Subject: perf scripts python: Implement add sample function and thread processing The intern_stack function is responsible for retrieving or creating a stack_id based on the provided frame_id and prefix_id. It first generates a key using the frame_id and prefix_id values. If the stack corresponding to the key is found in the stackMap, it is returned. Otherwise, a new stack is created by appending the prefix_id and frame_id to the stackTable. The key and the index of the newly created stack are added to the stackMap for future reference. The _intern_frame function is responsible for retrieving or creating a frame_id based on the provided frame string. If the frame_id corresponding to the frameString is found in the frameMap, it is returned. Otherwise, a new frame is created by appending relevant information to the frameTable and adding the frameString to the string_id through _intern_string. The _intern_string function will gets a matching string, or saves the new string and returns a String ID. Signed-off-by: Anup Sharma Link: https://lore.kernel.org/r/4442f4b1ab4c7317cf940560a3a285fcdfbeeb08.1689961706.git.anupnewsmail@gmail.com Cc: Mark Rutland Cc: Ian Rogers Cc: Peter Zijlstra Cc: Adrian Hunter Cc: Arnaldo Carvalho de Melo Cc: Jiri Olsa Cc: Namhyung Kim Cc: Alexander Shishkin Cc: Ingo Molnar Cc: linux-kernel@vger.kernel.org Cc: linux-perf-users@vger.kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/gecko.py | 54 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) (limited to 'tools/perf/scripts/python') diff --git a/tools/perf/scripts/python/gecko.py b/tools/perf/scripts/python/gecko.py index 794a91bec464..97949249a3c0 100644 --- a/tools/perf/scripts/python/gecko.py +++ b/tools/perf/scripts/python/gecko.py @@ -13,6 +13,7 @@ import os import sys import json import argparse +from functools import reduce from dataclasses import dataclass, field from typing import List, Dict, Optional, NamedTuple, Set, Tuple, Any @@ -39,6 +40,10 @@ CATEGORIES = None # The product name is used by the profiler UI to show the Operating system and Processor. PRODUCT = os.popen('uname -op').read().strip() +# The category index is used by the profiler UI to show the color of the flame graph. +USER_CATEGORY_INDEX = 0 +KERNEL_CATEGORY_INDEX = 1 + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 class Frame(NamedTuple): string_id: StringID @@ -99,6 +104,55 @@ class Thread: stackMap: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict) frameMap: Dict[str, int] = field(default_factory=dict) + def _intern_stack(self, frame_id: int, prefix_id: Optional[int]) -> int: + """Gets a matching stack, or saves the new stack. Returns a Stack ID.""" + key = f"{frame_id}" if prefix_id is None else f"{frame_id},{prefix_id}" + # key = (prefix_id, frame_id) + stack_id = self.stackMap.get(key) + if stack_id is None: + # return stack_id + stack_id = len(self.stackTable) + self.stackTable.append(Stack(prefix_id=prefix_id, frame_id=frame_id)) + self.stackMap[key] = stack_id + return stack_id + + def _intern_string(self, string: str) -> int: + """Gets a matching string, or saves the new string. Returns a String ID.""" + string_id = self.stringMap.get(string) + if string_id is not None: + return string_id + string_id = len(self.stringTable) + self.stringTable.append(string) + self.stringMap[string] = string_id + return string_id + + def _intern_frame(self, frame_str: str) -> int: + """Gets a matching stack frame, or saves the new frame. Returns a Frame ID.""" + frame_id = self.frameMap.get(frame_str) + if frame_id is not None: + return frame_id + frame_id = len(self.frameTable) + self.frameMap[frame_str] = frame_id + string_id = self._intern_string(frame_str) + + symbol_name_to_category = KERNEL_CATEGORY_INDEX if frame_str.find('kallsyms') != -1 \ + or frame_str.find('/vmlinux') != -1 \ + or frame_str.endswith('.ko)') \ + else USER_CATEGORY_INDEX + + self.frameTable.append(Frame( + string_id=string_id, + relevantForJS=False, + innerWindowID=0, + implementation=None, + optimizations=None, + line=None, + column=None, + category=symbol_name_to_category, + subcategory=None, + )) + return frame_id + def _to_json_dict(self) -> Dict: """Converts current Thread to GeckoThread JSON format.""" # Gecko profile format is row-oriented data as List[List], -- cgit v1.2.3 From 2d889c6af1cc125380b03b6efdaed0a4b4611aed Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Fri, 21 Jul 2023 23:26:24 +0530 Subject: perf scripts python: Implement add sample function and thread processing The stack has been created for storing func and dso from the callchain. The sample has been added to a specific thread. It first checks if the thread exists in the Thread class. Then it call _add_sample function which is responsible for appending a new entry to the samples list. Also callchain parsing and storing part is implemented. Moreover removed the comment from thread. Signed-off-by: Anup Sharma Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/5a112be85ccdcdcd611e343f6a7a7482d01f6299.1689961706.git.anupnewsmail@gmail.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/gecko.py | 52 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) (limited to 'tools/perf/scripts/python') diff --git a/tools/perf/scripts/python/gecko.py b/tools/perf/scripts/python/gecko.py index 97949249a3c0..278c3aed282a 100644 --- a/tools/perf/scripts/python/gecko.py +++ b/tools/perf/scripts/python/gecko.py @@ -40,6 +40,9 @@ CATEGORIES = None # The product name is used by the profiler UI to show the Operating system and Processor. PRODUCT = os.popen('uname -op').read().strip() +# Here key = tid, value = Thread +tid_to_thread = dict() + # The category index is used by the profiler UI to show the color of the flame graph. USER_CATEGORY_INDEX = 0 KERNEL_CATEGORY_INDEX = 1 @@ -153,6 +156,25 @@ class Thread: )) return frame_id + def _add_sample(self, comm: str, stack: List[str], time_ms: Milliseconds) -> None: + """Add a timestamped stack trace sample to the thread builder. + Args: + comm: command-line (name) of the thread at this sample + stack: sampled stack frames. Root first, leaf last. + time_ms: timestamp of sample in milliseconds. + """ + # Ihreads may not set their names right after they are created. + # Instead, they might do it later. In such situations, to use the latest name they have set. + if self.comm != comm: + self.comm = comm + + prefix_stack_id = reduce(lambda prefix_id, frame: self._intern_stack + (self._intern_frame(frame), prefix_id), stack, None) + if prefix_stack_id is not None: + self.samples.append(Sample(stack_id=prefix_stack_id, + time_ms=time_ms, + responsiveness=0)) + def _to_json_dict(self) -> Dict: """Converts current Thread to GeckoThread JSON format.""" # Gecko profile format is row-oriented data as List[List], @@ -231,9 +253,36 @@ def process_event(param_dict: Dict) -> None: if not start_time: start_time = time_stamp + # Parse and append the callchain of the current sample into a stack. + stack = [] + if param_dict['callchain']: + for call in param_dict['callchain']: + if 'sym' not in call: + continue + stack.append(f'{call["sym"]["name"]} (in {call["dso"]})') + if len(stack) != 0: + # Reverse the stack, as root come first and the leaf at the end. + stack = stack[::-1] + + # During perf record if -g is not used, the callchain is not available. + # In that case, the symbol and dso are available in the event parameters. + else: + func = param_dict['symbol'] if 'symbol' in param_dict else '[unknown]' + dso = param_dict['dso'] if 'dso' in param_dict else '[unknown]' + stack.append(f'{func} (in {dso})') + + # Add sample to the specific thread. + thread = tid_to_thread.get(tid) + if thread is None: + thread = Thread(comm=comm, pid=pid, tid=tid) + tid_to_thread[tid] = thread + thread._add_sample(comm=comm, stack=stack, time_ms=time_stamp) + # Trace_end runs at the end and will be used to aggregate # the data into the final json object and print it out to stdout. def trace_end() -> None: + threads = [thread._to_json_dict() for thread in tid_to_thread.values()] + # Schema: https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L305 gecko_profile_with_meta = { "meta": { @@ -252,8 +301,7 @@ def trace_end() -> None: "markerSchema": [], }, "libs": [], - # threads will be implemented in later commits. - # "threads": threads, + "threads": threads, "processes": [], "pausedRanges": [], } -- cgit v1.2.3 From f9f72b2ab77e986ac30de09a735a002b37d81503 Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Fri, 21 Jul 2023 23:27:46 +0530 Subject: perf scripts python: Add command execution for gecko script This will enable the execution of gecko.py script using record and report commands in 'perf script'. And this will be also reflected at "perf script -l" command. For Example: perf script record gecko perf script report gecko Committer notes: As discussed on the perf tools office hours, I made -F 99 the default for the record script and removed the double -- on the report script so that the existing 'perf script' protocol for the combined operation: # perf script gecko Works, i.e. the record script pipes its stdout into the stdin of the report script, basically: /bin/sh /usr/libexec/perf-core/scripts/python/bin/gecko-record -F 99 -g -a -q -o - | \ /bin/sh /usr/libexec/perf-core/scripts/python/bin/gecko-report -i - Testing it: The resulting JSON file needs to be uploaded to https://profiler.firefox.com, Anup already has code to start a local http server on the trace_begin handler of the gecko python script, start firefox and feed it the JSON. The example below only collects sample for the specified workload, so that we don't produce thousands of lines, to collect system wide samples, use instead: # perf script gecko -a sleep 0.5 # nohup perf script gecko sleep 0.5 { "meta": { "interval": 1, "processType": 0, "product": "x86_64 GNU/Linux", "stackwalk": 1, "debug": 0, "gcpoison": 0, "asyncstack": 1, "startTime": 274601692.636, "shutdownTime": null, "version": 24, "presymbolicated": true, "categories": [ { "name": "User", "color": "yellow", "subcategories": [ "Other" ] }, { "name": "Kernel", "color": "orange", "subcategories": [ "Other" ] } ], "markerSchema": [] }, "libs": [], "threads": [ { "tid": 3344498, "pid": 3344498, "name": "sleep", "markers": { "schema": { "name": 0, "startTime": 1, "endTime": 2, "phase": 3, "category": 4, "data": 5 }, "data": [] }, "samples": { "schema": { "stack": 0, "time": 1, "responsiveness": 2 }, "data": [ [ 21, 274601692.636, 0 ], [ 23, 274601692.641, 0 ], [ 29, 274601692.643, 0 ], [ 42, 274601692.648, 0 ] ] }, "frameTable": { "schema": { "location": 0, "relevantForJS": 1, "innerWindowID": 2, "implementation": 3, "optimizations": 4, "line": 5, "column": 6, "category": 7, "subcategory": 8 }, "data": [ [ 0, false, 0, null, null, null, null, 1, null ], [ 1, false, 0, null, null, null, null, 1, null ], [ 2, false, 0, null, null, null, null, 1, null ], [ 3, false, 0, null, null, null, null, 1, null ], [ 4, false, 0, null, null, null, null, 1, null ], [ 5, false, 0, null, null, null, null, 1, null ], [ 6, false, 0, null, null, null, null, 1, null ], [ 7, false, 0, null, null, null, null, 1, null ], [ 8, false, 0, null, null, null, null, 1, null ], [ 9, false, 0, null, null, null, null, 1, null ], [ 10, false, 0, null, null, null, null, 1, null ], [ 11, false, 0, null, null, null, null, 1, null ], [ 12, false, 0, null, null, null, null, 1, null ], [ 13, false, 0, null, null, null, null, 1, null ], [ 14, false, 0, null, null, null, null, 1, null ], [ 15, false, 0, null, null, null, null, 1, null ], [ 16, false, 0, null, null, null, null, 1, null ], [ 17, false, 0, null, null, null, null, 1, null ], [ 18, false, 0, null, null, null, null, 1, null ], [ 19, false, 0, null, null, null, null, 1, null ], [ 20, false, 0, null, null, null, null, 1, null ], [ 21, false, 0, null, null, null, null, 1, null ], [ 22, false, 0, null, null, null, null, 1, null ], [ 23, false, 0, null, null, null, null, 1, null ], [ 24, false, 0, null, null, null, null, 1, null ], [ 25, false, 0, null, null, null, null, 1, null ], [ 26, false, 0, null, null, null, null, 1, null ], [ 27, false, 0, null, null, null, null, 1, null ], [ 28, false, 0, null, null, null, null, 1, null ], [ 29, false, 0, null, null, null, null, 1, null ], [ 30, false, 0, null, null, null, null, 1, null ], [ 31, false, 0, null, null, null, null, 1, null ], [ 32, false, 0, null, null, null, null, 1, null ], [ 33, false, 0, null, null, null, null, 1, null ], [ 34, false, 0, null, null, null, null, 1, null ], [ 35, false, 0, null, null, null, null, 1, null ], [ 36, false, 0, null, null, null, null, 1, null ], [ 37, false, 0, null, null, null, null, 1, null ], [ 38, false, 0, null, null, null, null, 1, null ] ] }, "stackTable": { "schema": { "prefix": 0, "frame": 1 }, "data": [ [ null, 0 ], [ 0, 1 ], [ 1, 2 ], [ 2, 3 ], [ 3, 4 ], [ 4, 5 ], [ 5, 6 ], [ 6, 7 ], [ 7, 8 ], [ 8, 9 ], [ 9, 10 ], [ 10, 11 ], [ 11, 12 ], [ 12, 13 ], [ 13, 14 ], [ 14, 15 ], [ 15, 16 ], [ 16, 17 ], [ 17, 18 ], [ 18, 19 ], [ 19, 20 ], [ 20, 21 ], [ 20, 22 ], [ 22, 23 ], [ 11, 24 ], [ 24, 25 ], [ 25, 26 ], [ 26, 27 ], [ 27, 28 ], [ 28, 29 ], [ 9, 11 ], [ 30, 24 ], [ 31, 25 ], [ 32, 30 ], [ 33, 31 ], [ 34, 32 ], [ 35, 29 ], [ 36, 33 ], [ 37, 34 ], [ 38, 35 ], [ 39, 36 ], [ 40, 37 ], [ 41, 38 ] ] }, "stringTable": [ "__func__.0 (in [kernel.kallsyms].rodata)", "perf_trace_ext4_fc_track_inode (in [kernel.kallsyms])", "perf_trace_ext4_es_insert_delayed_block (in [kernel.kallsyms])", "ext4_es_show_pblock (in [kernel.kallsyms])", "perf_trace_ext4_ext_rm_leaf (in [kernel.kallsyms])", "devcgroup_access_write (in [kernel.kallsyms])", "devcgroup_update_access (in [kernel.kallsyms])", "propagate_exception (in [kernel.kallsyms])", "revalidate_active_exceptions (in [kernel.kallsyms])", "perf_trace_ext4_fc_commit_stop (in [kernel.kallsyms])", "perf_fetch_caller_regs (in [kernel.kallsyms])", "khugepaged (in [kernel.kallsyms])", "khugepaged_wait_work (in [kernel.kallsyms])", "freezable_schedule_timeout (in [kernel.kallsyms])", "freezer_count (in [kernel.kallsyms])", "try_to_freeze (in [kernel.kallsyms])", "try_to_freeze_unsafe (in [kernel.kallsyms])", "split_huge_pages_write (in [kernel.kallsyms])", "migrate_pages (in [kernel.kallsyms])", "unmap_and_move (in [kernel.kallsyms])", "__unmap_and_move (in [kernel.kallsyms])", "collect_events (in [kernel.kallsyms])", "uncore_down_prepare (in [kernel.kallsyms])", "perf_iommu_read (in [kernel.kallsyms])", "khugepaged_do_scan (in [kernel.kallsyms])", "khugepaged_scan_mm_slot (in [kernel.kallsyms])", "khugepaged_scan_file (in [kernel.kallsyms])", "need_resched (in [kernel.kallsyms])", "get_current (in [kernel.kallsyms])", "move_to_new_page (in [kernel.kallsyms])", "khugepaged_scan_pmd (in [kernel.kallsyms])", "trace_mm_khugepaged_scan_pmd (in [kernel.kallsyms])", "migrate_huge_page_move_mapping (in [kernel.kallsyms])", "do_huge_pmd_numa_page (in [kernel.kallsyms])", "pmd_pfn (in [kernel.kallsyms])", "protnone_mask (in [kernel.kallsyms])", "__pte_needs_invert (in [kernel.kallsyms])", "reclaim_high (in [kernel.kallsyms])", "memcg_memory_event (in [kernel.kallsyms])" ], "registerTime": 0, "unregisterTime": null, "processType": "default" } ], "processes": [], "pausedRanges": [] } # Signed-off-by: Anup Sharma Tested-by: Arnaldo Carvalho de Melo Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/cbf03cda175ea3dd2c6cd87bd3f12d803446cb95.1689961706.git.anupnewsmail@gmail.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/bin/gecko-record | 2 ++ tools/perf/scripts/python/bin/gecko-report | 3 +++ 2 files changed, 5 insertions(+) create mode 100644 tools/perf/scripts/python/bin/gecko-record create mode 100644 tools/perf/scripts/python/bin/gecko-report (limited to 'tools/perf/scripts/python') diff --git a/tools/perf/scripts/python/bin/gecko-record b/tools/perf/scripts/python/bin/gecko-record new file mode 100644 index 000000000000..f0d1aa55f171 --- /dev/null +++ b/tools/perf/scripts/python/bin/gecko-record @@ -0,0 +1,2 @@ +#!/bin/bash +perf record -F 99 -g "$@" diff --git a/tools/perf/scripts/python/bin/gecko-report b/tools/perf/scripts/python/bin/gecko-report new file mode 100644 index 000000000000..0c12cc08f3ab --- /dev/null +++ b/tools/perf/scripts/python/bin/gecko-report @@ -0,0 +1,3 @@ +#!/bin/bash +# description: create firefox gecko profile json format from perf.data +perf script "$@" -s "$PERF_EXEC_PATH"/scripts/python/gecko.py -- cgit v1.2.3 From c43888e739bbf184eb95018188215a5487cc0b15 Mon Sep 17 00:00:00 2001 From: Arnaldo Carvalho de Melo Date: Wed, 2 Aug 2023 10:33:42 -0300 Subject: perf script python: Cope with declarations after statements found in Python.h With -Werror the build was failing on fedora rawhide: [perfbuilder@27cfe44d67ed perf-6.5.0-rc2]$ gcc -v Using built-in specs. COLLECT_GCC=gcc COLLECT_LTO_WRAPPER=/usr/libexec/gcc/x86_64-redhat-linux/13/lto-wrapper OFFLOAD_TARGET_NAMES=nvptx-none OFFLOAD_TARGET_DEFAULT=1 Target: x86_64-redhat-linux Configured with: ../configure --enable-bootstrap --enable-languages=c,c++,fortran,objc,obj-c++,ada,go,d,m2,lto --prefix=/usr --mandir=/usr/share/man --infodir=/usr/share/info --with-bugurl=http://bugzilla.redhat.com/bugzilla --enable-shared --enable-threads=posix --enable-checking=release --enable-multilib --with-system-zlib --enable-__cxa_atexit --disable-libunwind-exceptions --enable-gnu-unique-object --enable-linker-build-id --with-gcc-major-version-only --enable-libstdcxx-backtrace --with-libstdcxx-zoneinfo=/usr/share/zoneinfo --with-linker-hash-style=gnu --enable-plugin --enable-initfini-array --with-isl=/builddir/build/BUILD/gcc-13.2.1-20230728/obj-x86_64-redhat-linux/isl-install --enable-offload-targets=nvptx-none --without-cuda-driver --enable-offload-defaulted --enable-gnu-indirect-function --enable-cet --with-tune=generic --with-arch_32=i686 --build=x86_64-redhat-linux --with-build-config=bootstrap-lto --enable-link-serialization=1 Thread model: posix Supported LTO compression algorithms: zlib zstd gcc version 13.2.1 20230728 (Red Hat 13.2.1-1) (GCC) [perfbuilder@27cfe44d67ed perf-6.5.0-rc2]$ In file included from /usr/include/python3.12/Python.h:44, from scripts/python/Perf-Trace-Util/Context.c:14: /usr/include/python3.12/object.h: In function 'Py_SIZE': /usr/include/python3.12/object.h:217:5: error: ISO C90 forbids mixed declarations and code [-Werror=declaration-after-statement] 217 | PyVarObject *var_ob = _PyVarObject_CAST(ob); | ^~~~~~~~~~~ In file included from /usr/include/python3.12/Python.h:53: /usr/include/python3.12/cpython/longintrepr.h: In function '_PyLong_CompactValue': /usr/include/python3.12/cpython/longintrepr.h:121:5: error: ISO C90 forbids mixed declarations and code [-Werror=declaration-after-statement] 121 | Py_ssize_t sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); | ^~~~~~~~~~ In file included from /usr/include/python3.12/Python.h:44, from util/scripting-engines/trace-event-python.c:22: /usr/include/python3.12/object.h: In function 'Py_SIZE': /usr/include/python3.12/object.h:217:5: error: ISO C90 forbids mixed declarations and code [-Werror=declaration-after-statement] 217 | PyVarObject *var_ob = _PyVarObject_CAST(ob); | ^~~~~~~~~~~ CC /tmp/build/perf/util/units.o CC /tmp/build/perf/util/time-utils.o In file included from /usr/include/python3.12/Python.h:53: /usr/include/python3.12/cpython/longintrepr.h: In function '_PyLong_CompactValue': /usr/include/python3.12/cpython/longintrepr.h:121:5: error: ISO C90 forbids mixed declarations and code [-Werror=declaration-after-statement] 121 | Py_ssize_t sign = 1 - (op->long_value.lv_tag & _PyLong_SIGN_MASK); | ^~~~~~~~~~ So add -Wno-declaration-after-statement to the python scripting CFLAGS. Reviewed-by: Ian Rogers Cc: Adrian Hunter Cc: Jiri Olsa Cc: Namhyung Kim Link: https://lore.kernel.org/lkml/ZMpdKeO8gU%2FcWDqH@kernel.org Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/Perf-Trace-Util/Build | 3 ++- tools/perf/util/scripting-engines/Build | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'tools/perf/scripts/python') diff --git a/tools/perf/scripts/python/Perf-Trace-Util/Build b/tools/perf/scripts/python/Perf-Trace-Util/Build index 7d0e33ce6aba..5b0b5ff7e14a 100644 --- a/tools/perf/scripts/python/Perf-Trace-Util/Build +++ b/tools/perf/scripts/python/Perf-Trace-Util/Build @@ -1,3 +1,4 @@ perf-y += Context.o -CFLAGS_Context.o += $(PYTHON_EMBED_CCOPTS) -Wno-redundant-decls -Wno-strict-prototypes -Wno-unused-parameter -Wno-nested-externs +# -Wno-declaration-after-statement: The python headers have mixed code with declarations (decls after asserts, for instance) +CFLAGS_Context.o += $(PYTHON_EMBED_CCOPTS) -Wno-redundant-decls -Wno-strict-prototypes -Wno-unused-parameter -Wno-nested-externs -Wno-declaration-after-statement diff --git a/tools/perf/util/scripting-engines/Build b/tools/perf/util/scripting-engines/Build index c220fec97032..586b94e90f4e 100644 --- a/tools/perf/util/scripting-engines/Build +++ b/tools/perf/util/scripting-engines/Build @@ -5,4 +5,5 @@ perf-$(CONFIG_LIBPYTHON) += trace-event-python.o CFLAGS_trace-event-perl.o += $(PERL_EMBED_CCOPTS) -Wno-redundant-decls -Wno-strict-prototypes -Wno-unused-parameter -Wno-shadow -Wno-nested-externs -Wno-undef -Wno-switch-default -Wno-bad-function-cast -Wno-declaration-after-statement -Wno-switch-enum -CFLAGS_trace-event-python.o += $(PYTHON_EMBED_CCOPTS) -Wno-redundant-decls -Wno-strict-prototypes -Wno-unused-parameter -Wno-shadow -Wno-deprecated-declarations -Wno-switch-enum +# -Wno-declaration-after-statement: The python headers have mixed code with declarations (decls after asserts, for instance) +CFLAGS_trace-event-python.o += $(PYTHON_EMBED_CCOPTS) -Wno-redundant-decls -Wno-strict-prototypes -Wno-unused-parameter -Wno-shadow -Wno-deprecated-declarations -Wno-switch-enum -Wno-declaration-after-statement -- cgit v1.2.3 From 708a3e8b80a5364fc3dd4991f1e589a6d7a4a8e0 Mon Sep 17 00:00:00 2001 From: Wei Li Date: Tue, 15 Aug 2023 21:17:35 +0800 Subject: perf scripts python: Support syscall name parsing on arm64 In the result of "perf script syscall-counts" on arm64, the syscall events are not resolved currently. Add "aarch64" to audit uname list to support name parsing. * After the patch: [root@localhost ~]# perf script syscall-counts sleep 1 Press control+C to stop and show the summary syscall events: event count ---------------------------------------- ----------- mmap 6 close 5 mprotect 4 brk 3 newfstatat 3 openat 3 getrandom 1 prlimit64 1 munmap 1 clock_nanosleep 1 set_robust_list 1 set_tid_address 1 exit_group 1 read 1 faccessat 1 Signed-off-by: Wei Li Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Li Bin Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20230815131735.1237221-1-liwei391@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Util.py | 1 + 1 file changed, 1 insertion(+) (limited to 'tools/perf/scripts/python') diff --git a/tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Util.py b/tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Util.py index 7384dcb628c4..c37a03fb7ec5 100644 --- a/tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Util.py +++ b/tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Util.py @@ -54,6 +54,7 @@ try: import audit machine_to_id = { 'x86_64': audit.MACH_86_64, + 'aarch64': audit.MACH_AARCH64, 'alpha' : audit.MACH_ALPHA, 'ia64' : audit.MACH_IA64, 'ppc' : audit.MACH_PPC, -- cgit v1.2.3 From 41a37430f66560c4b9a9d68a977c8dab65b97ff8 Mon Sep 17 00:00:00 2001 From: Wei Li Date: Tue, 15 Aug 2023 21:18:05 +0800 Subject: perf scripts python: Update audit-libs package name for python3 'audit-libs-python' is the package for python2, update it for python3. On Ubuntu and Fedora, the new package is 'python3-audit'. Signed-off-by: Wei Li Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Li Bin Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/20230815131805.1237491-1-liwei391@huawei.com Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'tools/perf/scripts/python') diff --git a/tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Util.py b/tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Util.py index c37a03fb7ec5..b75d31858e54 100644 --- a/tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Util.py +++ b/tools/perf/scripts/python/Perf-Trace-Util/lib/Perf/Trace/Util.py @@ -74,9 +74,9 @@ try: except: if not audit_package_warned: audit_package_warned = True - print("Install the audit-libs-python package to get syscall names.\n" - "For example:\n # apt-get install python-audit (Ubuntu)" - "\n # yum install audit-libs-python (Fedora)" + print("Install the python-audit package to get syscall names.\n" + "For example:\n # apt-get install python3-audit (Ubuntu)" + "\n # yum install python3-audit (Fedora)" "\n etc.\n") def syscall_name(id): -- cgit v1.2.3 From 43803cb16f997794a256483f29beb68abef2fa9e Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Sun, 13 Aug 2023 03:06:27 +0530 Subject: perf scripts python: Add support for input args in gecko script Refines the argument handling mechanism in the "gecko-report" script to enable better compatibility and improved user experience. The script now differentiates between scenarios where arguments are provided for record and report cases where gecko.py arguments are passed. Signed-off-by: Anup Sharma Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ian Rogers Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/ZNf7W+EIrrCSHZN0@yoga Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/bin/gecko-report | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) mode change 100644 => 100755 tools/perf/scripts/python/bin/gecko-report (limited to 'tools/perf/scripts/python') diff --git a/tools/perf/scripts/python/bin/gecko-report b/tools/perf/scripts/python/bin/gecko-report old mode 100644 new mode 100755 index 0c12cc08f3ab..1867ec8d9757 --- a/tools/perf/scripts/python/bin/gecko-report +++ b/tools/perf/scripts/python/bin/gecko-report @@ -1,3 +1,7 @@ #!/bin/bash # description: create firefox gecko profile json format from perf.data -perf script "$@" -s "$PERF_EXEC_PATH"/scripts/python/gecko.py +if [ "$*" = "-i -" ]; then +perf script -s "$PERF_EXEC_PATH"/scripts/python/gecko.py +else +perf script -s "$PERF_EXEC_PATH"/scripts/python/gecko.py -- "$@" +fi -- cgit v1.2.3 From f208b2c6f984f9c8086205e826985ab4441ce2da Mon Sep 17 00:00:00 2001 From: Anup Sharma Date: Wed, 9 Aug 2023 18:51:22 +0530 Subject: perf scripts python gecko: Launch the profiler UI on the default browser with the appropriate URL All required libraries have been imported and make sure that none of them are external dependencies. To achieve this, created a virt env and verified. Modified usage information and added combined command. Modified the main() function to read the --save-only command-line option and set the output_file variable accordingly. Modified the trace_end() function to check for the output_file variable. If it is set, the profiler data is saved to a local file in Gecko Profile format, or the profiler.firefox.com is opened on the default browser. Included trace_begin() to initialize the Firefox Profiler and launch the default browser to display the profiler.firefox.com. Added a new function launchFirefox() to start a local server and launch the profiler UI on the default browser with the appropriate URL. Created the "CORSRequestHandler" class to enable Cross-Origin Resource Sharing. Summary: This integration now includes a exiting feature to conveniently host the Gecko Profile data on a local server and open it directly in the default web browser. This means that users can now effortlessly visualize and analyze the profiler results with just a single click. The addition of the --save-only command-line option allows users to save the profiler output to a local file in Gecko Profile format, but the real highlight lies in the capability to seamlessly launch a local server, making the data accessible to Firefox Profiler via a web browser. In addition, it's important to highlight that all data are hosted locally, eliminating any concerns about data privacy rules and regulations. Signed-off-by: Anup Sharma Tested-by: Ian Rogers Cc: Adrian Hunter Cc: Alexander Shishkin Cc: Ingo Molnar Cc: Jiri Olsa Cc: Mark Rutland Cc: Namhyung Kim Cc: Peter Zijlstra Link: https://lore.kernel.org/r/ZNOS0vo58DnVLpD8@yoga Signed-off-by: Arnaldo Carvalho de Melo --- tools/perf/scripts/python/gecko.py | 70 ++++++++++++++++++++++++++++++++++---- 1 file changed, 63 insertions(+), 7 deletions(-) (limited to 'tools/perf/scripts/python') diff --git a/tools/perf/scripts/python/gecko.py b/tools/perf/scripts/python/gecko.py index 278c3aed282a..bc5a72f94bfa 100644 --- a/tools/perf/scripts/python/gecko.py +++ b/tools/perf/scripts/python/gecko.py @@ -1,4 +1,4 @@ -# firefox-gecko-converter.py - Convert perf record output to Firefox's gecko profile format +# gecko.py - Convert perf record output to Firefox's gecko profile format # SPDX-License-Identifier: GPL-2.0 # # The script converts perf.data to Gecko Profile Format, @@ -7,14 +7,26 @@ # Usage: # # perf record -a -g -F 99 sleep 60 -# perf script report gecko > output.json +# perf script report gecko +# +# Combined: +# +# perf script gecko -F 99 -a sleep 60 import os import sys +import time import json +import string +import random import argparse +import threading +import webbrowser +import urllib.parse +from os import system from functools import reduce from dataclasses import dataclass, field +from http.server import HTTPServer, SimpleHTTPRequestHandler, test from typing import List, Dict, Optional, NamedTuple, Set, Tuple, Any # Add the Perf-Trace-Util library to the Python path @@ -40,9 +52,15 @@ CATEGORIES = None # The product name is used by the profiler UI to show the Operating system and Processor. PRODUCT = os.popen('uname -op').read().strip() +# store the output file +output_file = None + # Here key = tid, value = Thread tid_to_thread = dict() +# The HTTP server is used to serve the profile to the profiler UI. +http_server_thread = None + # The category index is used by the profiler UI to show the color of the flame graph. USER_CATEGORY_INDEX = 0 KERNEL_CATEGORY_INDEX = 1 @@ -278,9 +296,19 @@ def process_event(param_dict: Dict) -> None: tid_to_thread[tid] = thread thread._add_sample(comm=comm, stack=stack, time_ms=time_stamp) +def trace_begin() -> None: + global output_file + if (output_file is None): + print("Staring Firefox Profiler on your default browser...") + global http_server_thread + http_server_thread = threading.Thread(target=test, args=(CORSRequestHandler, HTTPServer,)) + http_server_thread.daemon = True + http_server_thread.start() + # Trace_end runs at the end and will be used to aggregate # the data into the final json object and print it out to stdout. def trace_end() -> None: + global output_file threads = [thread._to_json_dict() for thread in tid_to_thread.values()] # Schema: https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L305 @@ -305,22 +333,50 @@ def trace_end() -> None: "processes": [], "pausedRanges": [], } - json.dump(gecko_profile_with_meta, sys.stdout, indent=2) + # launch the profiler on local host if not specified --save-only args, otherwise print to file + if (output_file is None): + output_file = 'gecko_profile.json' + with open(output_file, 'w') as f: + json.dump(gecko_profile_with_meta, f, indent=2) + launchFirefox(output_file) + time.sleep(1) + print(f'[ perf gecko: Captured and wrote into {output_file} ]') + else: + print(f'[ perf gecko: Captured and wrote into {output_file} ]') + with open(output_file, 'w') as f: + json.dump(gecko_profile_with_meta, f, indent=2) + +# Used to enable Cross-Origin Resource Sharing (CORS) for requests coming from 'https://profiler.firefox.com', allowing it to access resources from this server. +class CORSRequestHandler(SimpleHTTPRequestHandler): + def end_headers (self): + self.send_header('Access-Control-Allow-Origin', 'https://profiler.firefox.com') + SimpleHTTPRequestHandler.end_headers(self) + +# start a local server to serve the gecko_profile.json file to the profiler.firefox.com +def launchFirefox(file): + safe_string = urllib.parse.quote_plus(f'http://localhost:8000/{file}') + url = 'https://profiler.firefox.com/from-url/' + safe_string + webbrowser.open(f'{url}') def main() -> None: + global output_file global CATEGORIES - parser = argparse.ArgumentParser(description="Convert perf.data to Firefox\'s Gecko Profile format") + parser = argparse.ArgumentParser(description="Convert perf.data to Firefox\'s Gecko Profile format which can be uploaded to profiler.firefox.com for visualization") # Add the command-line options # Colors must be defined according to this: # https://github.com/firefox-devtools/profiler/blob/50124adbfa488adba6e2674a8f2618cf34b59cd2/res/css/categories.css - parser.add_argument('--user-color', default='yellow', help='Color for the User category') - parser.add_argument('--kernel-color', default='orange', help='Color for the Kernel category') + parser.add_argument('--user-color', default='yellow', help='Color for the User category', choices=['yellow', 'blue', 'purple', 'green', 'orange', 'red', 'grey', 'magenta']) + parser.add_argument('--kernel-color', default='orange', help='Color for the Kernel category', choices=['yellow', 'blue', 'purple', 'green', 'orange', 'red', 'grey', 'magenta']) + # If --save-only is specified, the output will be saved to a file instead of opening Firefox's profiler directly. + parser.add_argument('--save-only', help='Save the output to a file instead of opening Firefox\'s profiler') + # Parse the command-line arguments args = parser.parse_args() # Access the values provided by the user user_color = args.user_color kernel_color = args.kernel_color + output_file = args.save_only CATEGORIES = [ { @@ -336,4 +392,4 @@ def main() -> None: ] if __name__ == '__main__': - main() + main() -- cgit v1.2.3