llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-lldb Author: John Harrison (ashgti) <details> <summary>Changes</summary> Originally commited in 362b9d78b4ee9107da2b5e90b3764b0f0fa610fe and then reverted in cb63b75e32a415c9bfc298ed7fdcd67e8d9de54c. This re-lands a subset of the changes to dap_server.py/DebugCommunication and addresses the python3.10 compatibility issue. This includes less type annotations since those were the reason for the failures on that specific version of python. I've done additional testing on python3.8, python3.10 and python3.13 to further validate these changes. --- Patch is 63.10 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/147787.diff 9 Files Affected: - (modified) lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py (+404-285) - (modified) lldb/packages/Python/lldbsuite/test/tools/lldb-dap/lldbdap_testcase.py (+47-35) - (modified) lldb/test/API/tools/lldb-dap/attach/TestDAP_attach.py (+4-4) - (modified) lldb/test/API/tools/lldb-dap/cancel/TestDAP_cancel.py (+8-13) - (modified) lldb/test/API/tools/lldb-dap/commands/TestDAP_commands.py (+6-9) - (modified) lldb/test/API/tools/lldb-dap/console/TestDAP_console.py (+3-9) - (modified) lldb/test/API/tools/lldb-dap/launch/TestDAP_launch.py (+8-17) - (modified) lldb/test/API/tools/lldb-dap/module/TestDAP_module.py (+2-4) - (modified) lldb/test/API/tools/lldb-dap/output/TestDAP_output.py (+3-3) ``````````diff diff --git a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py index d227a66a703c1..1677839df54c5 100644 --- a/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py +++ b/lldb/packages/Python/lldbsuite/test/tools/lldb-dap/dap_server.py @@ -12,15 +12,56 @@ import sys import threading import time -from typing import Any, Optional, Union, BinaryIO, TextIO +from typing import ( + Any, + Optional, + Dict, + cast, + List, + Callable, + IO, + Union, + BinaryIO, + TextIO, + TypedDict, + Literal, +) ## DAP type references -Event = dict[str, Any] -Request = dict[str, Any] -Response = dict[str, Any] + + +class Event(TypedDict): + type: Literal["event"] + seq: int + event: str + body: Any + + +class Request(TypedDict, total=False): + type: Literal["request"] + seq: int + command: str + arguments: Any + + +class Response(TypedDict): + type: Literal["response"] + seq: int + request_seq: int + success: bool + command: str + message: Optional[str] + body: Any + + ProtocolMessage = Union[Event, Request, Response] +class Breakpoint(TypedDict, total=False): + id: int + verified: bool + + def dump_memory(base_addr, data, num_per_line, outfile): data_len = len(data) hex_string = binascii.hexlify(data) @@ -58,7 +99,9 @@ def dump_memory(base_addr, data, num_per_line, outfile): outfile.write("\n") -def read_packet(f, verbose=False, trace_file=None): +def read_packet( + f: IO[bytes], trace_file: Optional[IO[str]] = None +) -> Optional[ProtocolMessage]: """Decode a JSON packet that starts with the content length and is followed by the JSON bytes from a file 'f'. Returns None on EOF. """ @@ -70,19 +113,13 @@ def read_packet(f, verbose=False, trace_file=None): prefix = "Content-Length: " if line.startswith(prefix): # Decode length of JSON bytes - if verbose: - print('content: "%s"' % (line)) length = int(line[len(prefix) :]) - if verbose: - print('length: "%u"' % (length)) # Skip empty line - line = f.readline() - if verbose: - print('empty: "%s"' % (line)) + separator = f.readline().decode() + if separator != "": + Exception("malformed DAP content header, unexpected line: " + separator) # Read JSON bytes - json_str = f.read(length) - if verbose: - print('json: "%s"' % (json_str)) + json_str = f.read(length).decode() if trace_file: trace_file.write("from adapter:\n%s\n" % (json_str)) # Decode the JSON bytes into a python dictionary @@ -95,7 +132,7 @@ def packet_type_is(packet, packet_type): return "type" in packet and packet["type"] == packet_type -def dump_dap_log(log_file): +def dump_dap_log(log_file: Optional[str]) -> None: print("========= DEBUG ADAPTER PROTOCOL LOGS =========", file=sys.stderr) if log_file is None: print("no log file available", file=sys.stderr) @@ -152,25 +189,41 @@ def __init__( self.log_file = log_file self.send = send self.recv = recv - self.recv_packets: list[Optional[ProtocolMessage]] = [] - self.recv_condition = threading.Condition() - self.recv_thread = threading.Thread(target=self._read_packet_thread) - self.process_event_body = None - self.exit_status: Optional[int] = None - self.capabilities: dict[str, Any] = {} - self.progress_events: list[Event] = [] - self.reverse_requests = [] - self.sequence = 1 - self.threads = None - self.thread_stop_reasons = {} - self.recv_thread.start() - self.output_condition = threading.Condition() - self.output: dict[str, list[str]] = {} - self.configuration_done_sent = False - self.initialized = False - self.frame_scopes = {} + + # Packets that have been received and processed but have not yet been + # requested by a test case. + self._pending_packets: List[Optional[ProtocolMessage]] = [] + # Received packets that have not yet been processed. + self._recv_packets: List[Optional[ProtocolMessage]] = [] + # Used as a mutex for _recv_packets and for notify when _recv_packets + # changes. + self._recv_condition = threading.Condition() + self._recv_thread = threading.Thread(target=self._read_packet_thread) + + # session state self.init_commands = init_commands - self.resolved_breakpoints = {} + self.exit_status: Optional[int] = None + self.capabilities: Optional[Dict] = None + self.initialized: bool = False + self.configuration_done_sent: bool = False + self.process_event_body: Optional[Dict] = None + self.terminated: bool = False + self.events: List[Event] = [] + self.progress_events: List[Event] = [] + self.reverse_requests: List[Request] = [] + self.module_events: List[Dict] = [] + self.sequence: int = 1 + self.output: Dict[str, str] = {} + + # debuggee state + self.threads: Optional[dict] = None + self.thread_stop_reasons: Dict[str, Any] = {} + self.frame_scopes: Dict[str, Any] = {} + # keyed by breakpoint id + self.resolved_breakpoints: Dict[str, bool] = {} + + # trigger enqueue thread + self._recv_thread.start() @classmethod def encode_content(cls, s: str) -> bytes: @@ -188,267 +241,326 @@ def validate_response(cls, command, response): ) def _read_packet_thread(self): - done = False try: - while not done: + while True: packet = read_packet(self.recv, trace_file=self.trace_file) # `packet` will be `None` on EOF. We want to pass it down to # handle_recv_packet anyway so the main thread can handle unexpected # termination of lldb-dap and stop waiting for new packets. - done = not self._handle_recv_packet(packet) + if not self._handle_recv_packet(packet): + break finally: dump_dap_log(self.log_file) - def get_modules(self, startModule: int = 0, moduleCount: int = 0): - module_list = self.request_modules(startModule, moduleCount)["body"]["modules"] + def get_modules( + self, start_module: Optional[int] = None, module_count: Optional[int] = None + ) -> Dict: + resp = self.request_modules(start_module, module_count) + if not resp["success"]: + raise ValueError(f"request_modules failed: {resp!r}") modules = {} + module_list = resp["body"]["modules"] for module in module_list: modules[module["name"]] = module return modules - def get_output(self, category, timeout=0.0, clear=True): - self.output_condition.acquire() - output = None + def get_output(self, category: str, clear=True) -> str: + output = "" if category in self.output: - output = self.output[category] + output = self.output.get(category, "") if clear: del self.output[category] - elif timeout != 0.0: - self.output_condition.wait(timeout) - if category in self.output: - output = self.output[category] - if clear: - del self.output[category] - self.output_condition.release() return output - def collect_output(self, category, timeout_secs, pattern, clear=True): - end_time = time.time() + timeout_secs - collected_output = "" - while end_time > time.time(): - output = self.get_output(category, timeout=0.25, clear=clear) - if output: - collected_output += output - if pattern is not None and pattern in output: - break - return collected_output if collected_output else None + def collect_output( + self, + category: str, + timeout: float, + pattern: Optional[str] = None, + clear=True, + ) -> str: + """Collect output from 'output' events. + Args: + category: The category to collect. + timeout: The max duration for collecting output. + pattern: + Optional, if set, return once this pattern is detected in the + collected output. + Returns: + The collected output. + """ + deadline = time.monotonic() + timeout + output = self.get_output(category, clear) + while deadline >= time.monotonic() and ( + pattern is None or pattern not in output + ): + event = self.wait_for_event(["output"], timeout=deadline - time.monotonic()) + if not event: # Timeout or EOF + break + output += self.get_output(category, clear=clear) + return output def _enqueue_recv_packet(self, packet: Optional[ProtocolMessage]): - self.recv_condition.acquire() - self.recv_packets.append(packet) - self.recv_condition.notify() - self.recv_condition.release() + with self.recv_condition: + self.recv_packets.append(packet) + self.recv_condition.notify() def _handle_recv_packet(self, packet: Optional[ProtocolMessage]) -> bool: - """Called by the read thread that is waiting for all incoming packets - to store the incoming packet in "self.recv_packets" in a thread safe - way. This function will then signal the "self.recv_condition" to - indicate a new packet is available. Returns True if the caller - should keep calling this function for more packets. + """Handles an incoming packet. + + Called by the read thread that is waiting for all incoming packets + to store the incoming packet in "self._recv_packets" in a thread safe + way. This function will then signal the "self._recv_condition" to + indicate a new packet is available. + + Args: + packet: A new packet to store. + + Returns: + True if the caller should keep calling this function for more + packets. """ - # If EOF, notify the read thread by enqueuing a None. - if not packet: - self._enqueue_recv_packet(None) - return False - - # Check the packet to see if is an event packet - keepGoing = True - packet_type = packet["type"] - if packet_type == "event": - event = packet["event"] - body = None - if "body" in packet: - body = packet["body"] - # Handle the event packet and cache information from these packets - # as they come in - if event == "output": - # Store any output we receive so clients can retrieve it later. - category = body["category"] - output = body["output"] - self.output_condition.acquire() - if category in self.output: - self.output[category] += output - else: - self.output[category] = output - self.output_condition.notify() - self.output_condition.release() - # no need to add 'output' event packets to our packets list - return keepGoing - elif event == "initialized": - self.initialized = True - elif event == "process": - # When a new process is attached or launched, remember the - # details that are available in the body of the event - self.process_event_body = body - elif event == "exited": - # Process exited, mark the status to indicate the process is not - # alive. - self.exit_status = body["exitCode"] - elif event == "continued": - # When the process continues, clear the known threads and - # thread_stop_reasons. - all_threads_continued = body.get("allThreadsContinued", True) - tid = body["threadId"] - if tid in self.thread_stop_reasons: - del self.thread_stop_reasons[tid] - self._process_continued(all_threads_continued) - elif event == "stopped": - # Each thread that stops with a reason will send a - # 'stopped' event. We need to remember the thread stop - # reasons since the 'threads' command doesn't return - # that information. - self._process_stopped() - tid = body["threadId"] - self.thread_stop_reasons[tid] = body - elif event.startswith("progress"): - # Progress events come in as 'progressStart', 'progressUpdate', - # and 'progressEnd' events. Keep these around in case test - # cases want to verify them. - self.progress_events.append(packet) - elif event == "breakpoint": - # Breakpoint events are sent when a breakpoint is resolved - self._update_verified_breakpoints([body["breakpoint"]]) - elif event == "capabilities": - # Update the capabilities with new ones from the event. - self.capabilities.update(body["capabilities"]) - - elif packet_type == "response": - if packet["command"] == "disconnect": - keepGoing = False - self._enqueue_recv_packet(packet) - return keepGoing + with self._recv_condition: + self._recv_packets.append(packet) + self._recv_condition.notify() + # packet is None on EOF + return packet is not None and not ( + packet["type"] == "response" and packet["command"] == "disconnect" + ) + + def _recv_packet( + self, + *, + predicate: Optional[Callable[[ProtocolMessage], bool]] = None, + timeout: Optional[float] = None, + ) -> Optional[ProtocolMessage]: + """Processes received packets from the adapter. + Updates the DebugCommunication stateful properties based on the received + packets in the order they are received. + NOTE: The only time the session state properties should be updated is + during this call to ensure consistency during tests. + Args: + predicate: + Optional, if specified, returns the first packet that matches + the given predicate. + timeout: + Optional, if specified, processes packets until either the + timeout occurs or the predicate matches a packet, whichever + occurs first. + Returns: + The first matching packet for the given predicate, if specified, + otherwise None. + """ + assert ( + threading.current_thread != self._recv_thread + ), "Must not be called from the _recv_thread" + + def process_until_match(): + self._process_recv_packets() + for i, packet in enumerate(self._pending_packets): + if packet is None: + # We need to return a truthy value to break out of the + # wait_for, use `EOFError` as an indicator of EOF. + return EOFError() + if predicate and predicate(packet): + self._pending_packets.pop(i) + return packet + + with self._recv_condition: + packet = self._recv_condition.wait_for(process_until_match, timeout) + return None if isinstance(packet, EOFError) else packet + + def _process_recv_packets(self) -> None: + """Process received packets, updating the session state.""" + with self._recv_condition: + for packet in self._recv_packets: + # Handle events that may modify any stateful properties of + # the DAP session. + if packet and packet["type"] == "event": + self._handle_event(packet) + elif packet and packet["type"] == "request": + # Handle reverse requests and keep processing. + self._handle_reverse_request(packet) + # Move the packet to the pending queue. + self._pending_packets.append(packet) + self._recv_packets.clear() + + def _handle_event(self, packet: Event) -> None: + """Handle any events that modify debug session state we track.""" + event = packet["event"] + body: Optional[Dict] = packet.get("body", None) + + if event == "output" and body: + # Store any output we receive so clients can retrieve it later. + category = body["category"] + output = body["output"] + if category in self.output: + self.output[category] += output + else: + self.output[category] = output + elif event == "initialized": + self.initialized = True + elif event == "process": + # When a new process is attached or launched, remember the + # details that are available in the body of the event + self.process_event_body = body + elif event == "exited" and body: + # Process exited, mark the status to indicate the process is not + # alive. + self.exit_status = body["exitCode"] + elif event == "continued" and body: + # When the process continues, clear the known threads and + # thread_stop_reasons. + all_threads_continued = body.get("allThreadsContinued", True) + tid = body["threadId"] + if tid in self.thread_stop_reasons: + del self.thread_stop_reasons[tid] + self._process_continued(all_threads_continued) + elif event == "stopped" and body: + # Each thread that stops with a reason will send a + # 'stopped' event. We need to remember the thread stop + # reasons since the 'threads' command doesn't return + # that information. + self._process_stopped() + tid = body["threadId"] + self.thread_stop_reasons[tid] = body + elif event.startswith("progress"): + # Progress events come in as 'progressStart', 'progressUpdate', + # and 'progressEnd' events. Keep these around in case test + # cases want to verify them. + self.progress_events.append(packet) + elif event == "breakpoint" and body: + # Breakpoint events are sent when a breakpoint is resolved + self._update_verified_breakpoints([body["breakpoint"]]) + elif event == "capabilities" and body: + if self.capabilities is None: + self.capabilities = {} + # Update the capabilities with new ones from the event. + self.capabilities.update(body["capabilities"]) + + def _handle_reverse_request(self, request: Request) -> None: + if request in self.reverse_requests: + return + self.reverse_requests.append(request) + arguments = request.get("arguments") + if request["command"] == "runInTerminal" and arguments is not None: + in_shell = arguments.get("argsCanBeInterpretedByShell", False) + proc = subprocess.Popen( + arguments["args"], + env=arguments.get("env", {}), + cwd=arguments.get("cwd", None), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + shell=in_shell, + ) + body = {} + if in_shell: + body["shellProcessId"] = proc.pid + else: + body["processId"] = proc.pid + self.send_packet( + { + "type": "response", + ... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/147787 _______________________________________________ lldb-commits mailing list lldb-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
