comparaison avec ssh://hg@hg.python.org/features/tracemalloc running ssh hg@hg.python.org 'hg -R features/tracemalloc serve --stdio' searching for changes changeset: 85564:255e8ad9a705 user: Victor Stinner date: Wed Sep 18 00:08:34 2013 +0200 files: Doc/library/tracemalloc.rst Lib/test/regrtest.py Lib/test/test_tracemalloc.py Lib/tracemalloc.py Modules/_tracemalloc.c description: Replace the Frame type with a simple and efficient tuple (filename, lineno) Loading a snapshot is 2 times faster. diff -r 3dbde0c24ca7 -r 255e8ad9a705 Doc/library/tracemalloc.rst --- a/Doc/library/tracemalloc.rst Tue Sep 17 17:09:52 2013 +0200 +++ b/Doc/library/tracemalloc.rst Wed Sep 18 00:08:34 2013 +0200 @@ -18,9 +18,8 @@ as early as possible by calling :func:`t the :envvar:`PYTHONTRACEMALLOC` environment variable to ``1``, or by using :option:`-X` ``tracemalloc`` command line option. -By default, the :attr:`Trace.traceback` attribute only stores one -:class:`~tracemalloc.Frame` instance per allocated memory block. Use -:func:`set_traceback_limit` to store more frames. +By default, the :attr:`Trace.traceback` attribute only stores one frame per +allocated memory block. Use :func:`set_traceback_limit` to store more frames. .. versionadded:: 3.4 @@ -213,7 +212,7 @@ Functions .. function:: get_traceback_limit() - Get the maximum number of :class:`Frame` instances stored in the + Get the maximum number of frames stored in the :attr:`~tracemalloc.Trace.traceback` attribute of a :class:`~tracemalloc.Trace` instance. @@ -309,7 +308,7 @@ Functions .. function:: set_traceback_limit(limit: int) - Set the maximum number of :class:`Frame` instances stored in the + Set the maximum number of frames stored in the :attr:`~tracemalloc.Trace.traceback` attribute of a :class:`~tracemalloc.Trace` instance. Clear all traces and statistics on Python memory allocations if the :mod:`tracemalloc` module is enabled, @@ -503,22 +502,6 @@ Filter :func:`get_traceback_limit` function. -Frame ------ - -.. class:: Frame - - Trace of a Python frame, used by :attr:`Trace.traceback` attribute. - - .. attribute:: filename - - Python filename, ``None`` if unknown. - - .. attribute:: lineno - - Python line number, ``None`` if unknown. - - GroupedStats ------------ @@ -746,11 +729,12 @@ Trace .. attribute:: traceback - Traceback where the memory block was allocated as a list of - :class:`~tracemalloc.Frame` instances, most recent first. + Traceback where the memory block was allocated as a list of ``(filename: + str, lineno: int)`` tuples, most recent first. *filename* and *lineno* + can be ``None``. - The list can be empty or incomplete if the :mod:`tracemalloc` module was - unable to retrieve the full traceback. + The traceback can be empty or incomplete if the :mod:`tracemalloc` module + was unable to retrieve the full traceback. The traceback is limited to :func:`get_traceback_limit` frames. Use :func:`set_traceback_limit` to store more frames. diff -r 3dbde0c24ca7 -r 255e8ad9a705 Lib/test/regrtest.py --- a/Lib/test/regrtest.py Tue Sep 17 17:09:52 2013 +0200 +++ b/Lib/test/regrtest.py Wed Sep 18 00:08:34 2013 +0200 @@ -376,10 +376,10 @@ def main(tests=None, testdir=None, verbo if tracemalloc.is_enabled(): if 0: tracemalloc.add_include_filter("", traceback=True) - #tracemalloc.set_traceback_limit(15) + tracemalloc.set_traceback_limit(15) tracemalloc.clear_traces() - if 1: + if 0: #top = tracemalloc.DisplayTop(25, group_per_file=True) top = tracemalloc.DisplayTopTask(10, file=sys.stderr) top.start(5) diff -r 3dbde0c24ca7 -r 255e8ad9a705 Lib/test/test_tracemalloc.py --- a/Lib/test/test_tracemalloc.py Tue Sep 17 17:09:52 2013 +0200 +++ b/Lib/test/test_tracemalloc.py Wed Sep 18 00:08:34 2013 +0200 @@ -16,8 +16,7 @@ def get_frames(nframe, lineno_delta): for index in range(nframe): code = frame.f_code lineno = frame.f_lineno + lineno_delta - frame_trace = tracemalloc.Frame((code.co_filename, lineno)) - frames.append(frame_trace) + frames.append((code.co_filename, lineno)) lineno_delta = 0 frame = frame.f_back return frames @@ -104,7 +103,8 @@ class TestTracemallocEnabled(unittest.Te # get the allocation location to filter allocations size = 12345 obj, frames = allocate_bytes(size) - tracemalloc.add_include_filter(frames[0].filename, frames[0].lineno) + filename, lineno = frames[0] + tracemalloc.add_include_filter(filename, lineno) # allocate one object tracemalloc.clear_traces() @@ -143,11 +143,11 @@ class TestTracemallocEnabled(unittest.Te self.assertIsInstance(line_stat, tracemalloc.TraceStats) break - frame = obj_frames[0] - self.assertIn(frame.filename, stats) - line_stats = stats[frame.filename] - self.assertIn(frame.lineno, line_stats) - line_stat = line_stats[frame.lineno] + filename, lineno = obj_frames[0] + self.assertIn(filename, stats) + line_stats = stats[filename] + self.assertIn(lineno, line_stats) + line_stat = line_stats[lineno] self.assertEqual(line_stat.size, total) self.assertEqual(line_stat.count, count) @@ -157,14 +157,14 @@ class TestTracemallocEnabled(unittest.Te obj, obj_frames = allocate_bytes(size) stats = tracemalloc.get_stats() - frame = obj_frames[0] - line_stats = stats[frame.filename][frame.lineno] + filename, lineno = obj_frames[0] + line_stats = stats[filename][lineno] self.assertEqual(line_stats.size, size) self.assertEqual(line_stats.count, 1) tracemalloc.clear_traces() stats2 = tracemalloc.get_stats() - self.assertNotIn(frame.lineno, stats2[frame.filename]) + self.assertNotIn(lineno, stats2[filename]) def test_timer(self): calls = [] @@ -286,8 +286,8 @@ class TestTracemallocEnabled(unittest.Te self.assertIsInstance(trace, tracemalloc.Trace) # test exclusive filter, based on previous filters - frame = obj_frames[0] - tracemalloc.add_exclude_filter(frame.filename, frame.lineno) + filename, lineno = obj_frames[0] + tracemalloc.add_exclude_filter(filename, lineno) tracemalloc.clear_traces() obj, obj_frames = allocate_bytes(size) trace = tracemalloc.get_object_trace(obj) @@ -546,12 +546,12 @@ class TestFilters(unittest.TestCase): def test_filter_match_trace(self): t1 = tracemalloc.Trace((123, [ - tracemalloc.Frame(("a.py", 2)), - tracemalloc.Frame(("b.py", 3))])) + ("a.py", 2), + ("b.py", 3)])) t2 = tracemalloc.Trace((123, [ - tracemalloc.Frame(("b.py", 4)), - tracemalloc.Frame(("b.py", 5))])) + ("b.py", 4), + ("b.py", 5)])) f = tracemalloc.Filter(True, "b.py", traceback=True) self.assertTrue(f.match_trace(t1)) @@ -638,12 +638,12 @@ class TestTop(unittest.TestCase): traceback_limit = 5 user_data = {} - traceback_a_2 = [tracemalloc.Frame(('a.py', 2)), - tracemalloc.Frame(('b.py', 4))] - traceback_a_5 = [tracemalloc.Frame(('a.py', 5)), - tracemalloc.Frame(('b.py', 4))] - traceback_b_1 = [tracemalloc.Frame(('b.py', 1))] - traceback_none_none = [tracemalloc.Frame((None, None))] + traceback_a_2 = [('a.py', 2), + ('b.py', 4)] + traceback_a_5 = [('a.py', 5), + ('b.py', 4)] + traceback_b_1 = [('b.py', 1)] + traceback_none_none = [(None, None)] timestamp = datetime.datetime(2013, 9, 12, 15, 16, 17) traces = { diff -r 3dbde0c24ca7 -r 255e8ad9a705 Lib/tracemalloc.py --- a/Lib/tracemalloc.py Tue Sep 17 17:09:52 2013 +0200 +++ b/Lib/tracemalloc.py Wed Sep 18 00:08:34 2013 +0200 @@ -97,16 +97,22 @@ def _format_traceback(traceback, filenam lines = ['Traceback (most recent call first):'] if traceback is not None: for frame in traceback: - filename = _format_filename(frame.filename, filename_parts, color) - lineno = _format_lineno(frame.lineno) + filename, lineno = frame + if filename and lineno: + line = linecache.getline(filename, lineno) + line = line.strip() + else: + line = None + + filename = _format_filename(filename, filename_parts, color) + lineno = _format_lineno(lineno) lines.append(' File "%s", line %s' % (filename, lineno)) - if frame.filename and frame.lineno: - line = linecache.getline(frame.filename, frame.lineno) - line = line.strip() - if line: - lines.append(' ' + line) + if line: + lines.append(' ' + line) else: - lines.append(' File "%s", line %' % (frame.filename, frame.lineno)) + filename = _format_filename(None, filename_parts, color) + lineno = _format_lineno(None) + lines.append(' File "%s", line %s' % (filename, lineno)) return lines _meminfo = collections.namedtuple('meminfo', 'rss vms') @@ -442,20 +448,19 @@ def _lazy_import_pickle(): def _get_first_frame(trace): if trace.traceback: - frame = trace.traceback[0] - return (frame.filename, frame.lineno) + return trace.traceback[0] else: return (None, None) def _compute_stats_frame(stats, group_per_file, size, frame): if not group_per_file: if frame is not None: - key = (frame.filename, frame.lineno) + key = frame else: key = (None, None) else: if frame is not None: - key = frame.filename + key = frame[0] else: key = None if key in stats: @@ -543,17 +548,6 @@ class Snapshot: traces = pickle.load(fp) else: traces = None - if traces is not None: - new_traces = {} - for address, item in traces.items(): - size = item[0] - traceback = [] - for index in range(1, len(item), 2): - filename = item[index] - lineno = item[index + 1] - traceback.append(Frame((filename, lineno))) - new_traces[address] = Trace((size, traceback)) - traces = new_traces return cls(timestamp, pid, traces, stats, process_memory, tracemalloc_size, traceback_limit, @@ -570,15 +564,6 @@ class Snapshot: 'traceback_limit': self.traceback_limit, 'user_data': self.user_data, } - if self.traces is not None: - traces = {} - for address, trace in self.traces.items(): - item = [trace.size] - for frame in trace.traceback: - item.extend((frame.filename, frame.lineno)) - traces[address] = item - else: - traces = None if self.process_memory is not None: # use a simple tuple rather than a namedtuple data['process_memory'] = tuple(self.process_memory) @@ -586,7 +571,7 @@ class Snapshot: try: with open(filename, "wb") as fp: pickle.dump(data, fp, pickle.HIGHEST_PROTOCOL) - pickle.dump(traces, fp, pickle.HIGHEST_PROTOCOL) + pickle.dump(self.traces, fp, pickle.HIGHEST_PROTOCOL) except: # Remove corrupted pickle file if os.path.exists(filename): diff -r 3dbde0c24ca7 -r 255e8ad9a705 Modules/_tracemalloc.c --- a/Modules/_tracemalloc.c Tue Sep 17 17:09:52 2013 +0200 +++ b/Modules/_tracemalloc.c Wed Sep 18 00:08:34 2013 +0200 @@ -1601,25 +1601,6 @@ tracemalloc_atfork(void) } #endif -PyDoc_STRVAR(tracemalloc_frame_structseq__doc__, -"Frame: trace of a Python frame."); - -static PyStructSequence_Field tracemalloc_frame_structseq_fields[] = { - {"filename", "Python filename, None if unknown"}, - {"lineno", "Python line number, None if unknown"}, - {0} -}; - -static PyStructSequence_Desc tracemalloc_frame_structseq_desc = { - "tracemalloc.Frame", /* name */ - tracemalloc_frame_structseq__doc__, /* doc */ - tracemalloc_frame_structseq_fields, - 2 -}; - -static PyTypeObject FrameType; - - PyDoc_STRVAR(tracemalloc_structseq__doc__, "Trace: debug information of an allocated memory block."); @@ -1675,9 +1656,6 @@ tracemalloc_init(void) if (PyStructSequence_InitType2(&TraceType, &tracemalloc_structseq_desc) < 0) return -1; - if (PyStructSequence_InitType2(&FrameType, - &tracemalloc_frame_structseq_desc) < 0) - return -1; if (PyStructSequence_InitType2(&LineStatsType, &trace_stat_structseq_desc) < 0) return -1; @@ -2024,14 +2002,14 @@ parse_trace(PyObject *pytrace, tracemall for (i=0; i < nframe; i++) { pyframe = PyList_GET_ITEM(traceback, i); assert(pyframe != NULL); - if (Py_TYPE(pyframe) != &FrameType) { - PyErr_SetString(PyExc_TypeError, "frames must be Frame instances"); + if (!PyTuple_Check(pyframe) || Py_SIZE(pyframe) != 2) { + PyErr_SetString(PyExc_TypeError, "frames must be 2-tuples"); return -1; } - filename = PyStructSequence_GET_ITEM(pyframe, 0); + filename = PyTuple_GET_ITEM(pyframe, 0); assert(filename != NULL); - pylineno = PyStructSequence_GET_ITEM(pyframe, 1); + pylineno = PyTuple_GET_ITEM(pyframe, 1); assert(pylineno != NULL); if (tracemalloc_parse_lineno(pylineno, &lineno) == 0) return -1; @@ -2336,21 +2314,21 @@ tracemalloc_frame_to_pyobject(PyObject * { PyObject *frame_obj, *lineno_obj; - frame_obj = PyStructSequence_New(&FrameType); + frame_obj = PyTuple_New(2); if (frame_obj == NULL) return NULL; if (filename == NULL) filename = Py_None; Py_INCREF(filename); - PyStructSequence_SET_ITEM(frame_obj, 0, filename); + PyTuple_SET_ITEM(frame_obj, 0, filename); lineno_obj = tracemalloc_lineno_as_obj(lineno); if (lineno_obj == NULL) { Py_DECREF(frame_obj); return NULL; } - PyStructSequence_SET_ITEM(frame_obj, 1, lineno_obj); + PyTuple_SET_ITEM(frame_obj, 1, lineno_obj); return frame_obj; } @@ -3081,9 +3059,6 @@ PyInit__tracemalloc(void) Py_INCREF((PyObject*) &FilterType); PyModule_AddObject(m, "Filter", (PyObject*)&FilterType); - Py_INCREF((PyObject*) &FrameType); - PyModule_AddObject(m, "Frame", (PyObject*)&FrameType); - Py_INCREF((PyObject*) &LineStatsType); PyModule_AddObject(m, "TraceStats", (PyObject*)&LineStatsType); changeset: 85565:07e39c3e46a4 user: Victor Stinner date: Wed Sep 18 00:27:03 2013 +0200 files: Lib/test/test_tracemalloc.py Lib/tracemalloc.py Modules/_tracemalloc.c description: Remove the Trace type diff -r 255e8ad9a705 -r 07e39c3e46a4 Lib/test/test_tracemalloc.py --- a/Lib/test/test_tracemalloc.py Wed Sep 18 00:08:34 2013 +0200 +++ b/Lib/test/test_tracemalloc.py Wed Sep 18 00:27:03 2013 +0200 @@ -42,12 +42,13 @@ class TestTracemallocEnabled(unittest.Te def test_get_object_trace(self): tracemalloc.clear_traces() - size = 12345 - obj, obj_frames = allocate_bytes(size) + obj_size = 12345 + obj, obj_frames = allocate_bytes(obj_size) trace = tracemalloc.get_object_trace(obj) - self.assertIsInstance(trace, tracemalloc.Trace) - self.assertEqual(trace.size, size) - self.assertEqual(trace.traceback, obj_frames) + self.assertIsInstance(trace, tuple) + size, traceback = trace + self.assertEqual(size, obj_size) + self.assertEqual(traceback, obj_frames) def test_set_traceback_limit(self): size = 10 @@ -61,28 +62,29 @@ class TestTracemallocEnabled(unittest.Te tracemalloc.set_traceback_limit(1) obj, obj_frames = allocate_bytes(size) trace = tracemalloc.get_object_trace(obj) - self.assertEqual(len(trace.traceback), 1) - self.assertEqual(trace.traceback, obj_frames) + self.assertEqual(len(trace[1]), 1) + self.assertEqual(trace[1], obj_frames) tracemalloc.set_traceback_limit(10) obj2, obj2_frames = allocate_bytes(size) trace = tracemalloc.get_object_trace(obj2) - self.assertEqual(len(trace.traceback), 10) - self.assertEqual(trace.traceback, obj2_frames) + self.assertEqual(len(trace[1]), 10) + self.assertEqual(trace[1], obj2_frames) def test_get_traces(self): tracemalloc.clear_traces() - size = 12345 - obj, obj_frames = allocate_bytes(size) + obj_size = 12345 + obj, obj_frames = allocate_bytes(obj_size) traces = tracemalloc.get_traces() address = tracemalloc.get_object_address(obj) self.assertIn(address, traces) trace = traces[address] - self.assertIsInstance(trace, tracemalloc.Trace) - self.assertEqual(trace.size, size) - self.assertEqual(trace.traceback, obj_frames) + self.assertIsInstance(trace, tuple) + size, traceback = trace + self.assertEqual(size, obj_size) + self.assertEqual(traceback, obj_frames) def test_get_process_memory(self): tracemalloc.clear_traces() @@ -280,16 +282,16 @@ class TestTracemallocEnabled(unittest.Te tracemalloc.add_include_filter('should never match 2') tracemalloc.add_include_filter(__file__) tracemalloc.clear_traces() - size = 1000 - obj, obj_frames = allocate_bytes(size) + obj_size = 1000 + obj, obj_frames = allocate_bytes(obj_size) trace = tracemalloc.get_object_trace(obj) - self.assertIsInstance(trace, tracemalloc.Trace) + self.assertIsNotNone(trace) # test exclusive filter, based on previous filters filename, lineno = obj_frames[0] tracemalloc.add_exclude_filter(filename, lineno) tracemalloc.clear_traces() - obj, obj_frames = allocate_bytes(size) + obj, obj_frames = allocate_bytes(obj_size) trace = tracemalloc.get_object_trace(obj) self.assertIsNone(trace) @@ -545,13 +547,8 @@ class TestFilters(unittest.TestCase): "too many joker characters in the filename pattern") def test_filter_match_trace(self): - t1 = tracemalloc.Trace((123, [ - ("a.py", 2), - ("b.py", 3)])) - - t2 = tracemalloc.Trace((123, [ - ("b.py", 4), - ("b.py", 5)])) + t1 = (123, [("a.py", 2), ("b.py", 3)]) + t2 = (123, [("b.py", 4), ("b.py", 5)]) f = tracemalloc.Filter(True, "b.py", traceback=True) self.assertTrue(f.match_trace(t1)) @@ -647,15 +644,15 @@ class TestTop(unittest.TestCase): timestamp = datetime.datetime(2013, 9, 12, 15, 16, 17) traces = { - 0x10001: tracemalloc.Trace((10, traceback_a_2)), - 0x10002: tracemalloc.Trace((10, traceback_a_2)), - 0x10003: tracemalloc.Trace((10, traceback_a_2)), + 0x10001: (10, traceback_a_2), + 0x10002: (10, traceback_a_2), + 0x10003: (10, traceback_a_2), - 0x20001: tracemalloc.Trace((2, traceback_a_5)), + 0x20001: (2, traceback_a_5), - 0x30001: tracemalloc.Trace((66, traceback_b_1)), + 0x30001: (66, traceback_b_1), - 0x40001: tracemalloc.Trace((2, traceback_none_none)), + 0x40001: (2, traceback_none_none), } tracemalloc_size = 100 process_memory = tracemalloc._meminfo(1024, 2048) @@ -665,14 +662,14 @@ class TestTop(unittest.TestCase): timestamp2 = datetime.datetime(2013, 9, 12, 15, 16, 50) traces2 = { - 0x10001: tracemalloc.Trace((10, traceback_a_2)), - 0x10002: tracemalloc.Trace((10, traceback_a_2)), - 0x10003: tracemalloc.Trace((10, traceback_a_2)), + 0x10001: (10, traceback_a_2), + 0x10002: (10, traceback_a_2), + 0x10003: (10, traceback_a_2), - 0x20001: tracemalloc.Trace((2, traceback_a_5)), - 0x20002: tracemalloc.Trace((5000, traceback_a_5)), + 0x20001: (2, traceback_a_5), + 0x20002: (5000, traceback_a_5), - 0x30001: tracemalloc.Trace((66, traceback_b_1)), + 0x30001: (66, traceback_b_1), } process_memory2 = tracemalloc._meminfo(1500, 2048) tracemalloc_size2 = 200 diff -r 255e8ad9a705 -r 07e39c3e46a4 Lib/tracemalloc.py --- a/Lib/tracemalloc.py Wed Sep 18 00:08:34 2013 +0200 +++ b/Lib/tracemalloc.py Wed Sep 18 00:27:03 2013 +0200 @@ -446,11 +446,13 @@ def _lazy_import_pickle(): import pickle return pickle +_UNKNOWN_FRAME = (None, None) + def _get_first_frame(trace): - if trace.traceback: - return trace.traceback[0] + if trace[1]: + return trace[1][0] else: - return (None, None) + return _UNKNOWN_FRAME def _compute_stats_frame(stats, group_per_file, size, frame): if not group_per_file: @@ -645,7 +647,7 @@ class Snapshot: raise ValueError("need traces") for address, trace in self.traces.items(): - stats[address] = TraceStats((trace.size, 1)) + stats[address] = TraceStats((trace[0], 1)) else: if group_by == 'filename': group_per_file = True @@ -669,14 +671,15 @@ class Snapshot: stats[key] = TraceStats((size, count)) else: for trace in self.traces.values(): - if trace.traceback: + size, traceback = trace + if traceback: if cumulative: - for frame in trace.traceback: - _compute_stats_frame(stats, group_per_file, trace.size, frame) + for frame in traceback: + _compute_stats_frame(stats, group_per_file, size, frame) else: - _compute_stats_frame(stats, group_per_file, trace.size, trace.traceback[0]) + _compute_stats_frame(stats, group_per_file, size, traceback[0]) else: - _compute_stats_frame(stats, group_per_file, trace.size, None) + _compute_stats_frame(stats, group_per_file, size, None) return GroupedStats(stats, group_by, cumulative, self.timestamp, diff -r 255e8ad9a705 -r 07e39c3e46a4 Modules/_tracemalloc.c --- a/Modules/_tracemalloc.c Wed Sep 18 00:08:34 2013 +0200 +++ b/Modules/_tracemalloc.c Wed Sep 18 00:27:03 2013 +0200 @@ -1601,29 +1601,6 @@ tracemalloc_atfork(void) } #endif -PyDoc_STRVAR(tracemalloc_structseq__doc__, -"Trace: debug information of an allocated memory block."); - -static PyStructSequence_Field tracemalloc_structseq_fields[] = { - {"size", "size in bytes of the memory block"}, - {"traceback", "traceback where the memory block was allocated as " - "a list of tracemalloc.frame instances (most recent first). " - "The list can be empty or incomplete if the tracemalloc " - "module was unable to retrieve the full traceback. " - "For efficiency, the traceback is truncated to 10 frames."}, - {0} -}; - -static PyStructSequence_Desc tracemalloc_structseq_desc = { - "tracemalloc.Trace", /* name */ - tracemalloc_structseq__doc__, /* doc */ - tracemalloc_structseq_fields, - 2 -}; - -static PyTypeObject TraceType; - - PyDoc_STRVAR(trace_stat_structseq__doc__, "TraceStats: statistics on Python memory allocations\n" "of a specific line number."); @@ -1653,9 +1630,6 @@ tracemalloc_init(void) if (tracemalloc_config.init) return 0; - if (PyStructSequence_InitType2(&TraceType, - &tracemalloc_structseq_desc) < 0) - return -1; if (PyStructSequence_InitType2(&LineStatsType, &trace_stat_structseq_desc) < 0) return -1; @@ -1979,13 +1953,18 @@ parse_trace(PyObject *pytrace, tracemall Py_ssize_t nframe, i; int lineno; - pysize = PyStructSequence_GET_ITEM(pytrace, 0); + if (!PyTuple_Check(pytrace) || Py_SIZE(pytrace) != 2) { + PyErr_SetString(PyExc_TypeError, "a trace must be a 2-tuple"); + return NULL; + } + + pysize = PyTuple_GET_ITEM(pytrace, 0); assert(pysize != NULL); trace->size = PyLong_AsSsize_t(pysize); if (trace->size == -1 && PyErr_Occurred()) return -1; - traceback = PyStructSequence_GET_ITEM(pytrace, 1); + traceback = PyTuple_GET_ITEM(pytrace, 1); assert(traceback != NULL); if (!PyList_Check(traceback)) { PyErr_SetString(PyExc_TypeError, "traceback must be a list"); @@ -2031,8 +2010,7 @@ tracemalloc_pyfilter_match_trace(PyObjec char stack_buffer[ TRACE_SIZE(MAX_NFRAME) ]; tracemalloc_trace_t *trace; - if (!PyArg_ParseTuple(args, "O!:match_trace", - &TraceType, &pytrace)) + if (!PyArg_ParseTuple(args, "O:match_trace", &pytrace)) return NULL; trace = (tracemalloc_trace_t *)stack_buffer; @@ -2340,7 +2318,7 @@ tracemalloc_trace_to_pyobject(tracemallo PyObject *size, *frames, *frame; int i; - trace_obj = PyStructSequence_New(&TraceType); + trace_obj = PyTuple_New(2); if (trace_obj == NULL) return NULL; @@ -2349,7 +2327,7 @@ tracemalloc_trace_to_pyobject(tracemallo Py_DECREF(trace_obj); return NULL; } - PyStructSequence_SET_ITEM(trace_obj, 0, size); + PyTuple_SET_ITEM(trace_obj, 0, size); frames = PyList_New(trace->nframe); if (frames == NULL) { @@ -2366,7 +2344,7 @@ tracemalloc_trace_to_pyobject(tracemallo } PyList_SET_ITEM(frames, i, frame); } - PyStructSequence_SET_ITEM(trace_obj, 1, frames); + PyTuple_SET_ITEM(trace_obj, 1, frames); return trace_obj; } @@ -3053,9 +3031,6 @@ PyInit__tracemalloc(void) if (PyType_Ready(&FilterType) < 0) return NULL; - Py_INCREF((PyObject*) &TraceType); - PyModule_AddObject(m, "Trace", (PyObject*)&TraceType); - Py_INCREF((PyObject*) &FilterType); PyModule_AddObject(m, "Filter", (PyObject*)&FilterType); changeset: 85566:ade672d5b5ba tag: tip user: Victor Stinner date: Wed Sep 18 01:15:16 2013 +0200 files: Lib/tracemalloc.py Modules/_tracemalloc.c PERF description: benchmark diff -r 07e39c3e46a4 -r ade672d5b5ba Lib/tracemalloc.py --- a/Lib/tracemalloc.py Wed Sep 18 00:27:03 2013 +0200 +++ b/Lib/tracemalloc.py Wed Sep 18 01:15:16 2013 +0200 @@ -503,12 +503,20 @@ class Snapshot: raise ValueError("with_traces or with_stats must be True") timestamp = datetime.datetime.now() pid = os.getpid() + print("Snapshot.create", file=sys.stderr) + print(get_filters(), file=sys.stderr) if with_traces: + t0 = _time_monotonic() traces = get_traces() + dt = _time_monotonic() - t0 + print("get_traces", dt, file=sys.stderr) else: traces = None if with_stats: + t0 = _time_monotonic() stats = get_stats() + dt = _time_monotonic() - t0 + print("get_stats", dt, file=sys.stderr) else: stats = None tracemalloc_size = get_tracemalloc_size() @@ -695,9 +703,12 @@ class TakeSnapshot: self.user_data_callback = None def take_snapshot(self): + t0 = _time_monotonic() snapshot = Snapshot.create(with_traces=self.with_traces, with_stats=self.with_stats, user_data_callback=self.user_data_callback) + dt = _time_monotonic() - t0 + print("create snapshot", dt, file=sys.stderr) filename = self.filename_template filename = filename.replace("$pid", str(snapshot.pid)) @@ -708,7 +719,11 @@ class TakeSnapshot: filename = filename.replace("$counter", "%04i" % self.counter) + t0 = _time_monotonic() snapshot.write(filename) + dt = _time_monotonic() - t0 + print("write snapshot", dt, file=sys.stderr) + self.counter += 1 return snapshot, filename @@ -979,4 +994,4 @@ if __name__ == "__main__": else: main() -add_exclude_filter(__file__) +#add_exclude_filter(__file__) diff -r 07e39c3e46a4 -r ade672d5b5ba Modules/_tracemalloc.c --- a/Modules/_tracemalloc.c Wed Sep 18 00:27:03 2013 +0200 +++ b/Modules/_tracemalloc.c Wed Sep 18 01:15:16 2013 +0200 @@ -583,6 +583,7 @@ struct { typedef struct { /* include (1) or exclude (0) matching frame? */ int include; + Py_hash_t pattern_hash; PyObject *pattern; #ifndef TRACE_NORMALIZE_FILENAME int use_joker; @@ -929,7 +930,7 @@ tracemalloc_update_stats(tracemalloc_tra assert(trace_stats->count != 0 || trace_stats->size == 0); if (trace_stats->count == 0) { - assert(!tracemalloc_config.reentrant); + /* assert(!tracemalloc_config.reentrant); */ cfuhash_delete_data(line_hash, line_key); if (line_hash->entries == 0) @@ -1075,7 +1076,13 @@ tracemalloc_match_filename(tracemalloc_f #ifndef TRACE_NORMALIZE_FILENAME if (!filter->use_joker) { if (!tracemalloc_endswith_pyc_pyo(filename)) { - int res = PyUnicode_Compare(filename, filter->pattern); + int res; + Py_hash_t hash = PyObject_Hash(filename); + + if (hash != filter->pattern_hash) + return 0; + + res = PyUnicode_Compare(filename, filter->pattern); return (res == 0); } else { @@ -1470,6 +1477,7 @@ tracemalloc_filter_init(tracemalloc_filt pattern = new_pattern; filter->include = include; + filter->pattern_hash = PyObject_Hash(pattern); filter->pattern = pattern; #ifndef TRACE_NORMALIZE_FILENAME filter->use_joker = (njoker != 0); @@ -1955,7 +1963,7 @@ parse_trace(PyObject *pytrace, tracemall if (!PyTuple_Check(pytrace) || Py_SIZE(pytrace) != 2) { PyErr_SetString(PyExc_TypeError, "a trace must be a 2-tuple"); - return NULL; + return -1; } pysize = PyTuple_GET_ITEM(pytrace, 0); @@ -2354,6 +2362,47 @@ typedef struct { PyObject *dict; } tracemalloc_get_traces_t; +static PyObject* +tracemalloc_trace_to_pyobject2(tracemalloc_trace_t *trace, PyObject *key_obj) +{ + PyObject *trace_obj = NULL; + PyObject *size, *frames, *frame; + int i; + + trace_obj = PyTuple_New(3); + if (trace_obj == NULL) + return NULL; + + Py_INCREF(key_obj); + PyTuple_SET_ITEM(trace_obj, 0, key_obj); + + size = PyLong_FromSize_t(trace->size); + if (size == NULL) { + Py_DECREF(trace_obj); + return NULL; + } + PyTuple_SET_ITEM(trace_obj, 1, size); + + frames = PyTuple_New(trace->nframe); + if (frames == NULL) { + Py_DECREF(trace_obj); + return NULL; + } + for (i=0; i < trace->nframe; i++) { + frame = tracemalloc_frame_to_pyobject(TRACE_FILENAMES(trace)[i], + TRACE_LINENOS(trace, trace->nframe)[i]); + if (frame == NULL) { + Py_DECREF(trace_obj); + Py_DECREF(frames); + return NULL; + } + PyTuple_SET_ITEM(frames, i, frame); + } + PyTuple_SET_ITEM(trace_obj, 2, frames); + + return trace_obj; +} + static int tracemalloc_get_traces_fill(cfuhash_entry *entry, void *user_data) { @@ -2370,14 +2419,13 @@ tracemalloc_get_traces_fill(cfuhash_entr if (key_obj == NULL) return 1; - tracemalloc_obj = tracemalloc_trace_to_pyobject(trace); + tracemalloc_obj = tracemalloc_trace_to_pyobject2(trace, key_obj); + Py_DECREF(key_obj); if (tracemalloc_obj == NULL) { - Py_DECREF(key_obj); return 1; } - res = PyDict_SetItem(get_traces->dict, key_obj, tracemalloc_obj); - Py_DECREF(key_obj); + res = PyList_Append(get_traces->dict, tracemalloc_obj); Py_DECREF(tracemalloc_obj); if (res < 0) return 1; @@ -2397,25 +2445,33 @@ py_tracemalloc_get_traces(PyObject *self tracemalloc_get_traces_t get_traces; int err; - get_traces.dict = PyDict_New(); - if (get_traces.dict == NULL) - return NULL; - - if (!tracemalloc_config.enabled) - return get_traces.dict; - TRACE_LOCK(); get_traces.tracemalloc_allocs = cfuhash_copy(tracemalloc_allocs); TRACE_UNLOCK(); + get_traces.dict = PyList_New(get_traces.tracemalloc_allocs->entries); + if (get_traces.dict == NULL) + return NULL; + Py_SIZE(get_traces.dict) = 0; + + if (!tracemalloc_config.enabled) + return get_traces.dict; + if (get_traces.tracemalloc_allocs == NULL) { Py_DECREF(get_traces.dict); PyErr_NoMemory(); return NULL; } +#if 1 + tracemalloc_config.reentrant = 1; err = cfuhash_foreach(get_traces.tracemalloc_allocs, tracemalloc_get_traces_fill, &get_traces); + tracemalloc_config.reentrant = 0; +#else + err = cfuhash_foreach(get_traces.tracemalloc_allocs, + tracemalloc_get_traces_fill, &get_traces); +#endif if (err) Py_CLEAR(get_traces.dict); diff -r 07e39c3e46a4 -r ade672d5b5ba PERF --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/PERF Wed Sep 18 01:15:16 2013 +0200 @@ -0,0 +1,32 @@ +Original +======== + +create snapshot 12.497519316035323 +write snapshot 8.971860605990514 +2013-09-18 00:37:44: Write a snapshot of memory allocations into /tmp/tracemalloc-2019-0002.pickle (21.5 sec) + +Load snapshot /tmp/tracemalloc-2019-0002.pickle (with traces) +Load snapshot /tmp/tracemalloc-2019-0002.pickle: 382 files, 203513 traces (limit=15 frames) (9.5 sec) + + +no Frame +======== + +create snapshot 10.965272768982686 +write snapshot 7.1928553459001705 +2013-09-18 00:29:49: Write a snapshot of memory allocations into /tmp/tracemalloc-1687-0002.pickle (18.2 sec) + +Load snapshot /tmp/tracemalloc-1687-0002.pickle (with traces) +Load snapshot /tmp/tracemalloc-1687-0002.pickle: 378 files, 188605 traces (limit=15 frames) (4.0 sec) + + +no Frame no Trace +================= + +create snapshot 12.997173686046153 +write snapshot 3.1387799030635506 +2013-09-18 00:32:02: Write a snapshot of memory allocations into /tmp/tracemalloc-1785-0002.pickle (16.1 sec) + +Load snapshot /tmp/tracemalloc-1785-0002.pickle (with traces) +Load snapshot /tmp/tracemalloc-1785-0002.pickle: 378 files, 204409 traces (limit=15 frames) (3.2 sec) +