branch: elpa/buttercup commit cccdedff38208ad4aa989ccdab8e0b059adf3728 Merge: b4e0986 dacfacc Author: Ola Nilsson <ola.nils...@gmail.com> Commit: GitHub <nore...@github.com>
Merge pull request #197 from snogge/omit-traceback New traceback style 'omit' --- Makefile | 2 +- bin/buttercup | 21 +++-- bin/buttercup.bat | 21 +++-- buttercup.el | 68 ++++++++++---- tests/test-buttercup.el | 242 ++++++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 299 insertions(+), 55 deletions(-) diff --git a/Makefile b/Makefile index 4a543cf..276f24c 100644 --- a/Makefile +++ b/Makefile @@ -25,4 +25,4 @@ release: clean test tar -c $(DISTFILES) --transform "s,^,buttercup-$(VERSION)/," --transform 's/README.md/README.txt/' > "dist/buttercup-$(VERSION).tar" clean: - rm -f *.elc + rm -f *.elc tests/*.elc diff --git a/bin/buttercup b/bin/buttercup index 601a8e0..2230e84 100755 --- a/bin/buttercup +++ b/bin/buttercup @@ -47,16 +47,17 @@ Buttercup options: --traceback STYLE When printing backtraces for errors that occur during tests, print them in the chosen - STYLE. Available styles are "full", which - shows the full function call for each stack - frame on a single line, "crop", which - truncates each stack frame to 80 characters - (the default), and "pretty", which uses - Emacs' pretty-printing facilities to print - each stack frame, and also annotates each - frame with a lambda or M to indicate whether - it is a normal function call or a - macro/special form. + STYLE. Available styles are + "full", which shows the full function call for + each stack frame on a single line, + "crop", which truncates each stack frame to 80 + characters (the default), + "pretty", which uses Emacs' pretty-printing + facilities to print each stack frame, and also + annotates each frame with a lambda or M to + indicate whether it is a normal function call + or a macro/special form and + "omit", which omits the backtraces alltogether. --stale-file-error Fail the test run if stale .elc files are loaded. EOF diff --git a/bin/buttercup.bat b/bin/buttercup.bat index edff81d..108c910 100644 --- a/bin/buttercup.bat +++ b/bin/buttercup.bat @@ -51,16 +51,17 @@ echo --no-color, -c Do not colorize test output. echo. echo --traceback STYLE When printing backtraces for errors that occur echo during tests, print them in the chosen -echo STYLE. Available styles are "full", which -echo shows the full function call for each stack -echo frame on a single line, "crop", which -echo truncates each stack frame to 80 characters -echo ^(the default^), and "pretty", which uses -echo Emacs' pretty-printing facilities to print -echo each stack frame, and also annotates each -echo frame with a lambda or M to indicate whether -echo it is a normal function call or a -echo macro/special form. +echo STYLE. Available styles are +echo "full", which shows the full function call for +echo each stack frame on a single line, +echo "crop", which truncates each stack frame to 80 +echo characters (the default), +echo "pretty", which uses Emacs' pretty-printing +echo facilities to print each stack frame, and also +echo annotates each frame with a lambda or M to +echo indicate whether it is a normal function call +echo or a macro/special form and +echo "omit", which omits the backtraces alltogether. echo. echo --stale-file-error Fail the test run if stale .elc files are loaded. exit /b diff --git a/buttercup.el b/buttercup.el index 220ba08..ed09d5c 100644 --- a/buttercup.el +++ b/buttercup.el @@ -56,7 +56,9 @@ The function MUST have one of the following forms: \(lambda () EXPR) +\(lambda () (buttercup--mark-stackframe) EXPR) \(closure (ENVLIST) () EXPR) +\(closure (ENVLIST) () (buttercup--mark-stackframe) EXPR) \(lambda () (quote EXPR) EXPR) \(closure (ENVLIST) () (quote EXPR) EXPR) @@ -65,12 +67,14 @@ forms are useful if EXPR is a macro call, in which case the `quote' ensures access to the un-expanded form." (pcase fun (`(closure ,(pred listp) nil ,expr) expr) + (`(closure ,(pred listp) nil (buttercup--mark-stackframe) ,expr) expr) (`(closure ,(pred listp) nil (quote ,expr) . ,_rest) expr) (`(closure ,(pred listp) nil ,_expr . ,(pred identity)) (error "Closure contains multiple expressions: %S" fun)) (`(closure ,(pred listp) ,(pred identity) . ,(pred identity)) (error "Closure has nonempty arglist: %S" fun)) (`(lambda nil ,expr) expr) + (`(lambda nil (buttercup--mark-stackframe) ,expr) expr) (`(lambda nil (quote ,expr) . ,_rest) expr) (`(lambda nil ,_expr . ,(pred identity)) (error "Function contains multiple expressions: %S" fun)) @@ -132,9 +136,16 @@ This macro knows three forms: \(expect ARG) Fail the current test if ARG is not true." (let ((wrapped-args - (mapcar (lambda (expr) `(lambda () (quote ,expr) ,expr)) args))) + (mapcar (lambda (expr) `(lambda () + (quote ,expr) + (buttercup--mark-stackframe) + ,expr)) + args))) `(buttercup-expect - (lambda () (quote ,arg) ,arg) + (lambda () + (quote ,arg) + (buttercup--mark-stackframe) + ,arg) ,(or matcher :to-be-truthy) ,@wrapped-args))) @@ -885,6 +896,7 @@ most probably including one or more calls to `expect'." `(buttercup-it ,description (lambda () (buttercup-with-converted-ert-signals + (buttercup--mark-stackframe) ,@body))) `(buttercup-xit ,description))) @@ -1756,10 +1768,12 @@ Finally print the elapsed time for SPEC." (buttercup-colorize (concat " " failure) color))) (buttercup--print " (%s)\n" (buttercup-elapsed-time-string spec)))) -(defun buttercup-reporter-batch--print-failed-spec-report (failed-spec color) +(cl-defun buttercup-reporter-batch--print-failed-spec-report (failed-spec color) "Print a failure report for FAILED-SPEC. Colorize parts of the output if COLOR is non-nil." + (when (eq buttercup-stack-frame-style 'omit) + (cl-return-from buttercup-reporter-batch--print-failed-spec-report)) (let ((description (buttercup-spec-failure-description failed-spec)) (stack (buttercup-spec-failure-stack failed-spec)) (full-name (buttercup-spec-full-name failed-spec))) @@ -1780,7 +1794,7 @@ Colorize parts of the output if COLOR is non-nil." "FAILED") description)) ((and (consp description) (eq (car description) 'error)) - (buttercup--print "%S: %S\n\n" + (buttercup--print "%S: %S\n" (car description) (cadr description))) (t @@ -1924,31 +1938,49 @@ ARGS according to `debugger'." (throw 'buttercup-debugger-continue (list 'failed args (buttercup--backtrace)))) +(defalias 'buttercup--mark-stackframe 'ignore + "Marker to find where the backtrace start.") + (defun buttercup--backtrace () "Create a backtrace, a list of frames returned from `backtrace-frame'." - (let* ((n 0) - (frame (backtrace-frame n)) - (frame-list nil) - (in-program-stack nil)) - (while frame + ;; Read the backtrace frames from 0 (the closest) upward. + (cl-do* ((n 0 (1+ n)) + (frame (backtrace-frame n) (backtrace-frame n)) + (frame-list nil) + (in-program-stack nil)) + ((not frame) frame-list) + ;; discard frames until (and including) `buttercup--debugger', they + ;; only contain buttercup code (when in-program-stack (push frame frame-list)) (when (eq (elt frame 1) 'buttercup--debugger) (setq in-program-stack t)) - (when (eq (elt frame 1) - 'buttercup--funcall) - (setq in-program-stack nil - frame-list (nthcdr 6 frame-list))) - (setq n (1+ n) - frame (backtrace-frame n))) - frame-list)) + ;; keep frames until one of the known functions are found, after + ;; this is just the buttercup framework and not interesting for + ;; users incorrect for testing buttercup. Some frames before the + ;; function also have to be discarded + (cl-labels ((tree-find (key tree) + (cl-block tree-find + (while (consp tree) + (let ((elem (pop tree))) + (when (or (and (consp elem) + (tree-find key elem)) + (eql key elem)) + (cl-return-from tree-find t)))) + (cl-return-from tree-find + (and tree (eql tree key)))))) + (when (and in-program-stack (tree-find 'buttercup--mark-stackframe frame)) + (pop frame-list) + (cl-return frame-list))))) (defun buttercup--format-stack-frame (frame &optional style) "Format stack FRAME according to STYLE. -STYLE can be one of `full', `crop', or `pretty'. +STYLE can be one of `full', `crop', `pretty', or `omit'. If STYLE is nil, use `buttercup-stack-frame-style' or `crop'." - (pcase (or style buttercup-stack-frame-style 'crop) + (setq style (or style buttercup-stack-frame-style 'crop)) + (pcase style + (`omit) ; needed to verify valid styles (`full (format " %S" (cdr frame))) (`crop (let ((line (buttercup--format-stack-frame frame 'full))) diff --git a/tests/test-buttercup.el b/tests/test-buttercup.el index cdf493a..9455e49 100644 --- a/tests/test-buttercup.el +++ b/tests/test-buttercup.el @@ -28,6 +28,7 @@ (require 'ansi-color) (require 'ert) (require 'cl-lib) +(require 'imenu) (defun make-list-of-closures (items) "For each element of ITEMS, return a closure returning it." @@ -39,31 +40,33 @@ "Execute BODY with local buttercup state variables. Keyword arguments kan be used to override the values of certain variables: - :color -> `buttercup-color' - :reporter -> `buttercup-reporter' - :suites -> `buttercup-suites' - :quiet -> `buttercup-reporter-batch-quiet-statuses' + :color -> `buttercup-color' + :frame-style -> `buttercup-stack-frame-style' + :reporter -> `buttercup-reporter' + :suites -> `buttercup-suites' + :quiet -> `buttercup-reporter-batch-quiet-statuses' \n(fn &keys COLOR SUITES REPORTER &rest BODY)" (declare (debug t) (indent defun)) ;; extract keyword arguments (let ((keys '(:color buttercup-color + :frame-style buttercup-stack-frame-style :reporter buttercup-reporter :suites buttercup-suites :quiet buttercup-reporter-batch-quiet-statuses)) extra-vars) (while (plist-member keys (car body)) (push (list (plist-get keys (pop body)) (pop body)) extra-vars)) - `(let (buttercup--after-all - buttercup--after-each - buttercup--before-all + `(let (buttercup--after-each buttercup--before-each (buttercup--cleanup-functions :invalid) buttercup--current-suite (buttercup-reporter #'ignore) buttercup-suites + buttercup-color buttercup-reporter-batch-quiet-statuses buttercup-reporter-batch--suite-stack buttercup-reporter-batch--failures + (buttercup-stack-frame-style 'crop) (buttercup-warning-buffer-name " *ignored buttercup warnings*") ,@(nreverse extra-vars)) ,@body))) @@ -133,8 +136,10 @@ text properties using `ansi-color-apply'." (expect (length expansion) :to-equal 4) (expect (nth 0 expansion) :to-be 'buttercup-expect) (expect (functionp (nth 1 expansion))) + (expect (buttercup--wrapper-fun-p (nth 1 expansion))) (expect (nth 2 expansion) :to-be :to-equal) - (expect (functionp (nth 3 expansion))))) + (expect (functionp (nth 3 expansion))) + (expect (buttercup--wrapper-fun-p (nth 3 expansion))))) (it "with no matcher should use `:to-be-truthy' as the matcher" (let ((expansion (macroexpand '(expect (equal (+ 1 1) 2))))) @@ -299,7 +304,7 @@ text properties using `ansi-color-apply'." (buttercup-suite-add-child grandparent parent) (buttercup-suite-add-child parent child) - (expect (buttercup-suite-parents child) + (expect (buttercup-suite-or-spec-parents child) :to-equal (list parent grandparent))))) @@ -311,7 +316,7 @@ text properties using `ansi-color-apply'." (buttercup-suite-add-child grandparent parent) (buttercup-suite-add-child parent child) - (expect (buttercup-spec-parents child) + (expect (buttercup-suite-or-spec-parents child) :to-equal (list parent grandparent))))) @@ -557,6 +562,7 @@ text properties using `ansi-color-apply'." '(buttercup-it "description" (lambda () (buttercup-with-converted-ert-signals + (buttercup--mark-stackframe) body))))) (it "without argument should expand to xit." @@ -835,7 +841,7 @@ text properties using `ansi-color-apply'." (let ((suite (describe "A bad spy scope" (before-all (spy-on 'some-function))))) - (expect (run--suite suite) + (expect (buttercup--run-suite suite) :to-throw)))) (it "used directly in describe" (with-local-buttercup @@ -1290,9 +1296,9 @@ text properties using `ansi-color-apply'." (before-each (setq defined-specs 10 pending-specs 0 failed-specs 0) - (spy-on 'buttercup-suites-total-specs-defined :and-call-fake (lambda (&rest a) defined-specs)) - (spy-on 'buttercup-suites-total-specs-pending :and-call-fake (lambda (&rest a) pending-specs)) - (spy-on 'buttercup-suites-total-specs-failed :and-call-fake (lambda (&rest a) failed-specs))) + (spy-on 'buttercup-suites-total-specs-defined :and-call-fake (lambda (&rest _) defined-specs)) + (spy-on 'buttercup-suites-total-specs-pending :and-call-fake (lambda (&rest _) pending-specs)) + (spy-on 'buttercup-suites-total-specs-failed :and-call-fake (lambda (&rest _) failed-specs))) (it "should print a summary of run and failing specs" (setq failed-specs 6) @@ -1343,7 +1349,6 @@ text properties using `ansi-color-apply'." (with-local-buttercup :color nil (expect (buttercup-reporter-batch 'buttercup-done (list spec)) :not :to-throw))) - ;; TODO: Backtrace tests ) (describe "on an unknown event" @@ -1351,6 +1356,211 @@ text properties using `ansi-color-apply'." (expect (buttercup-reporter-batch 'unknown-event nil) :to-throw))))) +(describe "Backtraces" + :var (print-buffer) + ;; redirect output to a buffer + (before-each + (setq print-buffer (generate-new-buffer "*btrcp-reporter-test*")) + (spy-on 'send-string-to-terminal :and-call-fake + (apply-partially #'send-string-to-ansi-buffer print-buffer)) + ;; Convenience function + (spy-on 'buttercup-output :and-call-fake + (lambda () + "Return the text of print-buffer." + (with-current-buffer print-buffer + (buffer-string))))) + (after-each + (kill-buffer print-buffer) + (setq print-buffer nil)) + ;; define a buttercup-reporter-batch variant that only outputs on + ;; buttercup-done + (before-each + (spy-on 'backtrace-reporter :and-call-fake + (lambda (event arg) + (if (eq event 'buttercup-done) + (buttercup-reporter-batch event arg) + (cl-letf (((symbol-function 'buttercup--print) #'ignore)) + (buttercup-reporter-batch event arg)))))) + ;; suppress the summary line + (before-each + (spy-on 'buttercup-reporter-batch--print-summary)) + ;; define a known backtrace with a typical error + (before-all + (defun bc-bt-foo (a) (bc-bt-bar a)) + (defun bc-bt-bar (a) (bc-bt-baz a)) + (defun bc-bt-baz (a) + (or (number-or-marker-p a) + (signal 'wrong-type-argument `(number-or-marker-p ,a))))) + (after-all + (fmakunbound 'bc-bt-foo) + (fmakunbound 'bc-bt-bar) + (fmakunbound 'bc-bt-baz)) + (it "should be printed for each failed spec" + (with-local-buttercup + :reporter #'backtrace-reporter + (describe "suite" + (it "expect 2" (expect (+ 1 2) :to-equal 2)) + (it "expect nil" (expect nil))) + (buttercup-run :noerror)) + (expect (buttercup-output) :to-match + (rx string-start + (= 2 (seq (= 40 ?=) "\n" + "suite expect " (or "2" "nil") "\n" + "\n" + "Traceback (most recent call last):\n" + (* (seq " " (+ not-newline) "\n")) + (or "FAILED" "error") ": " (+ not-newline) "\n\n")) + string-end))) + (describe "with style" + :var (test-suites long-string) + ;; Set up tests to test + (before-each + (setq long-string + ;; It's important that this string doesn't contain any + ;; regex special characters, it's used in a `rx' `eval' + ;; form that will escape them. Later Emacsen have + ;; `literal' that is much easier to use. + "a string that will be truncated in backtrace crop, at least 70 chars long") + (with-local-buttercup + (describe "suite" + (it "bc-bt-backtrace" + (expect + (bc-bt-foo long-string) + :to-be-truthy))) + (setq test-suites buttercup-suites))) + (it "`crop' should print truncated lines" + (with-local-buttercup + :suites test-suites :reporter #'backtrace-reporter + :frame-style 'crop + (buttercup-run :noerror) + (setq long-string (truncate-string-to-width long-string 62)) + (expect (buttercup-output) :to-match + (rx-to-string + `(seq + string-start + (= 40 ?=) "\n" + "suite bc-bt-backtrace\n" + "\n" + "Traceback (most recent call last):\n" + " (bc-bt-foo \"" (eval ,long-string) "...\n" + " (bc-bt-bar \"" (eval ,long-string) "...\n" + " (bc-bt-baz \"" (eval ,long-string) "...\n" + (* (seq " " (or (seq (= 74 not-newline) (= 3 ?.)) + (seq (** 0 74 not-newline) (= 3 (not (any ?.))))) "\n")) + "error: (" (* anything) ")\n\n" + string-end))))) + (it "`full' should print full lines" + (with-local-buttercup + :suites test-suites :reporter #'backtrace-reporter + :frame-style 'full + (buttercup-run :noerror) + (expect (buttercup-output) :to-match + (rx-to-string + `(seq + string-start + (= 40 ?=) "\n" + "suite bc-bt-backtrace\n" + "\n" + "Traceback (most recent call last):\n" + " (bc-bt-foo \"" (eval ,long-string) "\")\n" + " (bc-bt-bar \"" (eval ,long-string) "\")\n" + " (bc-bt-baz \"" (eval ,long-string) "\")\n" + (* (seq " " (* not-newline) (= 3 (not (any ?.))) "\n")) + "error: (" (* anything) ")\n\n" + string-end))))) + (it "`pretty' should pretty-print frames" + (with-local-buttercup + :suites test-suites :reporter #'backtrace-reporter + :frame-style 'pretty + (buttercup-run :noerror) + (expect (buttercup-output) :to-match + (rx-to-string + `(seq + string-start + (= 40 ?=) "\n" + "suite bc-bt-backtrace\n" + "\n" + "Traceback (most recent call last):\n" + "λ (bc-bt-foo \"" (regex ,long-string) "\")\n" + "λ (bc-bt-bar \"" (regex ,long-string) "\")\n" + "λ (bc-bt-baz \"" (regex ,long-string) "\")\n" + (* (seq (or ?M ?λ) " (" (* not-newline) ; frame start + (*? (seq "\n " (* not-newline))) ; any number of pp lines + (* not-newline) ")\n")) ;; frame end + "error: (" (* anything) ")\n\n" + string-end))))) + (it "`omit' should print nothing" + (with-local-buttercup + :suites test-suites :reporter #'backtrace-reporter + :frame-style 'omit + (buttercup-run :noerror) + (expect (buttercup-output) :to-equal "")))) + (it "should signal an error for unknown styles" + (let ((buttercup-stack-frame-style 'not-a-valid-style)) + (expect (buttercup--format-stack-frame '(t myfun 1 2)) + :to-throw 'error '("Unknown stack trace style: not-a-valid-style")))) + (describe "should generate correct backtrace for" + (cl-macrolet + ((matcher-spec + (description &rest matcher) + `(it ,description + (with-local-buttercup + :reporter #'backtrace-reporter + (describe "backtrace for" + (it "matcher" + (expect (bc-bt-baz "text") ,@matcher))) + (buttercup-run :noerror) + (expect (buttercup-output) :to-equal + ,(mapconcat + #'identity + `(,(make-string 40 ?=) + "backtrace for matcher" + "" + "Traceback (most recent call last):" + " (bc-bt-baz \"text\")" + ,(concat + " (or (number-or-marker-p a) (signal " + (if (< emacs-major-version 27) + "(quote wrong-type-argument) (list (quot..." + "'wrong-type-argument (list 'number-or-m...")) + " (signal wrong-type-argument (number-or-marker-p \"text\"))" + "error: (wrong-type-argument number-or-marker-p \"text\")" + "" "") "\n")))))) + (matcher-spec "no matcher") + (matcher-spec ":to-be-truthy" :to-be-truthy) + (matcher-spec ":not :to-be-truthy" :not :to-be-truthy) + (matcher-spec ":to-be" :to-be 3) + (matcher-spec ":not :to-be" :not :to-be 3) + (matcher-spec ":to-equal" :to-equal 3) + (matcher-spec ":not :to-equal" :not :to-equal 3) + (matcher-spec ":to-have-same-items-as" :to-have-same-items-as '(3)) + (matcher-spec ":not :to-have-same-items-as" :not :to-have-same-items-as '(3)) + (matcher-spec ":to-match" :to-match ".") + (matcher-spec ":not :to-match" :not :to-match ".") + (matcher-spec ":to-be-in" :to-be-in '(2)) + (matcher-spec ":not :to-be-in" :not :to-be-in '(2)) + (matcher-spec ":to-contain" :to-contain 2) + (matcher-spec ":not :to-contain" :not :to-contain 2) + (matcher-spec ":to-be-less-than" :to-be-less-than 2) + (matcher-spec ":not :to-be-less-than" :not :to-be-less-than 2) + (matcher-spec ":to-be-greater-than" :to-be-greater-than 2) + (matcher-spec ":not :to-be-greater-than" :not :to-be-greater-than 2) + (matcher-spec ":to-be-weakly-less-than" :to-be-weakly-less-than 2) + (matcher-spec ":not :to-be-weakly-less-than" :not :to-be-weakly-less-than 2) + (matcher-spec ":to-be-weakly-greater-than" :to-be-weakly-greater-than 2) + (matcher-spec ":not :to-be-weakly-greater-than" :not :to-be-weakly-greater-than 2) + (matcher-spec ":to-be-close-to" :to-be-close-to 2 0.3) + (matcher-spec ":not :to-be-close-to" :not :to-be-close-to 2 0.2) + ;; (matcher-spec ":to-throw" :to-throw) + ;; (matcher-spec ":not :to-throw" :not :to-throw) + (matcher-spec ":to-have-been-called" :to-have-been-called) + (matcher-spec ":not :to-have-been-called" :not :to-have-been-called) + (matcher-spec ":to-have-been-called-with" :to-have-been-called-with 2) + (matcher-spec ":not :to-have-been-called-with" :not :to-have-been-called-with 2) + (matcher-spec ":to-have-been-called-times" :to-have-been-called-times 2) + (matcher-spec ":not :to-have-been-called-times" :not :to-have-been-called-times 2)))) + + (describe "When using quiet specs in the batch reporter" :var (print-buffer) (before-each @@ -1659,7 +1869,7 @@ text properties using `ansi-color-apply'." :var (el-time elc-time) (before-each (spy-on 'file-attributes :and-call-fake - (lambda (filename &optional id-format) + (lambda (filename &optional _id-format) (make-list 10 (make-list 4