branch: elpa/flymake-collection
commit eb8a7eb96e200cada6559db436298136cd0d5fd1
Author: Mohsin Kaleem <mohk...@kisara.moe>
Commit: Mohsin Kaleem <mohk...@kisara.moe>
tests: Add test harness
---
Makefile | 11 +++
src/flymake-collection-define.el | 3 +
tests/checkers/Dockerfile | 49 ++++++++++
tests/checkers/README.org | 0
tests/checkers/installers/pylint.bash | 1 +
tests/checkers/run-test-case | 166 ++++++++++++++++++++++++++++++++++
tests/checkers/test-cases/pylint.yml | 29 ++++++
7 files changed, 259 insertions(+)
diff --git a/Makefile b/Makefile
index 6b2afc0719..b3cf917aac 100644
--- a/Makefile
+++ b/Makefile
@@ -25,6 +25,17 @@ compile: ## Check for byte-compiler errors
| grep . && exit 1 || true ;\
done
+.PHONY: test
+test: ## Run all defined test cases.
+ @echo "[test] Running all test cases"
+ @docker build -t flymake-collection-test ./tests/checkers
+ @docker run \
+ --rm \
+ --volume "$$(pwd)":/src:ro \
+ --volume "$$(pwd)/tests/checkers":/test:ro \
+ flymake-collection-test \
+ sh -c 'find /test/test-cases/ -iname \*.yml | parallel -I{} chronic
/test/run-test-case {}'
+
.PHONY: clean
clean: ## Remove build artifacts
@echo "[clean]" $(subst .el,.elc,$(SRC))
diff --git a/src/flymake-collection-define.el b/src/flymake-collection-define.el
index 27ff03abe3..038a8bf0ea 100644
--- a/src/flymake-collection-define.el
+++ b/src/flymake-collection-define.el
@@ -293,6 +293,9 @@ exit status %d\nStderr: %s"
;; Finished linting, cleanup any temp-files and then kill
;; the process buffer.
,@cleanup-form
+ (when (eq (plist-get flymake-collection-define--procs
',name)
+ ,proc-symb)
+ (cl-remf flymake-collection-define--procs ',name))
(kill-buffer (process-buffer ,proc-symb)))))))
;; Push the new-process to the process to the process alist.
(setq flymake-collection-define--procs
diff --git a/tests/checkers/Dockerfile b/tests/checkers/Dockerfile
new file mode 100644
index 0000000000..8a43b66778
--- /dev/null
+++ b/tests/checkers/Dockerfile
@@ -0,0 +1,49 @@
+# Ubuntu 20.04 LTS supported until April 2025
+FROM ubuntu:20.04
+
+# Suppress some interactive prompts by answering them with environment
+# variables.
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+
+# Install emacs. Instructions adapted from
[[https://www.masteringemacs.org/article/speed-up-emacs-libjansson-native-elisp-compilation][here]].
+WORKDIR /build/emacs
+RUN apt-get update && \
+ apt-get install -y \
+ apt-transport-https \
+ ca-certificates \
+ curl \
+ gnupg-agent \
+ software-properties-common \
+ libjansson4 \
+ libjansson-dev \
+ git && \
+ git clone -b emacs-28 --single-branch git://git.savannah.gnu.org/emacs.git
. && \
+ sed -i 's/# deb-src/deb-src/' /etc/apt/sources.list && \
+ apt-get update && \
+ apt-get build-dep -y emacs && \
+ ./autogen.sh && \
+ ./configure && \
+ make -j4 && \
+ make install && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
+
+WORKDIR /build
+COPY installers /build/installers
+RUN apt-get update && \
+ apt-get install -y \
+ parallel python3.8 python3-pip moreutils git && \
+ python3.8 -m pip install pyyaml && \
+ find installers/ -type f -iname '*.bash' -exec {} \; && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists/*
+
+# Install the latest available version of flymake for these tests.
+RUN cd "$(mktemp -d)" && \
+ git clone https://github.com/emacs-straight/flymake.git . && \
+ mkdir /tmp/emacs && \
+ cp flymake.el /etc/emacs/flymake.el && \
+ rm -rf "$(pwd)"
+
+WORKDIR /src
diff --git a/tests/checkers/README.org b/tests/checkers/README.org
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tests/checkers/installers/pylint.bash
b/tests/checkers/installers/pylint.bash
new file mode 100755
index 0000000000..febb9baf66
--- /dev/null
+++ b/tests/checkers/installers/pylint.bash
@@ -0,0 +1 @@
+python3.8 -m pip install pylint
diff --git a/tests/checkers/run-test-case b/tests/checkers/run-test-case
new file mode 100755
index 0000000000..5e309f07bc
--- /dev/null
+++ b/tests/checkers/run-test-case
@@ -0,0 +1,166 @@
+#!/usr/bin/env python3.8
+"""Test case runner for flymake-collection.
+"""
+import json
+import logging
+import pathlib
+import subprocess
+import sys
+import tempfile
+from dataclasses import dataclass
+from typing import List, Optional, Tuple
+
+import yaml
+
+
+@dataclass
+class TestLint:
+ """A flymake diagnostic."""
+
+ point: Tuple[int, int] # Line and column
+ level: str
+ message: str
+
+
+@dataclass
+class TestCase:
+ """A test case called `name`, given `file` that should give back
`lints`."""
+
+ name: str
+ file: str
+ lints: List[TestLint]
+
+ def __post_init__(self):
+ self.lints = [TestLint(**it) for it in self.lints]
+
+ def run(self, checker: str) -> bool:
+ with tempfile.NamedTemporaryFile("w") as file:
+ file.write(self.file)
+ file.flush()
+ actual_lints = run_flymake(pathlib.Path(file.name), checker)
+ if actual_lints is None:
+ return False
+
+ failed = False
+ for lint in self.lints:
+ try:
+ pos = actual_lints.index(lint)
+ except ValueError:
+ logging.error("Expected to encounter lint: %s", lint)
+ failed = True
+ else:
+ actual_lints.pop(pos)
+ for lint in actual_lints:
+ logging.error("Encountered unexpected lint: %s", lint)
+ failed = True
+ return not failed
+
+
+@dataclass
+class TestConfig:
+ checker: str
+ tests: List[TestCase]
+
+ def __post_init__(self):
+ self.tests = [TestCase(**it) for it in self.tests]
+
+
+def run_flymake(src: pathlib.Path, checker: str) -> Optional[List[TestLint]]:
+ with tempfile.NamedTemporaryFile("w") as script,
tempfile.NamedTemporaryFile(
+ "r"
+ ) as out:
+ script.write(
+ f"""
+(require 'flymake)
+(require 'json)
+
+(add-to-list 'load-path "/src/src")
+(add-to-list 'load-path "/src/src/checkers")
+(require (intern "{checker}") "{checker}.el")
+
+(setq src (find-file-literally "{src}")
+ out (find-file "{out.name}"))
+
+(defun column-number (point)
+ "Returns the column number at POINT."
+ (interactive)
+ (save-excursion
+ (goto-char point)
+ (current-column)))
+
+(with-current-buffer src
+ ({checker}
+ (lambda (diags)
+ (with-current-buffer out
+ (cl-loop for diag in diags
+ collect
+ (insert
+ (json-encode
+ `((point . ,(with-current-buffer src
+ (let ((beg (flymake--diag-beg diag)))
+ (list (line-number-at-pos beg)
+ (column-number beg)))))
+ (level . ,(flymake--diag-type diag))
+ (message . ,(substring-no-properties (flymake--diag-text
diag)))))
+ "\n"))
+ (save-buffer))))
+
+;; Block until the checker process finishes.
+(while flymake-collection-define--procs
+ (sleep-for 0.25)))
+ """
+ )
+ script.flush()
+ proc = subprocess.run(
+ ["emacs", "-Q", "--script", script.name],
+ capture_output=True,
+ encoding="utf-8",
+ )
+ if proc.returncode != 0:
+ logging.error("Failed to run checker using emacs")
+ logging.error("Emacs exited with stderr: %s", proc.stderr)
+ return None
+
+ lints = []
+ for line in out:
+ if line.strip() == "":
+ continue
+ lints.append(TestLint(**json.loads(line)))
+ return lints
+
+
+def main(args, vargs, parser) -> bool:
+ failed = False
+ logging.info("Loading test config file=%s", args.test)
+ with args.test.open("r") as test_file:
+ cfg_obj = yaml.load(test_file, Loader=yaml.SafeLoader)
+ try:
+ cfg = TestConfig(**cfg_obj)
+ except ValueError:
+ logging.exception("Failed to read test configuration")
+ return False
+
+ logging.info("Running tests with checker=%s", cfg.checker)
+ for i, test in enumerate(cfg.tests):
+ logging.info("Running test case %d name=%s", i, test.name)
+ if not test.run(cfg.checker):
+ failed = True
+
+ return not failed
+
+
+if __name__ == "__main__":
+ import argparse
+
+ parser = argparse.ArgumentParser()
+
+ parser.add_argument(
+ "test", type=pathlib.Path, help="Path to test cases config file"
+ )
+
+ args = parser.parse_args()
+ vargs = vars(args)
+
+ logging.basicConfig(level=logging.DEBUG)
+
+ sys.exit(0 if main(args, vargs, parser) else 1)
diff --git a/tests/checkers/test-cases/pylint.yml
b/tests/checkers/test-cases/pylint.yml
new file mode 100644
index 0000000000..3052b66b80
--- /dev/null
+++ b/tests/checkers/test-cases/pylint.yml
@@ -0,0 +1,29 @@
+---
+checker: flymake-collection-pylint
+tests:
+ - name: no-lints
+ file: |
+ """A test case with no output from pylint."""
+
+ print("hello world")
+ lints: []
+ - name: notes
+ file: |
+ """A test case with a warning lint."""
+
+ print(f"hello world")
+ print(f"hello world")
+ lints:
+ - point: [3, 5]
+ level: warning
+ message: W1309 Using an f-string that does not have any interpolated
variables (pylint)
+ - point: [4, 5]
+ level: warning
+ message: W1309 Using an f-string that does not have any interpolated
variables (pylint)
+ - name: syntax-error
+ file: |
+ definitely should not work
+ lints:
+ - point: [1, 11]
+ level: error
+ message: E0001 invalid syntax (<unknown>, line 1) (pylint)
