Followup-For: Bug #948706 Package: greylistd Version: 0.8.8.8 Tags: patch Hi, I run into the very same problems while setting up a system. Since I found some spare cycles I looked at the code and made some patches.
For simpler development I have set up a git repo and have import the packages found on snapshots.debian.org to provide a package history. You can find the git repo under https://github.com/eurovibes/greylistd Regards Bene Spranger
>From 376957c38203c951b9b6071c4ebfc13139d3ab2c Mon Sep 17 00:00:00 2001 From: Benedikt Spranger <b...@eurovibes.org> Date: Mon, 4 May 2020 00:25:24 +0200 Subject: [PATCH 01/10] Conciliate indentation greylistd-setup-exim4 fail with TabError: inconsistent use of tabs and spaces in indentation Untabify greylistd-setup-exim4. Signed-off-by: Benedikt Spranger <b...@eurovibes.org> --- program/greylistd-setup-exim4 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/program/greylistd-setup-exim4 b/program/greylistd-setup-exim4 index a8c467a..821670c 100755 --- a/program/greylistd-setup-exim4 +++ b/program/greylistd-setup-exim4 @@ -1,7 +1,7 @@ #!/usr/bin/python3 ######################################################################## -### FILE: greylist-setup-exim4 -### PURPOSE: Add a greylisting statement to Exim 4 configuration file +### FILE: greylist-setup-exim4 +### PURPOSE: Add a greylisting statement to Exim 4 configuration file ######################################################################## from sys import version, stdin, stderr, argv, exit @@ -251,7 +251,7 @@ def exim4_configure (lines, aclname, options): try: netmask = int(options["netmask"]) except: - nmstring=options["netmask"] + nmstring=options["netmask"] raise RuntimeError("Invalid netmask size: '%(nmstring)s'"%vars()) ### org raise "Invalid netmask size: '%s'"%options["netmask"] -- 2.26.0.rc2
>From 5265e1c4c346567334d10370f0f81c366e3a865f Mon Sep 17 00:00:00 2001 From: Benedikt Spranger <b.spran...@linutronix.de> Date: Mon, 4 May 2020 14:55:55 +0200 Subject: [PATCH 02/10] Ensure python 3.6 or greater The greylistd package switched to Python 3 and uses some features introduced in Python 3.6. Update the version check to fit. Signed-off-by: Benedikt Spranger <b.spran...@linutronix.de> --- program/greylistd-setup-exim4 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/program/greylistd-setup-exim4 b/program/greylistd-setup-exim4 index 821670c..5ddc1fa 100755 --- a/program/greylistd-setup-exim4 +++ b/program/greylistd-setup-exim4 @@ -10,8 +10,8 @@ from os import listdir, spawnl, P_WAIT from re import compile ### Ensure that we can run this program -if version < "2.3": - stderr.write("This program requires Python 2.3 or newer\n") +if version < "3.6": + stderr.write("This program requires Python 3.0 or newer\n") exit(1) -- 2.26.0.rc2
>From ed347398c93602b59e4dcea0998d9f13cc6447f1 Mon Sep 17 00:00:00 2001 From: Benedikt Spranger <b...@eurovibes.org> Date: Mon, 4 May 2020 00:56:36 +0200 Subject: [PATCH 03/10] Remove trailing whitespaces Perform a overall code cleanup by removing trailing whitespaces. No functional change. Signed-off-by: Benedikt Spranger <b...@eurovibes.org> --- program/greylist | 4 +--- program/greylistd | 24 +++++++++++------------- program/greylistd-setup-exim4 | 12 ++++++------ 3 files changed, 18 insertions(+), 22 deletions(-) diff --git a/program/greylist b/program/greylist index 2603a33..88050d8 100755 --- a/program/greylist +++ b/program/greylist @@ -46,7 +46,6 @@ sockfile = "/var/run/greylistd/socket" commands = ("add", "delete", "check", "update", "stats", "list", "clear", "save", "reload", "mrtg") - def usage (progname, message=None): @@ -114,7 +113,6 @@ elif not action in commands: usage(progname, "Invalid action: '%s'"%action) - confParser = ConfigParser() confParser.read(conffile) try: @@ -149,7 +147,7 @@ while stat: except IOError: break - + else: if not firstword and stat.strip(): firstword = stat.split(None, 1)[0] diff --git a/program/greylistd b/program/greylistd index 58348b6..906dfab 100755 --- a/program/greylistd +++ b/program/greylistd @@ -3,10 +3,10 @@ ### FILE: greylistd.py ### PURPOSE: Simple greylisting daemon. See "greylistd(8)". ### For an introduction to greylisting, see: -### http://projects.puremagic.com/greylisting/ +### http://projects.puremagic.com/greylisting/ ### -### This program listens for connections on a UNIX domain -### socket, presumably from an MTA such as Exim. Nominally, +### This program listens for connections on a UNIX domain +### socket, presumably from an MTA such as Exim. Nominally, ### it reads an identifier (referred to as a "triplet"), ### and returns a single word ("white" or "grey") depending ### on prior knowledge of said identifier. @@ -270,7 +270,7 @@ def saveToFile (datafile, dictionary, perm=0600): fp = file(datafile, 'w') chmod(datafile, perm) - + for (section, subdict) in dictionary.items(): fp.write("[%s]\n"%section) @@ -306,15 +306,13 @@ def loadConfigAndData (): expireKeys(now) data[STATS][LASTSAVE] = now - - def saveData (datafile): ### Save data hashes and timestamps tempfile = "%s.%s"%(datafile, getpid()) saveToFile(tempfile, data) - + try: rename(tempfile, datafile) except OSError as e: @@ -401,7 +399,7 @@ def listTriplets (fp, socket, options): last, first, num = listdata[key] ldate = strftime("%Y-%m-%d %H:%M:%S", localtime(last)) client.send(listformat%(ldate, num, value.strip())) - + except ValueError: if not parseErrors: log("While reading triplets from %s:"% @@ -484,7 +482,7 @@ def do_check (options, args, update=False): if update and config[DATA][TRIPLETFILE] and config[DATA][SAVETRIPLETS]: newTriplets[key] = triplet.lower() - elif ((state == GREY) and + elif ((state == GREY) and (data[GREY][key][IDX_FIRST] + config[TIMEOUTS][RETRYMIN] < now)): state = WHITE @@ -576,7 +574,7 @@ def do_mrtg (): text.append(duration(now - starttime)) text.append(gethostname()) - + text.append('') return "\n".join(text) @@ -595,7 +593,7 @@ def do_list (options, socket): except IOError as e: raise CommandError("Cannot read from '%s': %s\n"%(config[DATA][TRIPLETFILE], e[1])) - + def do_clear (options): for listkey in (options or datatypes): @@ -787,7 +785,7 @@ try: while sockets: interval = config[DATA][UPDATEINTERVAL] lastsave = data[STATS][LASTSAVE] - + if interval and (lastsave + interval < time()): (inlist, outlist, errlist) = select(sockets, [], [], 0) else: @@ -835,7 +833,7 @@ except Exception as e: code = frame.f_code line = frame.f_lineno filename = code.co_filename - + log("### Fatal event in %s, line %d:"%(filename, line), LOG_ERR) log(">>> %s"%e, LOG_ERR) diff --git a/program/greylistd-setup-exim4 b/program/greylistd-setup-exim4 index 5ddc1fa..e8176ab 100755 --- a/program/greylistd-setup-exim4 +++ b/program/greylistd-setup-exim4 @@ -39,7 +39,7 @@ exim4conf_texts = { # This statement has been added by "greylistd-setup-exim4", # and can be removed by running "greylistd-setup-exim4 remove". # Any changes you make here will then be lost. - # + # # Perform greylisting on incoming messages from remote hosts. # We do NOT greylist messages with no envelope sender, because that # would conflict with remote hosts doing callback verifications, and we @@ -104,7 +104,7 @@ exim4conf_texts = { # This statement has been added by "greylistd-setup-exim4", # and can be removed by running "greylistd-setup-exim4 remove". # Any changes you make here will then be lost. - # + # # Perform greylisting on incoming messages with no envelope sender here. # We did not subject these to greylisting after RCPT TO:, because that # would interfere with remote hosts doing sender callout verifications. @@ -233,7 +233,7 @@ def exim4_configure (lines, aclname, options): raise RunException("Already Configured") acltype = options.get("acltype", None) - + if not acltype: for knowntype in exim4conf_texts.keys(): if knowntype in aclname: @@ -245,7 +245,7 @@ def exim4_configure (lines, aclname, options): elif not acltype in exim4conf_texts: raise RunException("Invalid ACL type: '%s'"%acltype) - + if "netmask" in options: try: @@ -296,7 +296,7 @@ def exim4_setup (filename, aclname, function, options, doupdate): except RunException as e: ok, message = False, e[0] - + if not "quiet" in options: if len(filename) > 40: @@ -378,7 +378,7 @@ for arg in argv: (option, value) = arg.lstrip("-").split("=", 1) except: (option, value) = arg.lstrip("-"), None - + options[option] = value elif not action: -- 2.26.0.rc2
>From a3ea12ba35fbe40ec96b2c6a35edfd8e39016276 Mon Sep 17 00:00:00 2001 From: Benedikt Spranger <b...@eurovibes.org> Date: Mon, 4 May 2020 00:27:55 +0200 Subject: [PATCH 04/10] Replace removed function file() In Python 2, file() was the type of an open file. It was used in two ways: * To open files, i.e. as an alias for open(). The documentation mentions that open is more appropriate for this case. * To check if an object is a file, as in isinstance(f, file). The function was removed in python 3. Replace file() as suggested by open(). Signed-off-by: Benedikt Spranger <b...@eurovibes.org> --- program/greylistd | 10 +++++----- program/greylistd-setup-exim4 | 5 ++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/program/greylistd b/program/greylistd index 906dfab..1459a7f 100755 --- a/program/greylistd +++ b/program/greylistd @@ -210,7 +210,7 @@ def typeConvert(typeobject, string): def loadFromFile (datafile, dictionary, typelist=None): try: - fp = file(datafile) + fp = open(datafile) section = None logPfx = 'In %s:'%datafile @@ -267,7 +267,7 @@ def loadFromFile (datafile, dictionary, typelist=None): def saveToFile (datafile, dictionary, perm=0600): try: - fp = file(datafile, 'w') + fp = open(datafile, 'w') chmod(datafile, perm) @@ -326,12 +326,12 @@ def syncTriplets (datafile, perm=0600): target = "%s.%s"%(source, getpid()) try: - infile = file(source, "r") + infile = open(source, "r") except IOError: infile = None try: - outfile = file(target, "w") + outfile = open(target, "w") chmod(target, perm) @@ -587,7 +587,7 @@ def do_list (options, socket): do_save() try: - infile = file(config[DATA][TRIPLETFILE], "r") + infile = open(config[DATA][TRIPLETFILE], "r") error = listTriplets(infile, socket, options) infile.close() diff --git a/program/greylistd-setup-exim4 b/program/greylistd-setup-exim4 index e8176ab..b6d1afe 100755 --- a/program/greylistd-setup-exim4 +++ b/program/greylistd-setup-exim4 @@ -284,9 +284,8 @@ def exim4_setup (filename, aclname, function, options, doupdate): message = function(lines, aclname, options) if doupdate: - fp = file(filename, "w") - fp.writelines(lines) - fp.close() + with open(filename, "w") as fp: + fp.writelines(lines) ok = True -- 2.26.0.rc2
>From 43cab15f4aa79802e70932d693b3d00604d37787 Mon Sep 17 00:00:00 2001 From: Benedikt Spranger <b...@eurovibes.org> Date: Mon, 4 May 2020 00:53:03 +0200 Subject: [PATCH 05/10] Replace leading zero by octal literal. greylistd fail to run: SyntaxError: leading zeros in decimal integer literals are not permitted; use an 0o prefix for octal integers. The leading zero for octal values is not supported by python 3 any more. Replace leading zero by octal literal '0o'. Signed-off-by: Benedikt Spranger <b...@eurovibes.org> --- program/greylistd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/program/greylistd b/program/greylistd index 1459a7f..ace170b 100755 --- a/program/greylistd +++ b/program/greylistd @@ -265,7 +265,7 @@ def loadFromFile (datafile, dictionary, typelist=None): -def saveToFile (datafile, dictionary, perm=0600): +def saveToFile (datafile, dictionary, perm=0o600): try: fp = open(datafile, 'w') @@ -321,7 +321,7 @@ def saveData (datafile): -def syncTriplets (datafile, perm=0600): +def syncTriplets (datafile, perm=0o600): source = datafile target = "%s.%s"%(source, getpid()) -- 2.26.0.rc2
>From 53e1d614f42ae1bea026b2312de1e0db48b0138f Mon Sep 17 00:00:00 2001 From: Benedikt Spranger <b...@eurovibes.org> Date: Mon, 4 May 2020 00:58:32 +0200 Subject: [PATCH 06/10] Remove unnessesary define The comment says: We are now in Python 3.x and True and False are available. Remove the crust. Signed-off-by: Benedikt Spranger <b...@eurovibes.org> --- program/greylist | 4 ---- 1 file changed, 4 deletions(-) diff --git a/program/greylist b/program/greylist index 88050d8..9138c51 100755 --- a/program/greylist +++ b/program/greylist @@ -25,10 +25,6 @@ from sys import argv, stdout, stderr, exit from configparser import ConfigParser -### Define values not yet availble in Python 2.1 -False, True = (0 is 1), (1 is 1) - - ### Exit codes based on responses from greylistd exitcodes = { "error:" : -1, "white" : 0, -- 2.26.0.rc2
>From b6d4024672d91695da6f6814c3dbd665ecb3c412 Mon Sep 17 00:00:00 2001 From: Benedikt Spranger <b...@eurovibes.org> Date: Mon, 4 May 2020 09:13:57 +0200 Subject: [PATCH 07/10] Replace deprecated function getargspec() getargspec() is deprecated since Python version 3.0. Replace it with the successor getfullargspec(). Signed-off-by: Benedikt Spranger <b...@eurovibes.org> --- program/greylistd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/program/greylistd b/program/greylistd index ace170b..5f1d124 100755 --- a/program/greylistd +++ b/program/greylistd @@ -39,7 +39,7 @@ from sys import version as PyVersion, stdout, stderr, exit, exc_info from signal import signal, SIGTERM, SIGHUP, SIGUSR1, SIG_IGN, SIG_DFL from syslog import openlog, syslog, LOG_NOTICE, LOG_WARNING, LOG_ERR from select import select -from inspect import getargspec +from inspect import getfullargspec ### Ensure that we can run this program @@ -648,7 +648,7 @@ def runCommand (line, client): else: function = do_update - args, varargs, varkw, defaults = getargspec(function) + args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations = getfullargspec(function) while words and words[0].startswith("-"): options.append(words.pop(0).lstrip("-")) -- 2.26.0.rc2
>From 80140014c23ad76ece241cdb898d1fcf7110f46c Mon Sep 17 00:00:00 2001 From: Benedikt Spranger <b...@eurovibes.org> Date: Mon, 4 May 2020 09:14:28 +0200 Subject: [PATCH 08/10] Get line number of traceback The line number information is now part of the traceback. Get it from there instead of evaluate the frame. Signed-off-by: Benedikt Spranger <b...@eurovibes.org> --- program/greylistd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/program/greylistd b/program/greylistd index 5f1d124..aeb4cc8 100755 --- a/program/greylistd +++ b/program/greylistd @@ -831,7 +831,7 @@ except Exception as e: frame = tb.tb_frame code = frame.f_code - line = frame.f_lineno + line = tb.tb_lineno filename = code.co_filename log("### Fatal event in %s, line %d:"%(filename, line), LOG_ERR) -- 2.26.0.rc2
>From d705d9ffcd350ca8d307c688f80d01025c675e8a Mon Sep 17 00:00:00 2001 From: Benedikt Spranger <b...@eurovibes.org> Date: Mon, 4 May 2020 09:13:17 +0200 Subject: [PATCH 09/10] Python 3 byte object / string rework. In Python 3 str and bytes are not the same typeByte objects any more. Rework the code to distinguish between both types and add conversions were needed. Signed-off-by: Benedikt Spranger <b...@eurovibes.org> --- program/greylist | 8 ++++---- program/greylistd | 38 +++++++++++++++++++---------------- program/greylistd-setup-exim4 | 4 ++-- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/program/greylist b/program/greylist index 9138c51..1174c55 100755 --- a/program/greylist +++ b/program/greylist @@ -123,13 +123,13 @@ sock = socket(AF_UNIX, SOCK_STREAM) try: sock.connect(sockfile) except Exception as e: - stderr.write("%s: %s\n"%(sockfile, e[1])) + stderr.write("%s: %s\n"%(sockfile, str(e))) exit(-1) try: - sock.send(" ".join(argv[1:])) + sock.send(" ".join(argv[1:]).encode()) except Exception as e: - stderr.write("%s: %s\n"%(sockfile, e[1])) + stderr.write("%s: %s\n"%(sockfile, str(e))) exit(-1) @@ -139,7 +139,7 @@ firstword = None while stat: stat = sock.recv(1024) try: - stdout.write("%s"%stat) + stdout.write("%s"%stat.decode()) except IOError: break diff --git a/program/greylistd b/program/greylistd index aeb4cc8..95aa3e3 100755 --- a/program/greylistd +++ b/program/greylistd @@ -261,7 +261,7 @@ def loadFromFile (datafile, dictionary, typelist=None): fp.close() except IOError as e: - raise RunException("Cannot read from '%s': %s"%(datafile, e[1])) + raise RunException("Cannot read from '%s': %s"%(datafile, str(e))) @@ -284,10 +284,10 @@ def saveToFile (datafile, dictionary, perm=0o600): fp.close() except IOError as e: - raise RunException("Cannot write to %s: %s"%(datafile, e[1])) + raise RunException("Cannot write to %s: %s"%(datafile, str(e))) except OSError as e: - raise RunException("Cannot set mode 0%o on %s: %s"%(perm, datafile, e[1])) + raise RunException("Cannot set mode 0%o on %s: %s"%(perm, datafile, str(e))) @@ -317,7 +317,7 @@ def saveData (datafile): rename(tempfile, datafile) except OSError as e: raise RunException("Cannot rename %s to %s: %s"%(tempfile, - datafile, e[1])) + datafile, str(e))) @@ -355,10 +355,10 @@ def syncTriplets (datafile, perm=0o600): outfile.close() except IOError as e: - raise RunException("Could not write to %s: %s"%(target, e[1])) + raise RunException("Could not write to %s: %s"%(target, str(e))) except OSError as e: - raise RunException("Cannot set mode 0%o on %s: %s"%(perm, target, e[1])) + raise RunException("Cannot set mode 0%o on %s: %s"%(perm, target, str(e))) if infile: @@ -367,7 +367,7 @@ def syncTriplets (datafile, perm=0o600): try: rename(target, source) except OSError as e: - raise RunException("Could not rename %s to %s: %s"%(target, source, e[1])) + raise RunException("Could not rename %s to %s: %s"%(target, source, str(e))) @@ -386,8 +386,10 @@ def listTriplets (fp, socket, options): listdata = data[listkey] line = "%slist data:"%listkey.capitalize() dash = "="*len(line) - client.send("\n%s\n%s\n"%(line, dash)) - client.send(listformat%("Last Seen", "Count", "Data")) + out = "\n%s\n%s\n"%(line, dash) + client.send(out.encode()) + out = listformat%("Last Seen", "Count", "Data") + client.send(out.encode()) fp.seek(0) for line in fp: @@ -398,7 +400,8 @@ def listTriplets (fp, socket, options): if key in listdata: last, first, num = listdata[key] ldate = strftime("%Y-%m-%d %H:%M:%S", localtime(last)) - client.send(listformat%(ldate, num, value.strip())) + out = listformat%(ldate, num, value.strip()) + client.send(out.encode()) except ValueError: if not parseErrors: @@ -592,7 +595,8 @@ def do_list (options, socket): infile.close() except IOError as e: - raise CommandError("Cannot read from '%s': %s\n"%(config[DATA][TRIPLETFILE], e[1])) + raise CommandError("Cannot read from '%s': %s\n"% + (str(config[DATA][TRIPLETFILE]), str(e))) def do_clear (options): @@ -668,7 +672,7 @@ def runCommand (line, client): elif arg == "args": useargs = True - arglist.append(words) + arglist.append(str(words)) elif arg == "socket": arglist.append(client) @@ -719,7 +723,7 @@ def createSocket (path, owner, mode): try: chown(path, uid, gid) except OSError as e: - raise RunException("Could not change ownership of socket %s: %s"%(path, e[1])) + raise RunException("Could not change ownership of socket %s: %s"%(path, str(e))) try: chmod(path, int(mode, 8)) @@ -727,7 +731,7 @@ def createSocket (path, owner, mode): except ValueError: raise RunException("Specified socket mode '%s' is not a valid octal number"%mode) except OSError as e: - raise RunException("Could not set mode 0%o on socket %s: %s"%(mode, path, e[1])) + raise RunException("Could not set mode 0%o on socket %s: %s"%(mode, path, str(e))) sock.listen(5) return sock @@ -807,12 +811,12 @@ try: client = inlist[0] try: - line = client.recv(16384) + line = client.recv(16384).decode() reply = runCommand(line, client) - client.send(reply) + client.send(reply.encode()) except SocketError as e: - log("Socket error: %s"%e[1]) + log("Socket error: %s"%str(e)) client.close() sockets.remove(client) diff --git a/program/greylistd-setup-exim4 b/program/greylistd-setup-exim4 index b6d1afe..c784603 100755 --- a/program/greylistd-setup-exim4 +++ b/program/greylistd-setup-exim4 @@ -290,11 +290,11 @@ def exim4_setup (filename, aclname, function, options, doupdate): ok = True except IOError as e: - ok, message = False, e[1] + ok, message = False, str(e) except RunException as e: - ok, message = False, e[0] + ok, message = False, str(e) if not "quiet" in options: -- 2.26.0.rc2
>From 7792714e19f3b5c645298892832caf07d93f03e3 Mon Sep 17 00:00:00 2001 From: Benedikt Spranger <b...@eurovibes.org> Date: Tue, 5 May 2020 00:15:54 +0200 Subject: [PATCH 10/10] Fight the Exim4 deconfigure madness. The Exim4 deconfiguration was broken. There are left over parts from the greylist configuration which are may harmfull. Revisit the code and fix the deconfiguration. Signed-off-by: Benedikt Spranger <b...@eurovibes.org> --- program/greylistd-setup-exim4 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/program/greylistd-setup-exim4 b/program/greylistd-setup-exim4 index c784603..cc861e0 100755 --- a/program/greylistd-setup-exim4 +++ b/program/greylistd-setup-exim4 @@ -25,7 +25,8 @@ exim4conf_default_places = ( ### Words that separate blocks in the Exim 4 configuation file exim4conf_blocks = [ - "begin", "accept", "defer", "deny", "discard", "drop", "require", "warn" ] + "begin", "accept", "defer", "deny", "discard", "drop", "require", "warn", + ".ifdef", ".ifndef", ".else", ".endif"] ### What blocks do we remove when unconfiguring greylistd? ### Every line in the block must match at least one item in this list, @@ -192,6 +193,9 @@ def find_block (lines, decl, rxList, commentPfx="#", blocks=exim4conf_blocks): elif currentDecl == decl: if line.split()[0] in blocks: if region: + if startComment and "blacklisted by greylist" in lines[startComment]: + region = (startBlock, index+1) + continue return region elif startComment is not None: -- 2.26.0.rc2