spamassassin: major service fixes -> added more perl packages to spamassassin and abstracted service to not use /etc/spamassassin, also running sa-update as spamd user periodically (#26470)

thanks @grahamc for IRC comments

nixos/modules/services/mail/spamassassin.nix

@@ -3,43 +3,129 @@
 with lib;
 
 let
-
   cfg = config.services.spamassassin;
+  spamassassin-local-cf = pkgs.writeText "local.cf" cfg.config;
+  spamassassin-init-pre = pkgs.writeText "init.pre" cfg.initPreConf;
+
+  spamdEnv = pkgs.buildEnv {
+    name = "spamd-env";
+    paths = [];
+    postBuild = ''
+      ln -sf ${spamassassin-init-pre} $out/init.pre
+      ln -sf ${spamassassin-local-cf} $out/local.cf
+    '';
+  };
 
 in
 
 {
-
-  ###### interface
-
   options = {
 
     services.spamassassin = {
-
       enable = mkOption {
         default = false;
-        description = "Whether to run the SpamAssassin daemon.";
+        description = "Whether to run the SpamAssassin daemon";
       };
 
       debug = mkOption {
         default = false;
-        description = "Whether to run the SpamAssassin daemon in debug mode.";
+        description = "Whether to run the SpamAssassin daemon in debug mode";
       };
 
-    };
+      config = mkOption {
+        type = types.lines;
+        description = ''
+          The SpamAssassin local.cf config
+
+          If you are using this configuration:
+            add_header all Status _YESNO_, score=_SCORE_ required=_REQD_ tests=_TESTS_ autolearn=_AUTOLEARN_ version=_VERSION_
+
+          Then you can Use this sieve filter:
+            require ["fileinto", "reject", "envelope"];
+            
+            if header :contains "X-Spam-Flag" "YES" {
+              fileinto "spam";
+            }
+
+          Or this procmail filter:
+            :0:
+            * ^X-Spam-Flag: YES
+            /var/vpopmail/domains/lastlog.de/js/.maildir/.spam/new
+
+          To filter your messages based on the additional mail headers added by spamassassin.
+        '';
+        example = ''
+          #rewrite_header Subject [***** SPAM _SCORE_ *****]
+          required_score          5.0
+          use_bayes               1
+          bayes_auto_learn        1
+          add_header all Status _YESNO_, score=_SCORE_ required=_REQD_ tests=_TESTS_ autolearn=_AUTOLEARN_ version=_VERSION_
+        '';
+        default = "";
+      };
 
+      initPreConf = mkOption {
+        type = types.str;
+        description = "The SpamAssassin init.pre config.";
+        default = 
+        ''          
+          #
+          # to update this list, run this command in the rules directory:
+          # grep 'loadplugin.*Mail::SpamAssassin::Plugin::.*' -o -h * | sort | uniq     
+          #
+
+          #loadplugin Mail::SpamAssassin::Plugin::AccessDB
+          #loadplugin Mail::SpamAssassin::Plugin::AntiVirus
+          loadplugin Mail::SpamAssassin::Plugin::AskDNS
+          # loadplugin Mail::SpamAssassin::Plugin::ASN
+          loadplugin Mail::SpamAssassin::Plugin::AutoLearnThreshold
+          #loadplugin Mail::SpamAssassin::Plugin::AWL
+          loadplugin Mail::SpamAssassin::Plugin::Bayes
+          loadplugin Mail::SpamAssassin::Plugin::BodyEval
+          loadplugin Mail::SpamAssassin::Plugin::Check
+          #loadplugin Mail::SpamAssassin::Plugin::DCC
+          loadplugin Mail::SpamAssassin::Plugin::DKIM
+          loadplugin Mail::SpamAssassin::Plugin::DNSEval
+          loadplugin Mail::SpamAssassin::Plugin::FreeMail
+          loadplugin Mail::SpamAssassin::Plugin::Hashcash
+          loadplugin Mail::SpamAssassin::Plugin::HeaderEval
+          loadplugin Mail::SpamAssassin::Plugin::HTMLEval
+          loadplugin Mail::SpamAssassin::Plugin::HTTPSMismatch
+          loadplugin Mail::SpamAssassin::Plugin::ImageInfo
+          loadplugin Mail::SpamAssassin::Plugin::MIMEEval
+          loadplugin Mail::SpamAssassin::Plugin::MIMEHeader
+          # loadplugin Mail::SpamAssassin::Plugin::PDFInfo
+          #loadplugin Mail::SpamAssassin::Plugin::PhishTag
+          loadplugin Mail::SpamAssassin::Plugin::Pyzor
+          loadplugin Mail::SpamAssassin::Plugin::Razor2
+          # loadplugin Mail::SpamAssassin::Plugin::RelayCountry
+          loadplugin Mail::SpamAssassin::Plugin::RelayEval
+          loadplugin Mail::SpamAssassin::Plugin::ReplaceTags
+          # loadplugin Mail::SpamAssassin::Plugin::Rule2XSBody
+          # loadplugin Mail::SpamAssassin::Plugin::Shortcircuit
+          loadplugin Mail::SpamAssassin::Plugin::SpamCop
+          loadplugin Mail::SpamAssassin::Plugin::SPF
+          #loadplugin Mail::SpamAssassin::Plugin::TextCat
+          # loadplugin Mail::SpamAssassin::Plugin::TxRep
+          loadplugin Mail::SpamAssassin::Plugin::URIDetail
+          loadplugin Mail::SpamAssassin::Plugin::URIDNSBL
+          loadplugin Mail::SpamAssassin::Plugin::URIEval
+          # loadplugin Mail::SpamAssassin::Plugin::URILocalBL
+          loadplugin Mail::SpamAssassin::Plugin::VBounce
+          loadplugin Mail::SpamAssassin::Plugin::WhiteListSubject
+          loadplugin Mail::SpamAssassin::Plugin::WLBLEval
+        '';
+      };
+    };
   };
 
-
-  ###### implementation
-
   config = mkIf cfg.enable {
 
     # Allow users to run 'spamc'.
     environment.systemPackages = [ pkgs.spamassassin ];
 
     users.extraUsers = singleton {
-    name = "spamd";
+      name = "spamd";
       description = "Spam Assassin Daemon";
       uid = config.ids.uids.spamd;
       group = "spamd";
@@ -50,13 +136,65 @@ in
       gid = config.ids.gids.spamd;
     };
 
+    systemd.services.sa-update = {
+      script = ''
+        set +e 
+        ${pkgs.su}/bin/su -s "${pkgs.bash}/bin/bash" -c "${pkgs.spamassassin}/bin/sa-update --gpghomedir=/var/lib/spamassassin/sa-update-keys/ --siteconfigpath=${spamdEnv}/" spamd
+
+        v=$?
+        set -e
+        if [ $v -gt 1 ]; then
+          echo "sa-update execution error"
+          exit $v
+        fi
+        if [ $v -eq 0 ]; then
+          systemctl reload spamd.service
+        fi
+      '';
+    };
+
+    systemd.timers.sa-update = { 
+      description = "sa-update-service";
+      partOf      = [ "sa-update.service" ];
+      wantedBy    = [ "timers.target" ];
+      timerConfig = {
+        OnCalendar = "1:*";
+        Persistent = true;
+      };
+    };
+
     systemd.services.spamd = {
       description = "Spam Assassin Server";
 
       wantedBy = [ "multi-user.target" ];
       after = [ "network.target" ];
 
-      script = "${pkgs.spamassassin}/bin/spamd ${optionalString cfg.debug "-D"} --username=spamd --groupname=spamd --nouser-config --virtual-config-dir=/var/lib/spamassassin/user-%u --allow-tell --pidfile=/var/run/spamd.pid";
+      serviceConfig = {
+        ExecStart = "${pkgs.spamassassin}/bin/spamd ${optionalString cfg.debug "-D"} --username=spamd --groupname=spamd --siteconfigpath=${spamdEnv} --virtual-config-dir=/var/lib/spamassassin/user-%u --allow-tell --pidfile=/var/run/spamd.pid";
+        ExecReload = "${pkgs.coreutils}/bin/kill -HUP $MAINPID";
+      };
+
+      # 0 and 1 no error, exitcode > 1 means error:
+      # https://spamassassin.apache.org/full/3.1.x/doc/sa-update.html#exit_codes
+      preStart = ''
+        # this abstraction requires no centralized config at all
+        if [ -d /etc/spamassassin ]; then
+          echo "This spamassassin does not support global '/etc/spamassassin' folder for configuration as this would be impure. Merge your configs into 'services.spamassassin' and remove the '/etc/spamassassin' folder to make this service work. Also see 'https://github.com/NixOS/nixpkgs/pull/26470'."; 
+          exit 1
+        fi
+        echo "Recreating '/var/lib/spamasassin' with creating '3.004001' (or similar) and 'sa-update-keys'"
+        mkdir -p /var/lib/spamassassin
+        chown spamd:spamd /var/lib/spamassassin -R
+        set +e 
+        ${pkgs.su}/bin/su -s "${pkgs.bash}/bin/bash" -c "${pkgs.spamassassin}/bin/sa-update --gpghomedir=/var/lib/spamassassin/sa-update-keys/ --siteconfigpath=${spamdEnv}/" spamd
+        v=$?
+        set -e
+        if [ $v -gt 1 ]; then
+          echo "sa-update execution error"
+          exit $v
+        fi
+        chown spamd:spamd /var/lib/spamassassin -R
+      '';
     };
   };
 }

pkgs/servers/mail/spamassassin/default.nix

@@ -1,17 +1,7 @@
-{ stdenv, fetchurl, buildPerlPackage, perl, HTMLParser, NetDNS, NetAddrIP, DBFile
+{ stdenv, fetchurl, buildPerlPackage, perl, perlPackages, HTMLParser, NetDNS, NetAddrIP, DBFile
 , HTTPDate, MailDKIM, LWP, IOSocketSSL, makeWrapper, gnupg1
 }:
 
-# TODO: Add the Perl modules ...
-#
-#   DBI
-#   Encode::Detect
-#   IP::Country::Fast
-#   Mail::SPF
-#   Net::Ident
-#   Razor2::Client::Agent
-#
-
 buildPerlPackage rec {
   name = "SpamAssassin-3.4.1";
 
@@ -20,14 +10,17 @@ buildPerlPackage rec {
     sha256 = "0la6s5ilamf9129kyjckcma8cr6fpb6b5f2fb64v7106iy0ckhd0";
   };
 
-  buildInputs = [ makeWrapper HTMLParser NetDNS NetAddrIP DBFile HTTPDate MailDKIM
-    LWP IOSocketSSL ];
+  # https://bz.apache.org/SpamAssassin/show_bug.cgi?id=7434
+  patches = [ ./sa-update_add--siteconfigpath.patch ];
+
+  buildInputs = with perlPackages; [ makeWrapper HTMLParser NetDNS NetAddrIP DBFile HTTPDate MailDKIM
+    LWP IOSocketSSL DBI EncodeDetect IPCountry NetIdent Razor2ClientAgent MailSPF NetDNSResolverProgrammable ];
 
   # Enabling 'taint' mode is desirable, but that flag disables support
   # for the PERL5LIB environment variable. Needs further investigation.
   makeFlags = "PERL_BIN=${perl}/bin/perl PERL_TAINT=no";
 
-  makeMakerFlags = "CONFDIR=/etc/spamassassin LOCALSTATEDIR=/var/lib/spamassassin";
+  makeMakerFlags = "CONFDIR=/homeless/shelter LOCALSTATEDIR=/var/lib/spamassassin";
 
   doCheck = false;
 
@@ -45,6 +38,6 @@ buildPerlPackage rec {
     description = "Open-Source Spam Filter";
     license = stdenv.lib.licenses.asl20;
     platforms = stdenv.lib.platforms.linux;
-    maintainers = [ stdenv.lib.maintainers.peti ];
+    maintainers = with stdenv.lib.maintainers; [ peti qknight ];
   };
 }

pkgs/servers/mail/spamassassin/sa-update_add--siteconfigpath.patch

@@ -0,0 +1,59 @@
+From 60abf3ee8864980a95b32e2d6cf60e26b49654c0 Mon Sep 17 00:00:00 2001
+From: joachim schiele <js@lastlog.de>
+Date: Wed, 7 Jun 2017 22:15:39 +0200
+Subject: [PATCH 2/2] Adding --siteconfigpath to not be forced to use global /etc/spamasassin configuration directory but to provide an alternative location.
+
+---
+ sa-update.raw | 5 +++++
+ lib/Mail/SpamAssassin/PerMsgStatus.pm | 2 +
+ 2 file changed, 7 insertions(+), 1 deletion(-)
+
+diff --git a/sa-update.raw b/sa-update.raw
+index bb7396d..39f681f 100755
+--- a/sa-update.raw
++++ b/sa-update.raw
+@@ -196,6 +196,7 @@ GetOptions(
+   'gpgkeyfile=s'			=> \$opt{'gpgkeyfile'},
+   'channelfile=s'			=> \$opt{'channelfile'},
+   'updatedir=s'				=> \$opt{'updatedir'},
++  'siteconfigpath=s'                           => \$opt{'siteconfigpath'},
+   'gpg!'				=> \$GPG_ENABLED,
+ 
+   '4'                                   => sub { $opt{'force_pf'} = 'inet' },
+@@ -267,6 +268,9 @@ else {
+   $opt{'updatedir'} = $SA->sed_path('__local_state_dir__/__version__');
+ }
+ 
++if (defined $opt{'siteconfigpath'}) {
++  $LOCAL_RULES_DIR = untaint_file_path($opt{'siteconfigpath'});
++}
+ 
+ # check only disabled gpg
+ # https://issues.apache.org/SpamAssassin/show_bug.cgi?id=5854
+@@ -1808,6 +1812,7 @@ Options:
+   --updatedir path        Directory to place updates, defaults to the
+                           SpamAssassin site rules directory
+                           (default: @@LOCAL_STATE_DIR@@/@@VERSION@@)
++  --siteconfigpath=path   Path for site configs
+   --refreshmirrors        Force the MIRRORED.BY file to be updated
+   -D, --debug [area=n,...]  Print debugging messages
+   -v, --verbose           Be verbose, like print updated channel names;
+-- 
+diff --git a/lib/Mail/SpamAssassin/PerMsgStatus.pm b/lib/Mail/SpamAssassin/PerMsgStatus.pm
+index 6d8beaa..6ad87dc 100644
+--- a/lib/Mail/SpamAssassin/PerMsgStatus.pm
++++ b/lib/Mail/SpamAssassin/PerMsgStatus.pm
+@@ -389,7 +389,8 @@ sub check_timed {
+     if (!$self->{main}->have_plugin("check_main")) {
+       die "check: no loaded plugin implements 'check_main': cannot scan!\n".
+             "Check that the necessary '.pre' files are in the config directory.\n".
+-              "At a minimum, v320.pre loads the Check plugin which is required.\n";
++              "At a minimum, v320.pre loads the Check plugin which is required.\n".
++              "NixOS: Since there is no '/etc/spamassassin' simply restart 'spamd.service' which on its behalf will run 'sa-learn --siteconfigpath=/nix/store/l4hr4yxk8mb4wbkha6vm6rh92pj19rj6-spamd-env ...' to update the /var/lib/spamassassin configs.\n";
+     }
+   }
+ 
+-- 
+
+2.12.2
+

pkgs/top-level/perl-packages.nix

@@ -4650,6 +4650,20 @@ let self = _self // overrides; _self = with self; {
     };
   };
 
+  EncodeDetect = buildPerlPackage rec {
+    name = "Encode-Detect-1.01";
+    src = fetchurl {
+      url = "mirror://cpan/authors/id/J/JG/JGMYERS/${name}.tar.gz";
+      sha256 = "834d893aa7db6ce3f158afbd0e432d6ed15a276e0940db0a74be13fd9c4bbbf1";
+    };
+    propagatedBuildInputs = [ ModuleBuild ];
+    meta = {
+      description = "An Encode::Encoding subclass that detects the encoding of data";
+      license = stdenv.lib.licenses.free;
+    };
+  };
+
+
   EncodeEUCJPASCII = buildPerlPackage {
     name = "Encode-EUCJPASCII-0.03";
     src = fetchurl {
@@ -4747,7 +4761,7 @@ let self = _self // overrides; _self = with self; {
       sha256 = "1vzpz6syb82ir8svp2wjh95x6lpf01lgkxn2xy60ixrszc24zdya";
     };
   };
-
+ 
   EV = buildPerlPackage rec {
     name = "EV-4.22";
     src = fetchurl {
@@ -7039,6 +7053,32 @@ let self = _self // overrides; _self = with self; {
     };
   };
 
+  IPCountry = buildPerlPackage rec {
+    name = "IP-Country-2.28";
+    src = fetchurl {
+      url = "mirror://cpan/authors/id/N/NW/NWETTERS/${name}.tar.gz";
+      sha256 = "88db833a5ab22ed06cb53d6f205725e3b5371b254596053738885e91fa105f75";
+    };
+    propagatedBuildInputs = [ GeographyCountries ];
+    meta = {
+      description = "Fast lookup of country codes from IP addresses";
+      license = stdenv.lib.licenses.mit;
+    };
+  };
+
+  GeographyCountries = buildPerlPackage rec {
+    name = "Geography-Countries-2009041301";
+    src = fetchurl {
+      url = "mirror://cpan/authors/id/A/AB/ABIGAIL/${name}.tar.gz";
+      sha256 = "48c42e40e8281ba7c981743a854c48e6def2d51eb0925ea6c96e25c74497f20f";
+    };
+    meta = {
+      description = "2-letter, 3-letter, and numerical codes for countries";
+      license = stdenv.lib.licenses.mit;
+    };
+  };
+
+
   IPCRun = buildPerlPackage {
     name = "IPC-Run-0.92";
     src = fetchurl {
@@ -8061,6 +8101,28 @@ let self = _self // overrides; _self = with self; {
     };
   };
 
+  MailSPF = buildPerlPackage rec {
+    name = "Mail-SPF-v2.9.0";
+    #src = /root/nixops/Mail-SPF-v2.9.0;
+    src = fetchurl {
+      url = "mirror://cpan/authors/id/J/JM/JMEHNLE/mail-spf/${name}.tar.gz";
+      sha256 = "61cb5915f1c7acc7a931ffc1bfc1291bdfac555e2a46eb2391b995ea9ecb6162";
+    };
+    # remove this patch patches = [ ../development/perl-modules/Mail-SPF.patch ];
+
+    buildInputs = [ ModuleBuild NetDNSResolverProgrammable ];
+    propagatedBuildInputs = [ Error NetAddrIP NetDNS URI ];
+
+    buildPhase = "perl Build.PL --install_base=$out --install_path=\"sbin=$out/bin\"; ./Build build ";
+
+    doCheck = false; # The main test performs network access
+    meta = {
+      description = "An object-oriented implementation of Sender Policy Framework";
+      license = stdenv.lib.licenses.bsd3;
+    };
+  };
+
+
   MailTools = buildPerlPackage rec {
     name = "MailTools-2.14";
     src = fetchurl {
@@ -9727,6 +9789,19 @@ let self = _self // overrides; _self = with self; {
   # Deprecated.
   NamespaceClean = self.namespaceclean;
 
+  NetIdent = buildPerlPackage rec {
+    name = "Net-Ident-1.24";
+    src = fetchurl {
+      url = "mirror://cpan/authors/id/T/TO/TODDR/${name}.tar.gz";
+      sha256 = "5f5f1142185a67b87406a3fb31f221564f61838a70ef4c07284a66c55e82ad05";
+    };
+    meta = {
+      homepage = http://wiki.github.com/toddr/Net-Ident/;
+      description = "Lookup the username on the remote end of a TCP/IP connection";
+      license = stdenv.lib.licenses.mit;
+    };
+  };
+
   NetAddrIP = buildPerlPackage rec {
     name = "NetAddr-IP-4.079";
     src = fetchurl {
@@ -10017,6 +10092,21 @@ let self = _self // overrides; _self = with self; {
     };
   };
 
+  NetDNSResolverProgrammable = buildPerlPackage rec {
+    name = "Net-DNS-Resolver-Programmable-v0.003";
+    src = fetchurl {
+      url = "mirror://cpan/authors/id/J/JM/JMEHNLE/net-dns-resolver-programmable/${name}.tar.gz";
+      sha256 = "8d402260941f259c83bf1b2564408e75288df028f604136c29da11a9a6a076ec";
+    };
+    buildInputs = [ ModuleBuild ];
+    propagatedBuildInputs = [ NetDNS ];
+    meta = {
+      description = "Programmable DNS resolver class for offline emulation of DNS";
+      license = with stdenv.lib.licenses; [ artistic1 gpl1Plus ];
+    };
+  };
+
+
   NetServer = buildPerlPackage {
     name = "Net-Server-2.007";
     src = fetchurl {
@@ -11286,6 +11376,21 @@ let self = _self // overrides; _self = with self; {
     };
   };
 
+  Razor2ClientAgent = buildPerlPackage rec {
+    name = "Razor2-Client-Agent-2.84";
+    src = fetchurl {
+      url = "mirror://cpan/authors/id/T/TO/TODDR/${name}.tar.gz";
+      sha256 = "d7c2ed7f347a673b1425e4da7656073d6c52847bc7403bf57e3a404b52f7e501";
+    };
+    propagatedBuildInputs = [ DigestSHA1 URI ];
+    meta = {
+      homepage = http://razor.sourceforge.net/;
+      description = "Collaborative, content-based spam filtering network agent";
+      license = stdenv.lib.licenses.mit;
+    };
+  };
+
+
   Readonly = buildPerlModule rec {
     name = "Readonly-2.05";
     src = fetchurl {