2 examples:

/var/log/mail.log
scan for smtp auth failures like

Jul 14 08:34:54 ns1 postfix/smtpd[20888]: warning: unknown[xx.xx.xx.xx]: SASL LOGIN authentication failed: UGFzc3dvcmQ6

/var/log/auth.log
scan for dovecot imap login failures like

Jul 14 08:34:05 ns1 auth: pam_unix(dovecot:auth): authentication failure; logname= uid=0 euid=0 tty=dovecot ruser=contact rhost=xx.xx.xx.xx

in /etc/csf/csf.conf define

CUSTOM1_LOG = "/var/log/mail.log"
CUSTOM2_LOG = "/var/log/auth.log"

in /etc/csf/regex.custom.pm
add

if (($lgfile eq $config{CUSTOM1_LOG}) and ($line =~ /^\S+\s+\d+\s+\S+ \S+ postfix\/smtpd\[\d+\]: warning:.*\[(\d+\.\d+\.\d+\.\d+)\]: SASL [A-Z]*? authentication failed/)) {
return ("Failed SASL login from",$1,"mysaslmatch","5","list of ports to block this IP","14400");
}

if (($lgfile eq $config{CUSTOM2_LOG}) and ($line =~ /^\S+\s+\d+\s+\S+ \S+ auth:\ pam_unix\(dovecot:auth\): authentication failure; .* rhost\=(\d+\.\d+\.\d+\.\d+)/)) {
return ("Failed dovecot auth login from",$1,"dovecot_auth_match","5","list of ports to block this IP","14400");
}