#lang en

<<TableOfContents()>>

= Description =
SMTP is the mail backbone on the internet. A lot of stuff has been written. Most of this page is about FreeBSD and sendmail.

= TLS =
We use Letsencrypt fot the TLS in our sendmail configuration. First, get a certificate. We don't gonna tell you how, it's beyond the scope of this page. But your certificate is in ''/usr/local/etc/letsencrypt/live/<SMTP-HOSTNAME>/''. Change the default configuration in ''/etc/mail/<HOSTNAME>.mc'' to:

{{{
define(`CERT_DIR', `/usr/local/etc/letsencrypt/live/<SMTP-HOSTNAME>')dnl
define(`confSERVER_CERT', `CERT_DIR/cert.pem')dnl
define(`confSERVER_KEY', `CERT_DIR/privkey.pem')dnl
define(`confCLIENT_CERT', `CERT_DIR/cert.pem')dnl
define(`confCLIENT_KEY', `CERT_DIR/privkey.pem')dnl
define(`confCACERT', `CERT_DIR/chain.pem')dnl
define(`confCACERT_PATH', `CERT_DIR')dnl
define(`confDH_PARAMETERS', `CERT_DIR/dh_4096.param')dnl
}}}

And at the end of the configfile:
{{{
LOCAL_CONFIG
O CipherList=kEECDH:+kEECDH+SHA:kEDH:+kEDH+SHA:+kEDH+CAMELLIA:kECDH:+kECDH+SHA:kRSA:+kRSA+SHA:+kRSA+CAMELLIA:!aNULL:!eNULL:!SSLv2:!RC4:!MD5:!DES:!EXP:!SEED:!IDEA:!3DES
O ServerSSLOptions=+SSL_OP_NO_SSLv2 +SSL_OP_NO_SSLv3 +SSL_OP_NO_TLSv1 +SSL_OP_NO_TLSv1_1 +SSL_OP_CIPHER_SERVER_PREFERENCE -SSL_OP_LEGACY_SERVER_CONNECT -SSL_OP_ALLOW_UNSAFE_LEGACY_RENEGOTIATION
O ClientSSLOptions=+SSL_OP_NO_SSLv2 +SSL_OP_NO_SSLv3 +SSL_OP_NO_TLSv1 +SSL_OP_NO_TLSv1_1
}}}

Letsencrypt don't give you a DH file, so lets make one in ''/usr/local/etc/letsencrypt/live/<SMTP-HOSTNAME>/'':

{{{
$ openssl dhparam -out dh.param 4096
Generating DH parameters, 4096 bit long safe prime, generator 2
This is going to take a long time
..............................
<snip>
........................++*++*++*
}}}

Then run ''make'' and ''make install'' in ''/etc/mail''. After this has been done, only a restart is necessary, {{{service sendmail restart}}}

= SPF =
SPF records are TXT records. In the past the SPF record type was used, but now it's just TXT. It will describe where the email will originated from, and only for the domain the record is made for. Not for the subdomains! It's not inherent. Make an extra record for them.

An example for a SPF record is:
{{{
lists.example.org. 3600 TXT "v=spf1 a include:_spf.google.com ~all"
}}}

There are several options.

|| '''Options''' || '''Description''' ||
|| all     || Matches all local and remote IPs and goes at the end of the SPF record. Example: "v=spf1 +all" ||
|| ip4	   || Specifies a single IPv4 address or an acceptable IPv4 address range. A mask of /32 is assumed if no prefix-length is included. Example: "v=spf1 ip4:192.168.0.1/16 -all" ||
|| ip6	   || Same concept found in ip4, but, obviously, with IPv6 addresses, instead. If no prefix-length is given, /128 is assumed (singling out an individual host address). Example: "v=spf1 ip6:1080::8:800:200C:417A/96 -all" ||
|| a	   || Specifies all IPs in the DNS A record. Example: "v=spf1 a:domain.com -all" ||
|| mx	   || Specifies all A records for each host's MX record. Example: "v=spf1 mx mx:domain.com -all" ||
|| ptr	   || Specifies all A records for each host's PTR record. Example: "v=spf1 ptr:domain.com -all" ||
|| exists  || Specifies one or more domains normally singled out as exceptions to the SPF definitions. An A query is performed on the provided domain; if a result is found a match occurs. Example: "v=spf1 exists:domain.com -all" ||
|| include || Specifies other domains that are authorized domains. Example: "v=spf1 include:outlook.microsoft.com -all" ||

And every option has his qualifier.

|| '''Qualifier''' || '''Description''' ||
|| +	|| Pass = The address passed the test; accept the message. Example: "v=spf1 +all" ||
|| -	|| (Hard) Fail = The address failed the test; bounce any e-mail that does not comply. Example: "v=spf1 -all" ||
|| ~	|| Soft Fail = The address failed the test, but the result is not definitive; accept & tag any non-compliant mail. Example: "v=spf1 ~all" ||
|| ?	|| Neutral = The address did not pass or fail the test; do whatever (probably accept the mail). Example: "v=spf1 ?all" ||

The record is evaluated from left to right. More info on the site of [[https://www.digitalocean.com/community/tutorials/how-to-use-an-spf-record-to-prevent-spoofing-improve-e-mail-reliability|Digital Ocean]]

= DMARC =

= DKIM =

= DANE =
DANE is a replacement for the whole CA problems like Diginotar. You need a DNS with DNSSEC enabled. You can use https://www.huque.com/bin/gen_tlsa to generate a TLSA record.

= Checking =
Ways to check your mailserver.

== Manual SMTP session ==
You can telnet to your mailserver and send an email by hand. See also [[https://www.atmail.com/blog/smtp-101-manual-smtp-sessions/]]
{{{
telnet smtp.domain.tld 25

HELO sciuro.org
MAIL FROM: test@sciuro.org
RCPT TO: bo.ter.ham@domain.tld
DATA
From: test@example.org
To: Bo Ter Ham <bo.ter.ham@domain.tld>
Subject: Testmail checking SPF
Hi.    

This is a test mail for checking is SPF is properly configured.

Bye!
.

}}}

= Postfix snippets =
== find-senders-to.sh ==

{{{
#!/bin/bash
set -Eeuo pipefail

TO_ADDR=$1

LOGFILES="/var/log/mail.log*"
LOOKBEHIND=20 # Lines

while IFS= read -r -d '^' segment; do
   id="$(echo "$segment" | grep "to=<$TO_ADDR>" | awk '{print $6}' | cut -d ':' -f 1 | head -n 1)"
   from="$(echo "$segment" | grep -oP "$id: from=<\K.*?(?=>)")"
   [ -z "$from" ] && from='<>'
   echo "$id: from=$from to=$TO_ADDR"
done < <(zgrep "to=<$TO_ADDR>" $LOGFILES -B "$LOOKBEHIND" --group-separator=^)
}}}

== relay-flush.sh ==

{{{
#! /bin/bash
set -Eeuo pipefail

MIN_AGE="${MIN_AGE:-120}"
RELAYHOST="${RELAYHOST:smtp.mail.org}"
DRYRUN="${DRYRUN:-false}"
SILENT="${SILENT:-false}"
POSTFIX_CONFIG="${POSTFIX_CONFIG:-/etc/postfix/main.cf}"

function enable_relay () {
   sed -ie "s/^relayhost \?=.*/relayhost = $RELAYHOST/g" "$POSTFIX_CONFIG"
   systemctl reload postfix.service
   echo "Relay '"$RELAYHOST"' enabled!"
}

function disable_relay () {
   sed -ie "s/^relayhost \?=.*/relayhost =/g" "$POSTFIX_CONFIG"
   systemctl reload postfix.service
   echo "Relay disabled."
}

function get_messages () {
   # Only select the first line for each message
   queue="$(/sbin/postqueue -p | grep -P '\w{3} \w{2,3} \d{1,2} \d\d:\d\d:\d\d  \S*@\S*.\S' || echo '')"
   # Filter out active messages (id appended with '*')
   queue="$(echo "$queue" | grep -v '*' || echo '')"

   current_timestamp="$(date '+%s')"
   
   while IFS= read -r line
   do
      [ "$line" == '' ] && continue
      line_timestamp="$(echo "$line" | awk '{print $4, $5, $6}' | date -f - '+%s')"
      line_queue_id="$(echo "$line" | awk '{print $1}')"
      if (( "$current_timestamp" - "$line_timestamp" < "$MIN_AGE" )); then
         skip_messages+=("$line_queue_id")
         continue
      fi
      flush_messages+=("$line_queue_id")
   done <<< "$queue"
}

function flush_messages () {
   messages=("$@")
   echo "Flushing ${#messages[@]} messages.."
   for id in "${messages[@]}"
   do
      if [ "$DRYRUN" == 'true' ]; then
         echo "$id would have been flushed"
      else
         /sbin/postqueue -vi "$id" 2>&1 | grep flush_send_file | grep status
      fi
   done
}

function wait_until_flushed () {
   echo 'Waiting for flush to finish...'
   active_messages=1
   while [ "$active_messages" -gt 0 ] 
   do
      sleep 1
      active_messages="$(/sbin/postqueue -p | { grep '*' || true; } | { grep -v 'Mail queue is empty' || true; } | wc -l )"
      echo "$active_messages messages remaining"
   done
}

function flush_stale_messages () {
   echo "== Flush stale messages =="
   flush_messages=()
   skip_messages=()

   echo "Getting messages.."
   get_messages
   if [ "${#flush_messages[@]}" -eq 0 ]; then
      echo "No messages to be flushed"
   else
      if [ "$DRYRUN" != 'true' ]; then
         enable_relay
         flush_messages "${flush_messages[@]}"
         wait_until_flushed
         disable_relay
      else
         flush_messages "${flush_messages[@]}"
      fi
   fi

   if [ "${#skip_messages[@]}" -ne 0 ]; then
      echo "${#skip_messages[@]} messages were not flushed" 
   fi
}

function flush_all_messages () {
   echo "== Flush all messages =="
   if [ "$DRYRUN" != 'true' ]; then
      enable_relay
      /sbin/postqueue -f
      wait_until_flushed
      disable_relay
   else
      echo "flush-all has no dryrun option"
   fi
}

function help () {
   echo "Usage: $0 [OPTIONS] [<mode>]"
   echo "Description of what this command does"
   echo " <mode> (stale|all|relay|norelay). Default = stale"
   echo "        stale:   Flush stale messages (older than AGE) to relay"
   echo "        all:     Flush all messages to relay"
   echo "        relay:   Route mail through relay, don't flush"
   echo "        norelay: Don't route mail through relay, don't flush "
   echo ""
   echo " -a     Minimum message age in seconds. Messages younger than AGE are not flushed. Default: $MIN_AGE"
   echo " -r     Relay host. Specify a relay host to flush messages to. Default: $RELAYHOST"
   echo " -d     Dry-run. Don't actually flush messages"
   echo " -s     Silent. Only show errors"
   echo " -h     display this output"
   exit 1
}

while getopts ':a:r:sdh' opt ; do
   case "$opt" in
      a) MIN_AGE="${OPTARG}";;
      r) RELAYHOST="${OPTARG}";;
      d) DRYRUN='true';;
      s) SILENT='true';;
      h) help ;;
      :)
         echo "$0: Must supply an argument to -$OPTARG." >&2
         exit 1
         ;;
      ?)
         echo "Invalid option: -${OPTARG}."
         exit 2
         ;;
   esac
done

mode=${@:$OPTIND:1}

# Prefix STDOUT, STDERR, handle silent mode
if [ "$SILENT" != 'true' ]; then
   exec > >(awk '{ print strftime("[%Y-%m-%d %H:%M:%S]"), $0 }')
else
   exec >/dev/null
fi
exec 2> >(awk '{ print strftime("[%Y-%m-%d %H:%M:%S] ERROR:"), $0 }' >&2)

trap '{ echo "Exited with error!"; }' ERR

if [ "$DRYRUN" == 'true' ]; then
   echo "Dry-run mode!"
else
   if [[ $EUID -ne 0 ]]; then
   echo "This script must be run as root"
   exit 1
   fi
fi

case "$mode" in
   '') flush_stale_messages;;
   'stale') flush_stale_messages;;
   'all') flush_all_messages;;
   'relay') enable_relay;;
   'norelay') disable_relay;;
   ?)
      echo "Invalid mode: $mode"
      help
      ;;
esac
}}}


== oneliners ==

{{{
# Retry all mail for yahoo.com
postqueue -p | tail -n +2 | awk 'BEGIN { RS = "" } /@yahoo\.com/ { print $1 }' | tr -d '*!' | while IFS= read -r line; do postqueue -i "$line"; done

# List delays
grep delay mail.log | awk '{print $9}' | sed -e 's/delay=//g' -e 's/,//g' | sort -n

# List delays for messages sent to outlook
grep mail.protection.outlook.com mail.log | grep -v 'Server busy' | awk '{print $9}' | sed -e 's/delay=//g' -e 's/,//g' | sort -n

# List receive time + id for outlook mails
postqueue -p | tail -n +2 | awk 'BEGIN { RS = "" } /outlook\.com/ { print $6,$1 }' | sort -n

# Show how many times a mail is send/deferred sorted on domain 
grep outlook.com /var/log/mail.log | grep 'status=' | grep 'Jun  8' | awk '{print $6,$7,$12 }' | sed -e 's/to=<.*@//g' -e 's/>//g' | sort -t " " -k2 | uniq -c | less

# Send and deferred mails per domain
grep outlook.com mail.log | grep 'status=' | grep 'Jun  8' | awk '{print $6,$7,$12 }' | sed -e 's/to=<.*@//g' -e 's/>//g' | sort -t " " -k2 | uniq -c | awk '{print $3,$4}' | sort | uniq -c

# Count all mails (from one day)
grep 'Jun  9' /var/log/mail.log | grep to= | awk '{print $6, $7}' | sed -e 's/to=<.*@//g' -e 's/>,//g' | sort | uniq | wc -l

# Count of mails to outlook
grep 'Jun  9' /var/log/mail.log | grep to= | grep outlook.com | awk '{print $6, $7}' | sed -e 's/to=<.*@//g' -e 's/>,//g' | sort | uniq | wc -l

# Look for bounced mails
grep status=bounced /var/log/mail.log | grep -v "Recipient address rejected\|reach does not exist\|Host or domain name not found.\|' not known (\|mailbox unavailable\|User Unknown\|This mailbox is disabled\|All recipient addresses rejected\|#5.1.0 Address rejected." | less

# Show flushed messages
grep queue_id /var/log/relay-flusher/script.log | awk '{print $6}' | while IFS= read line; do sudo grep "$line" /var/log/mail.log | grep 'from=\|to=' | awk '{print $6,$7}' | sort -u | awk '{print $2}' | tr '\n' ' ' ; echo    ; done

# Generate top 'FROM' adresses
grep "from=<" /var/log/mail.log > /tmp/from_lines.txt
while IFS= read -r qid; do grep "$qid from=" /tmp/from_lines.txt | awk '{print $7}'; done < <(grep status=sent /var/log/mail.log | awk '{print $6}') > top_from_adresses.txt
sort top_from_adresses.txt | uniq -c | sort -n > top_from_adresses


qshape deferred

postqueue -p | less

# Force expire & flush all gmail 'mailbox full' messages
while IFS= read -r id; do sudo postsuper -f "$id" && postqueue -i "$id"; done < <(mailq | grep "The email account that you tried to reach is over quota." -B 1 --no-group-separator | grep -v "The email account that you tried to reach is over quota." | awk '{print $1}' | grep -v '^.*#$')
}}}