diff --git a/haraka.sh b/haraka.sh
index f2b008a..0309186 100644
--- a/haraka.sh
+++ b/haraka.sh
@@ -19,6 +19,7 @@ if [ "${USE_LETSENCRYPT,,}" == "yes" -o "${USE_LETSENCRYPT,,}" == "true" ]; then
   ln -sf /etc/letsencrypt/live/$HOSTNAME/fullchain.pem tls-cert.pem
 else
   # Generate a self-signed certificate for the hostname provided
+  rm -f tls-key.pem tls-cert.pem
   openssl req -subj "/O=Self Signed Certificate/CN=$HOSTNAME" -new -newkey rsa:4096 -days 365 -nodes -x509 -sha256 -keyout tls-key.pem -out tls-cert.pem
 fi
 cd -