How to convert raw emails (MIME) from AWS SES to Gmail?

Question

I have a gmail account linked to my domain account.

AWS SES will send messages to my S3 bucket. From there, SNS will forward the message in a raw format to my gmail address.

How do I automatically convert the raw message into a standard email format?

Answer 1

The raw message is in the standard email format. I think what you want to know is how to parse that standard raw email into an object that you can manipulate so that you can forward it to yourself and have it look like a standard email. AWS provides a tutorial on how to forward emails with a lambda function, through SES, by first storing them in your S3 bucket: https://aws.amazon.com/blogs/messaging-and-targeting/forward-incoming-email-to-an-external-destination/

If you follow those instructions, you'll find that the email you recieve comes as an attachment, not looking like a standard email. The following code is an alteration of the Python code provided by AWS that does what you're looking for (substitute this for the code provided in the tutorial):

# Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
# Altered from original by Adam Winter
#
# This file is licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License. A copy of the
# License is located at
#
# http://aws.amazon.com/apache2.0/
#
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
# OF ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.

import os
import boto3
import email
import re
import html
from botocore.exceptions import ClientError
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.application import MIMEApplication
from email.mime.image import MIMEImage

region = os.environ['Region']

def get_message_from_s3(message_id):

    incoming_email_bucket = os.environ['MailS3Bucket']
    incoming_email_prefix = os.environ['MailS3Prefix']

    if incoming_email_prefix:
        object_path = (incoming_email_prefix + "/" + message_id)
    else:
        object_path = message_id

    object_http_path = (f"http://s3.console.aws.amazon.com/s3/object/{incoming_email_bucket}/{object_path}?region={region}")

    # Create a new S3 client.
    client_s3 = boto3.client("s3")

    # Get the email object from the S3 bucket.
    object_s3 = client_s3.get_object(Bucket=incoming_email_bucket,
        Key=object_path)
    # Read the content of the message.
    file = object_s3['Body'].read()

    file_dict = {
        "file": file,
        "path": object_http_path
    }

    return file_dict

def create_message(file_dict):

    stringMsg = file_dict['file'].decode('utf-8')

    # Create a MIME container.
    msg = MIMEMultipart('alternative')

    sender = os.environ['MailSender']
    recipient = os.environ['MailRecipient']

    # Parse the email body.
    mailobject = email.message_from_string(file_dict['file'].decode('utf-8'))
    #print(mailobject.as_string())

    # Get original sender for reply-to
    from_original = mailobject['Return-Path']
    from_original = from_original.replace('<', '');
    from_original = from_original.replace('>', '');
    print(from_original)

    # Create a new subject line.
    subject = mailobject['Subject']
    print(subject)

    if mailobject.is_multipart():

        index = stringMsg.find('Content-Type: multipart/')
        stringBody = stringMsg[index:]
        #print(stringBody)
        stringData = 'Subject: ' + subject + '\nTo: ' + sender + '\nreply-to: ' + from_original + '\n' + stringBody

        message = {
            "Source": sender,
            "Destinations": recipient,
            "Data": stringData
        }
        return message

        for part in mailobject.walk():
            ctype = part.get_content_type()
            cdispo = str(part.get('Content-Disposition'))

            # case for each common content type
            if ctype == 'text/plain' and 'attachment' not in cdispo:
                bodyPart = MIMEText(part.get_payload(decode=True), 'plain', part.get_content_charset())
                msg.attach(bodyPart)

            if ctype == 'text/html' and 'attachment' not in cdispo:
                mt = MIMEText(part.get_payload(decode=True), 'html', part.get_content_charset())
                email.encoders.encode_quopri(mt)
                del mt['Content-Transfer-Encoding']
                mt.add_header('Content-Transfer-Encoding', 'quoted-printable')
                msg.attach(mt)

            if 'attachment' in cdispo and 'image' in ctype:
                mi = MIMEImage(part.get_payload(decode=True), ctype.replace('image/', ''))
                del mi['Content-Type']
                del mi['Content-Disposition']
                mi.add_header('Content-Type', ctype)
                mi.add_header('Content-Disposition', cdispo)
                msg.attach(mi)

            if 'attachment' in cdispo and 'application' in ctype:
                ma = MIMEApplication(part.get_payload(decode=True), ctype.replace('application/', ''))
                del ma['Content-Type']
                del ma['Content-Disposition']
                ma.add_header('Content-Type', ctype)
                ma.add_header('Content-Disposition', cdispo)
                msg.attach(ma)


    # not multipart - i.e. plain text, no attachments, keeping fingers crossed
    else:
        body = MIMEText(mailobject.get_payload(decode=True), 'UTF-8')
        msg.attach(body)

    # The file name to use for the attached message. Uses regex to remove all
    # non-alphanumeric characters, and appends a file extension.
    filename = re.sub('[^0-9a-zA-Z]+', '_', subject_original)

    # Add subject, from and to lines.
    msg['Subject'] = subject
    msg['From'] = sender
    msg['To'] = recipient
    msg['reply-to'] = mailobject['Return-Path']

    # Create a new MIME object.
    att = MIMEApplication(file_dict["file"], filename)
    att.add_header("Content-Disposition", 'attachment', filename=filename)

    # Attach the file object to the message.
    msg.attach(att)
    message = {
        "Source": sender,
        "Destinations": recipient,
        "Data": msg.as_string()
    }
    return message

def send_email(message):
    aws_region = os.environ['Region']
# Create a new SES client.
    client_ses = boto3.client('ses', region)
    # Send the email.
    try:
        #Provide the contents of the email.
        response = client_ses.send_raw_email(
            Source=message['Source'],
            Destinations=[
                message['Destinations']
            ],
            RawMessage={
                'Data':message['Data']
            }
        )

    # Display an error if something goes wrong.
    except ClientError as e:
        print('send email ClientError Exception')
        output = e.response['Error']['Message']
    else:
        output = "Email sent! Message ID: " + response['MessageId']

    return output

def lambda_handler(event, context):
    # Get the unique ID of the message. This corresponds to the name of the file
    # in S3.
    message_id = event['Records'][0]['ses']['mail']['messageId']
    print(f"Received message ID {message_id}")

    # Retrieve the file from the S3 bucket.
    file_dict = get_message_from_s3(message_id)

    # Create the message.
    message = create_message(file_dict)

    # Send the email and print the result.
    result = send_email(message)
    print(result)

Answer 2

For those getting this error:

'bytes' object has no attribute 'encode'

In this line:

body = MIMEText(mailobject.get_payload(decode=True), 'UTF-8')

I could make it work. I am not an expert on this so the code might need some improvement. Also the email body includes html tags. But at least it got delivered.

If decoding the email still fails the error message will appear in your CloudWatch log. Also you will receive an email with the error message.

payload = mailobject.get_payload(decode=True)
try:
    decodedPayload = payload.decode()
    body = MIMEText(decodedPayload, 'UTF-8')
    msg.attach(body)
except Exception as error:
    errorMsg = "An error occured when decoding the email payload:\n" + str(error)
    print(errorMsg)
    body = errorMsg + "\nPlease download it manually from the S3 bucket."
    msg.attach(MIMEText(body, 'plain'))

It is up to you which information you want to add to the error email like the subject or the from address.

Just another hint: With the above code you will get an error because subject_original is undefinded. Just delete the following lines.

# The file name to use for the attached message. Uses regex to remove all
# non-alphanumeric characters, and appends a file extension.
filename = re.sub('[^0-9a-zA-Z]+', '_', subject_original)

# Create a new MIME object.
att = MIMEApplication(file_dict["file"], filename)
att.add_header("Content-Disposition", 'attachment', filename=filename)

# Attach the file object to the message.
msg.attach(att)

As far as I understand this code is supposed to add the original email as an attachment which was not what I wanted.