Python: Requests won't POST if I have punctuation in my data

Question

I have a small_file.txt file that contains:

1asdfaksdjfhlaksjdhflkjashdflkjhasldkjfhlaksdfhasdf:
2asdfaksdjfhlaksjdhflkjashdflkjhasldkjfhlaksdfhasdf:
3asdfaksdjfhlaksjdhflkjashdflkjhasldkjfhlaksdfhasdf
4asdfaksdjfhlaksjdhflkjashdflkjhasldkjfhlaksdfhasdf:

Notice the colons at the end, they are just regular strings.

When I try to send it using python requests it doesn't work. For some reason, it waits for the first line with a colon and then sends all the lines starting from there. So for example, in the file above, it will POST only:

3asdfaksdjfhlaksjdhflkjashdflkjhasldkjfhlaksdfhasdf
4asdfaksdjfhlaksjdhflkjashdflkjhasldkjfhlaksdfhasdf:

How can I fix this issue? I'm not sure what is going on.

Here is a simple version of my code:

import requests
import sys
import json
import os


token                    = 'nVQowAng0c'
url                      = "https://api.hipchat.com/v2/room/test_room/share/file"
headers                  = {'Content-type': 'multipart/related; boundary=boundary123456'}
headers['Authorization'] = "Bearer " + token


filepath = 'small_file.csv'
data     = open(filepath, 'rb').read()

payload = """\
--boundary123456
Content-Type: application/json; charset=UTF-8
Content-Disposition: attachment; name="metadata"
--boundary123456
Content-Disposition: attachment; name="file"; filename="{0}"
{1}
--boundary123456--\
""".format(os.path.basename(filepath), data)


r = requests.post(url, headers=headers, data=payload)
r.raise_for_status()

When I try to send something like a .csv file with a timestamp on every row, nothing will get sent because each row has a colon.

Answer 1

Your immediate error is that you misencoded the MIME multipart elements. Each part has two sections, headers and contents, with a double newline between. Yours is missing the second newline, add it in:

payload = """\
--boundary123456
Content-Type: application/json; charset=UTF-8
Content-Disposition: attachment; name="metadata"

--boundary123456
Content-Disposition: attachment; name="file"; filename="{0}"

{1}
--boundary123456--\
""".format(os.path.basename(filepath), data)

I'd not manually build the contents, but re-purpose the requests-toolbelt project to let you upload your data in a streaming fashion:

from requests_toolbelt import MultipartEncoder


class MultipartRelatedEncoder(MultipartEncoder):
    """A multipart/related encoder"""
    @property
    def content_type(self):
        return str(
            'multipart/related; boundary={0}'.format(self.boundary_value)
        )

    def _iter_fields(self):
        # change content-disposition from form-data to attachment
        for field in super(MultipartRelatedEncoder, self)._iter_fields():
            content_type = field.headers['Content-Type']
            field.make_multipart(
                content_disposition='attachment', 
                content_type=content_type)
            yield field


m = MultipartRelatedEncoder(
    fields={
        'metadata': (None, '', 'application/json; charset=UTF-8'),
        'file': (os.path.basename(filepath), open(filepath, 'rb'), 'text/csv'),
    }
)

headers['Content-type'] = m.content_type

r = requests.post(url, data=m, headers=headers)

I've adapted the requests_toolbelt.MultipartEncoder class to emit a multipart/related data stream rather than a multipart/form-data message.

Note that I pass in the open file object, and not the file data itself; this because the MultipartEncoder lets you stream the data to the remote server, the file doesn't have to be read into memory in one.

You probably want to pass in actual JSON data in the metadata part; replace the empty string in the (None, '', 'application/json; charset=UTF-8' tuple with a valid JSON document.

Answer 2

Here is the combined code from @Martijn Pieters:

# do this:
#     pip install requests_toolbelt

from os                import path
from sys               import exit, stderr
from requests          import post
from requests_toolbelt import MultipartEncoder


class MultipartRelatedEncoder(MultipartEncoder):
    """A multipart/related encoder"""
    @property
    def content_type(self):
        return str('multipart/related; boundary={0}'.format(self.boundary_value))

    def _iter_fields(self):
        # change content-disposition from form-data to attachment
        for field in super(MultipartRelatedEncoder, self)._iter_fields():
            content_type = field.headers['Content-Type']
            field.make_multipart(content_disposition = 'attachment',
                                 content_type        = content_type)
            yield field




def hipchat_file(token, room, filepath, host='api.hipchat.com'):

    if not path.isfile(filepath):
        raise ValueError("File '{0}' does not exist".format(filepath))


    url                      = "https://{0}/v2/room/{1}/share/file".format(host, room)
    headers                  = {'Content-type': 'multipart/related; boundary=boundary123456'}
    headers['Authorization'] = "Bearer " + token



    m = MultipartRelatedEncoder(fields={'metadata' : (None, '', 'application/json; charset=UTF-8'),
                                        'file'     : (path.basename(filepath), open(filepath, 'rb'), 'text/csv')})

    headers['Content-type'] = m.content_type

    r = post(url, data=m, headers=headers)

if __name__ == '__main__:

    my_token = <my token>   
    my_room  = <room name>    
    my_file  = <filepath>

    try:
        hipchat_file(my_token, my_room, my_file)
    except Exception as e:
        msg = "[ERROR] HipChat file failed: '{0}'".format(e)
        print(msg, file=stderr)
        exit(1)