Import a zip file to Python Notebook in IBM Data Science Experience(IBM DSX)

Question

I have a zip file train.zip(1.1GB) which I wanted to import into a Python Notebook, unzip and then set out to work on it. I imported it as a String IO object utilizing the option Inert StringIO Object.

from io import StringIO
import requests
import json
import pandas as pd

# @hidden_cell
# This function accesses a file in your Object Storage. The definition contains your credentials.
# You might want to remove those credentials before you share your notebook.
def get_object_storage_file_with_credentials_xxxxxx(container, filename):
    """This functions returns a StringIO object containing
    the file content from Bluemix Object Storage."""

    url1 = ''.join(['https://identity.open.softlayer.com', '/v3/auth/tokens'])
    data = {'auth': {'identity': {'methods': ['password'],
            'password': {'user': {'name': 'member_xxxxxx','domain': {'id': 'xxxxxxx'},
            'password': 'xxxxx),(xxxxx'}}}}}
    headers1 = {'Content-Type': 'application/json'}
    resp1 = requests.post(url=url1, data=json.dumps(data), headers=headers1)
    resp1_body = resp1.json()
    for e1 in resp1_body['token']['catalog']:
        if(e1['type']=='object-store'):
            for e2 in e1['endpoints']:
                        if(e2['interface']=='public'and e2['region']=='dallas'):
                            url2 = ''.join([e2['url'],'/', container, '/', filename])
    s_subject_token = resp1.headers['x-subject-token']
    headers2 = {'X-Auth-Token': s_subject_token, 'accept': 'application/json'}
    resp2 = requests.get(url=url2, headers=headers2)
    return StringIO(resp2.text)

# Your data file was loaded into a StringIO object and you can process the data.
# Please read the documentation of pandas to learn more about your possibilities to load your data.
# pandas documentation: http://pandas.pydata.org/pandas-docs/stable/io.html
data_1 = get_object_storage_file_with_credentials_20e75635ab104e58bd1a6e91635fed51('DefaultProjectxxxxxxxx', 'train.zip')

This gives an output:

data_1
<_io.StringIO at 0x7f8a288cd3a8>

But when I try to use Zipfile to unzip it, I'm greeted with the following error:

from zipfile import ZipFile
file = ZipFile(data_1)

BadZipFile: File is not a zip file

How to I access the file in IBM DSX?

Answer 1

You can use the function shown below to save a zip file from object storage. The credentials argument is the dictionary inserted to code in a DSX notebook. This function is also on gist

import zipfile
from io import BytesIO
import requests
import json
import pandas as pd

def get_zip_file(credentials):

    url1 = ''.join(['https://identity.open.softlayer.com', '/v3/auth/tokens']) 
    data = {'auth': {'identity': {'methods': ['password'], 'password': {'user': {'name': credentials['username'],'domain': {'id': credentials['domain_id']}, 'password': credentials['password']}}}}} 
    headers1 = {'Content-Type': 'application/json'} resp1 = requests.post(url=url1, data=json.dumps(data), headers=headers1) 
    resp1_body = resp1.json() 
    for e1 in resp1_body['token']['catalog']:
        if(e1['type']=='object-store'): 
            for e2 in e1['endpoints']:   
                if(e2['interface']=='public'and e2['region']==credentials['region']): url2 = ''.join([e2['url'],'/', credentials['container'], '/', credentials['filename']]) s_subject_token = resp1.headers['x-subject-token'] headers2 = {'X-Auth-Token': s_subject_token, 'accept': 'application/json'} 

    s_subject_token = resp1.headers['x-subject-token']
    headers2 = {'X-Auth-Token': s_subject_token, 'accept': 'application/json'}
    r = requests.get(url=url2, headers=headers2, stream=True)

    z = zipfile.ZipFile(BytesIO(r.content))
    z.extractall()# save zip contents to disk

    return(z)

z = get_zip_file(credentials)

Answer 2

Extract your uploaded zip file with the code below: use pwd to get your working directory and setup your path folders.

from io import BytesIO
import zipfile

zip_ref = zipfile.ZipFile(BytesIO(streaming_body_1.read()), 'r')
file_paths = zip_ref.namelist()
for path in file_paths:
    zip_ref.extract(path)