TD Ameritrade API multiple quotes string Async io message problem

Question

I have the TD API in Python and I'm trying to get 423 quotes from the multiple quote api portal and it doesn't always return 423. Does this sound like an async situation? This is my attempt at implementing async but I'm sure I didn't do it correct. Any pointers?

n = symbols_list
payload = {'symbol':n}

content = requests.get(url = endpoint, params = payload, headers = headers)
data = content.json()
time.sleep(1)

async def get(
    session: aiohttp.ClientSession,
    n: str,
    **kwargs
) -> dict:
    url = endpoint
    headers = headers
    print(f"Requesting {url}")
    resp = await session.request('GET', url=url, **kwargs)
    # Note that this may raise an exception for non-2xx responses
    # You can either handle that here, or pass the exception through
    data = await resp.json()
    print('Received data for {url}')
    return data


async def main(n, **kwargs):
    # Asynchronous context manager.  Prefer this rather
    # than using a different session for each GET request
    async with aiohttp.ClientSession() as session:
        tasks = []
        for symbols in n:
            try:
                tasks.append(get(session=session, n=symbols, **kwargs))
                payload = {'symbol':symbols}
                a = data
                b = a[symbols]['symbol']
                c = a[symbols]['lastPrice']
                d = a[symbols]['netChange']
                e = a[symbols]['totalVolume']
                f = a[symbols]['regularMarketLastPrice']
                g = a[symbols]['highPrice']
                h = a[symbols]['lowPrice']
                quotes = pd.DataFrame({'symbol' : [symbols], 'last' : [c], 'change' : [d], 'volume' : [e]
                              ,'OOCLast' : [f], 'high' : [g], 'low' : [h]})
                quotes.to_sql(name='quotes', con=engine, if_exists='append')
            except:
                pass
                
        # asyncio.gather() will wait on the entire task set to be
        # completed.  If you want to process results greedily as they come in,
        # loop over asyncio.as_com#pleted()
                htmls = await asyncio.gather(*tasks, return_exceptions=True)
                return htmls
               
            


if __name__ == '__main__':
    n = symbols_list
    # ...
    # Either take colors from stdin or make some default here
    await main(n)

Answer 1

This may be a bit more than you were looking for but you can modify it as you wish. I use the first call to get the symbols dataset so I can play around with a large list of symbols and test the api endpoints.

See comments inline:

import pandas as pd
import requests
apikey = 'you api'
sym_list = '[A-Z].*' # this will get ALL symbols tda supports at the time; takes a few seconds to run

endpoint = 'https://api.tdameritrade.com/v1/instruments'
payload ={
'apikey': apikey, #your api key goes here
'symbol': sym_list,
'projection' : 'symbol-regex'
}
response = requests.get(endpoint,params=payload)
print(response.url)
data = response.json()
# print(data)

df = pd.DataFrame(data).T #transpose dataframe 
df = df[df['exchange']=='NYSE'] # filter for just NYSE stock symbols
symbols = df.symbol.unique().tolist() # get symbols to a list for later (over 5,000 symbols)

Note: some sybmols won't return anything, but the following code is pretty consistent in and works well for me.

sym_list = symbols[0:501] # take a slice of the symbols from earlier

chunk_size = 500 # each call has this many symbols returned; pretty sure the max symbols per call is 500, but you can try any number here and it'll just loop the sym_list
num_calls = int(len(sym_list) / chunk_size) + 1

df_hold_list = [] #collect intermidiary calls
for i in range(0,num_calls):
    sym_string = ','.join(sym_list[chunk_size*i:chunk_size*(i+1)]) #make the list into a string
    # print(i, sym_string)
    endpoint = 'https://api.tdameritrade.com/v1/marketdata/quotes'
    payload ={
    'apikey': apikey, #your api key goes here
    'symbol': sym_string,
    }
    response = requests.get(endpoint,params=payload)
    # print(response.url)
    data = response.json()
    # print(data)

    df = pd.DataFrame(data).T #transpose dataframe
    df_hold_list.append(df)

dff = pd.concat(df_hold_list) # final dataframe; from here you can do whatever you wish with the data

Output:

         assetType assetMainType      cusip    symbol                                        description bidPrice  ... markChangeInDouble markPercentChangeInDouble regularMarketPercentChangeInDouble delayed realtimeEntitled assetSubType
A           EQUITY        EQUITY  00846U101         A            Agilent Technologies, Inc. Common Stock   120.80  ...               1.22                    0.9335                             0.9335    True            False          NaN
B           EQUITY        EQUITY  067806109         B                    Barnes Group, Inc. Common Stock    40.47  ...               0.50                    0.9878                             0.9878    True            False          NaN
C           EQUITY        EQUITY  172967424         C                       Citigroup, Inc. Common Stock    72.42  ...               0.09                    0.1244                             0.1244    True            False          NaN
D           EQUITY        EQUITY  25746U109         D                 Dominion Energy, Inc. Common Stock    76.20  ...               0.09                    0.1181                             0.1181    True            False          NaN
E           EQUITY        EQUITY  26874R108         E                            ENI S.p.A. Common Stock    24.00  ...              -0.21                   -0.8564                            -0.8564    True            False          ADR
...            ...           ...        ...       ...                                                ...      ...  ...                ...                       ...                                ...     ...              ...          ...
ENBA        EQUITY        EQUITY  29250N477      ENBA  Enbridge Inc 6.375% Fixed-to-Floating Rate Sub...    16.49  ...               0.00                    0.0000                             0.0000    True            False          NaN
ENBL        EQUITY        EQUITY  292480100      ENBL  Enable Midstream Partners, LP Common Units rep...     6.83  ...               0.09                    1.3216                             1.3216    True            False          NaN
CVII+       EQUITY        EQUITY  17144M110     CVII+  Churchill Capital Corp VII Warrants, each exer...     1.42  ...              -0.07                   -4.6980                            -6.0403    True            False          NaN
SOV-C.CL    EQUITY        EQUITY             SOV-C.CL  Santander Holdings USA, Inc. Dep Shs repstg 1/...    24.99  ...              25.01                    0.0000                             0.0000    True            False          NaN
ENIA        EQUITY        EQUITY  29274F104      ENIA      Enel Americas S.A. American Depositary Shares     8.62  ...              -0.06                   -0.6865                            -0.6865    True            False          ADR

[492 rows x 49 columns]

Answer 2

Jonathan. Do you think that having my api key in the headers portion of the requests.get parenthesis, instead of in the payload brackets above would have anything to do with it?

 ##Quotes

from sqlalchemy import create_engine
import pandas as pd
import requests
import sqlite3


engine = create_engine('sqlite:///iterist.db', echo=False)
con = sqlite3.connect("iterist.db")
cur=con.cursor()
cur.execute('drop table if exists quotes2')

distinct_symbols="""select distinct symbol from index_symbols order by symbol asc"""
main_list=pd.read_sql(distinct_symbols, engine)
df = pd.DataFrame(main_list, columns=['symbol'])
symbols_list=df['symbol'].values.tolist()

n = symbols_list

sym_list = n[0:401] # take a slice of the symbols from earlier

chunk_size = 400 # each call has this many symbols returned; pretty sure the max symbols per call is 500, but you can try any number here and it'll just loop the sym_list
num_calls = int(len(sym_list) / chunk_size) + 1

df_hold_list = [] #collect intermidiary calls
for i in range(0,num_calls):
    sym_string = ','.join(sym_list[chunk_size*i:chunk_size*(i+1)]) #make the list into a string
    #print(i, sym_string)
    endpoint = 'https://api.tdameritrade.com/v1/marketdata/quotes'
    payload ={
    'symbol': sym_string,
    }
    response = requests.get(endpoint,params=payload, headers=headers)
    # print(response.url)
    data = response.json()
    # print(data)
    df = pd.DataFrame(data).T #transpose dataframe
    #print(df)
    df_hold_list.append(df)

dff = pd.concat(df_hold_list) # final dataframe; from here you can do whatever you    wish with the data
dff.to_sql(name='quote2', con=engine, if_exists='replace')