when I try to get all article details from query on PubMed to Pandas DataFrame and export them all into CSV

Question

from pymed import PubMed
pubmed = PubMed(tool="PubMedSearcher", email="[email protected]")
search_term = "Your search term"
results = pubmed.query(search_term, max_results=500)
articleList = []
articleInfo = []
for article in results:

Print the type of object we've found (can be either PubMedBookArticle or PubMedArticle). We need to convert it to dictionary with available function

    articleDict = article.toDict()
    articleList.append(articleDict)

Generate list of dict records which will hold all article details that could be fetch from PUBMED API

for article in articleList:

#Sometimes article['pubmed_id'] contains list separated with comma - take first pubmedId in that list - thats article pubmedId

    pubmedId = article['pubmed_id'].partition('\n')[0]

Append article info to dictionary

    articleInfo.append({u'pubmed_id':pubmedId,
                   u'title':article['title'],
                   u'keywords':article['keywords'],
                   u'journal':article['journal'],
                   u'abstract':article['abstract'],
                   u'conclusions':article['conclusions'],
                   u'methods':article['methods'],
                   u'results': article['results'],
                   u'copyrights':article['copyrights'],
                   u'doi':article['doi'],
                   u'publication_date':article['publication_date'], 
                   u'authors':article['authors']})

Generate Pandas DataFrame from list of dictionaries

articlesPD = pd.DataFrame.from_dict(articleInfo)

articlesPD

when I try to execute above code I got KeyError: 'keywords', 'journal', 'conclusions', .. etc.

Answer 1

Based on the following code, article is an instance other than dict, so the fields should be access by . other than get or bracket []

Reference https://github.com/gijswobben/pymed/blob/master/pymed/article.py#L124

class PubMedArticle(object):

    def _initializeFromXML(self: object, xml_element: TypeVar("Element")) -> None:
        """ Helper method that parses an XML element into an article object.
        """

        # Parse the different fields of the article
        self.pubmed_id = self._extractPubMedId(xml_element)
        self.title = self._extractTitle(xml_element)
        self.keywords = self._extractKeywords(xml_element)
        self.journal = self._extractJournal(xml_element)
        self.abstract = self._extractAbstract(xml_element)
        self.conclusions = self._extractConclusions(xml_element)
        self.methods = self._extractMethods(xml_element)
        self.results = self._extractResults(xml_element)
        self.copyrights = self._extractCopyrights(xml_element)
        self.doi = self._extractDoi(xml_element)
        self.publication_date = self._extractPublicationDate(xml_element)
        self.authors = self._extractAuthors(xml_element)
        self.xml = xml_element

The following could be helpful.

from pymed import PubMed

import json

pubmed = PubMed(tool="PubMedSearcher", email="[email protected]")
search_term = "Your search term"
results = pubmed.query(search_term, max_results=500)
articleList = []
articleInfo = []


def get_data(article, name):
    return getattr(article, 'name', 'N/A')


for article in results:
    pubmedId = article.pubmed_id.partition('\n')[0]

    articleInfo.append({
        u'pubmed_id': pubmedId,
        u'title': article.title,
        u'keywords': get_data(article, 'keywords'),
        u'journal': get_data(article, 'journal'),
        u'abstract': article.abstract,
        u'conclusions': get_data(article, 'conclusions'),
        u'methods': get_data(article, 'methods'),
        u'results': get_data(article, 'results'),
        u'copyrights': article.copyrights,
        u'doi': article.doi,
        u'publication_date': article.publication_date,
        u'authors': article.authors})


print(json.dumps(articleInfo, indent=4, default=str))