I'm using the Bing News API to collect counts of news articles over select time periods.
However, I'm getting larger counts when I reduce the time period for some reason - when I tell my code to collect counts for 6 weeks after February, I get a smaller number than when I tell it to collect 3 weeks worth of data. What might be the problem?
import time
from datetime import datetime
api_key = 'MY API KEY'
endpoint = 'https://api.bing.microsoft.com/v7.0/news/search'
headers = {
'Ocp-Apim-Subscription-Key': api_key,
'Accept': 'application/json'
}
search_term = 'MY SEARCH TERMS'
count = 100
start_date = datetime(2022, 2, 24) # Set the start date
total_articles = 0
while start_date < datetime.now():
end_date = start_date + timedelta(days=42) # Define a time range
##Here is the problem: when I select 21, i get a larger number than when I do 21
payload = {
'q': search_term,
'count': count,
'freshness': 'Day',
'mkt': 'en-US',
'from': start_date.strftime('%Y-%m-%dT00:00:00Z'),
'to': end_date.strftime('%Y-%m-%dT00:00:00Z')
}
response = requests.get(endpoint, headers=headers, params=payload)
if response.status_code == 200:
news_data = response.json()
total_articles += news_data['totalEstimatedMatches'] if 'totalEstimatedMatches' in news_data else 0
print('Processing....')
elif response.status_code == 429:
print(f"Rate limit exceeded. Waiting before making the next request...")
time.sleep(10) # Wait for 5 seconds before making the next request
continue # Retry the same request after the delay
else:
print(f"Request failed for date range {start_date} - {end_date} with status code {response.status_code}")
start_date = end_date
print(f"Number of articles about {search_term}: {total_articles}")