How to scroll down through the right part of the zillow webpage using Selenium

Question

I'm trying to scroll down to the very bottom of the following website at the right part of the webpage.

I tried it with the following code - but unfortunately it doesn´t scroll down on the right side of the website

import time
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
import os, sys
import xlwings as xw
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver import ActionChains
from webdriver_manager.chrome import ChromeDriverManager
from fake_useragent import UserAgent


if __name__ == '__main__':
  print(f"Checking chromedriver...")
  os.environ['WDM_LOG_LEVEL'] = '0' 
  ua = UserAgent()
  userAgent = ua.random
  options = Options()
  # options.add_argument('--headless')
  options.add_argument("start-maximized")
  options.add_experimental_option("prefs", {"profile.default_content_setting_values.notifications": 1})    
  options.add_experimental_option("excludeSwitches", ["enable-automation"])
  options.add_experimental_option('excludeSwitches', ['enable-logging'])
  options.add_experimental_option('useAutomationExtension', False)
  options.add_argument('--disable-blink-features=AutomationControlled')
  options.add_argument(f'user-agent={userAgent}') 
  srv=Service(ChromeDriverManager().install())
  driver = webdriver.Chrome (service=srv, options=options)    
  waitWebDriver = WebDriverWait (driver, 10)         
  
  link = f"https://www.zillow.com/clayton-county-ga/houses/?searchQueryState=%7B%22pagination%22%3A%7B%7D%2C%22usersSearchTerm%22%3A%22Clayton%20County%2C%20GA%22%2C%22mapBounds%22%3A%7B%22west%22%3A-84.83476148874033%2C%22east%22%3A-84.0313862445997%2C%22south%22%3A33.22700148452994%2C%22north%22%3A33.70472214817801%7D%2C%22regionSelection%22%3A%5B%7B%22regionId%22%3A1622%2C%22regionType%22%3A4%7D%5D%2C%22isMapVisible%22%3Atrue%2C%22filterState%22%3A%7B%22beds%22%3A%7B%22min%22%3A3%7D%2C%22baths%22%3A%7B%22min%22%3A2%7D%2C%22ah%22%3A%7B%22value%22%3Atrue%7D%2C%22sort%22%3A%7B%22value%22%3A%22globalrelevanceex%22%7D%2C%22tow%22%3A%7B%22value%22%3Afalse%7D%2C%22mf%22%3A%7B%22value%22%3Afalse%7D%2C%22con%22%3A%7B%22value%22%3Afalse%7D%2C%22land%22%3A%7B%22value%22%3Afalse%7D%2C%22apa%22%3A%7B%22value%22%3Afalse%7D%2C%22manu%22%3A%7B%22value%22%3Afalse%7D%2C%22apco%22%3A%7B%22value%22%3Afalse%7D%2C%22sqft%22%3A%7B%22min%22%3A1000%2C%22max%22%3A3000%7D%2C%22lot%22%3A%7B%22max%22%3A43560%7D%2C%22built%22%3A%7B%22min%22%3A1965%7D%2C%22gar%22%3A%7B%22value%22%3Atrue%7D%7D%2C%22isListVisible%22%3Atrue%2C%22mapZoom%22%3A11%7D" 

  driver.get (link)       
  time.sleep(WAIT) 

  element = driver.find_element(By.XPATH,"//div[@id='px-captcha']")
  action = ActionChains(driver)
  click = ActionChains(driver)
  action.click_and_hold(element)
  action.perform()
  time.sleep(10)
  action.release(element)
  action.perform()
  time.sleep(0.2)
  action.release(element)     
  time.sleep(WAIT)      

  driver.find_element(By.XPATH,"//h1").click()  
  time.sleep(WAIT)      

  driver.execute_script("window.scrollTo(0, 10000)")               
  time.sleep(5000)    

How can i scroll down to the very bottom of the right side of the page?

At the following part of the site i want to scroll down to the very bottom:

Answer 1

An easy option is to use the Keys.End function. After trying the suggested answer I couldn't get it to work at all. Maybe it doesn't work now if Zillow changed their site. I found this code and it worked really well and was easier to understand.

This first clicks the button on the results to get you on that section of the site. Then hits the END key. Works fine. I had to have a wait timers of 3sec once the page loaded to give it enough time to load.

search_results = driver.find_element(by=By.CLASS_NAME, value="left-option")
search_results.click()
search_results.send_keys(Keys.END)

Answer 2

One of the most smart websites in the internet world is zillow that also provides API.If you really love and want to learn web scraping from heart why hardcode, why not how to extract data from api? Below is an example how to scrape data from api as json response using requests module only.

Script:

import requests

headers = {
    "User-Agent": "Mozilla/5.0 ",
    "Accept": "*/*",
    "Accept-Language": "en-US,en;q=0.5",
    "Pragma": "no-cache",
    "Cache-Control": "no-cache"
}

with requests.Session() as s:
    s.headers.update(headers)
    s.head('https://www.zillow.com/')
    for page in range(1,3):
        params = {
                "searchQueryState": '{"pagination":{"currentPage":2},"usersSearchTerm":"Clayton County, GA","mapBounds":{"west":-84.85604749948251,"east":-84.01010023385751,"south":33.18506146243053,"north":33.746411533949434},"regionSelection":[{"regionId":1622,"regionType":4}],"isMapVisible":false,"filterState":{"beds":{"min":3},"baths":{"min":2},"sqft":{"min":1000,"max":3000},"built":{"min":1965},"isCondo":{"value":false},"isApartment":{"value":false},"isMultiFamily":{"value":false},"isAllHomes":{"value":true},"sortSelection":{"value":"globalrelevanceex"},"lotSize":{"max":43560},"isLotLand":{"value":false},"hasGarage":{"value":true},"isTownhouse":{"value":false},"isManufactured":{"value":false},"isApartmentOrCondo":{"value":false}},"isListVisible":true}',
                "wants": '{"cat1":["listResults"],"cat2":["total"]}'
                }
   

        r = s.get('https://www.zillow.com/search/GetSearchPageState.htm',params=params).json()
        #print(r)
        for card in r['cat1']['searchResults']['listResults']:
            price=card['price']
            print(price)

Output:

$235,000
$299,000
$259,000
$290,000
$255,000
$300,000
$295,000
$250,000
$250,000
$259,900
$290,000
$315,000
$249,000
$289,900

... so on

Answer 3

To scroll down to the very bottom of the zillow website on the right side of the webpage you can scrollIntoView the pagination element once it is visible inducing WebDriverWait for the visibility_of_element_located() and you can use either of the following locator strategies:

• Code Block:

  driver.get("https://www.zillow.com/clayton-county-ga/houses/3-_beds/2.0-_baths/?searchQueryState=%7B%22pagination%22%3A%7B%7D%2C%22usersSearchTerm%22%3A%22Clayton%20County%2C%20GA%22%2C%22mapBounds%22%3A%7B%22west%22%3A-85.19662367135751%2C%22east%22%3A-83.66952406198251%2C%22south%22%3A33.16207210856734%2C%22north%22%3A33.76924644337602%7D%2C%22regionSelection%22%3A%5B%7B%22regionId%22%3A1622%2C%22regionType%22%3A4%7D%5D%2C%22isMapVisible%22%3Atrue%2C%22filterState%22%3A%7B%22beds%22%3A%7B%22min%22%3A3%7D%2C%22baths%22%3A%7B%22min%22%3A2%7D%2C%22sqft%22%3A%7B%22min%22%3A1000%2C%22max%22%3A3000%7D%2C%22built%22%3A%7B%22min%22%3A1965%7D%2C%22con%22%3A%7B%22value%22%3Afalse%7D%2C%22apa%22%3A%7B%22value%22%3Afalse%7D%2C%22mf%22%3A%7B%22value%22%3Afalse%7D%2C%22ah%22%3A%7B%22value%22%3Atrue%7D%2C%22sort%22%3A%7B%22value%22%3A%22globalrelevanceex%22%7D%2C%22lot%22%3A%7B%22max%22%3A43560%7D%2C%22land%22%3A%7B%22value%22%3Afalse%7D%2C%22gar%22%3A%7B%22value%22%3Atrue%7D%2C%22tow%22%3A%7B%22value%22%3Afalse%7D%2C%22manu%22%3A%7B%22value%22%3Afalse%7D%2C%22apco%22%3A%7B%22value%22%3Afalse%7D%7D%2C%22isListVisible%22%3Atrue%2C%22mapZoom%22%3A9%7D")
  driver.execute_script("return arguments[0].scrollIntoView(true);", WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, "//div[@class='search-pagination']"))))
  

• Browser Snapshot: