I have tried a number of ways, from xpath to link text to get Selenium to locate the \"Next Page\" button, and then click until the last page, but just couldn\'t get this to wor
To get Selenium to locate the Next Page » and then click until the last page you need to induce WebDriverWait for the element_to_be_clickable()
and you can use the following Locator Strategies:
Code Block:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import TimeoutException
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
driver = webdriver.Chrome(options=options, executable_path=r'C:\WebDrivers\chromedriver.exe')
driver.get('https://egov2.manchesternh.gov/Click2GovTX/accountsearch.html')
select = Select(WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "select#searchMethod"))))
select.select_by_value('2')
streetName = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "input.form-control[name='addressName']")))
streetName.send_keys("A")
streetName.send_keys(u'\ue007')
while True:
try:
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//p//following::nav[2]//ul//li[@class='next']//a"))).click()
print("Clicked on Next Page »")
except TimeoutException:
print("No more Next Page »")
break
driver.quit()
Console Output:
Clicked on Next Page »
Clicked on Next Page »
Clicked on Next Page »
Clicked on Next Page »
Clicked on Next Page »
.
.
.
No more Next Page »