使用相同的 url 从不同的页面抓取数据

问：

我正在尝试从此网页（https://www.fplanalytics.com/history1213.html）中抓取数据。我能够从第一页抓取数据，但是一旦我尝试转到下一页，它就会不断返回相同的数据。我注意到它总是检索相同的 URL。

有没有人知道如何从以下页面获取数据？

import requests
import os
import shutil
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
import time
import pandas as pd
from bs4 import BeautifulSoup

#  create list for html years
years= list(range(1213,2122,101))

# import html into python
driver = webdriver.Chrome(
   "C:/Users/aldi/Downloads/chromedriver.exe")
driver.get('https://www.fplanalytics.com/history1213.html')
driver.maximize_window()

soup = BeautifulSoup(driver.page_source, 'html.parser')
table = soup.find('table', {'id':'data-table'})

#create empty dataframe and name columns
columns_names = ["player","team","position", "minutes", "goals", "assists", "cs", "tot pts", "bonus"]
df = pd.DataFrame(columns = columns_names)

#grab table in one page
#trs = table.find_all('tr')[1:]
#for row in trs:
#    row_data = row.find_all('td')
#    row = [td.text for td in row_data]
#    length = len(df)
#    df.loc[length] = row
    
    
while True: 
    soup = BeautifulSoup(driver.page_source, 'html.parser')
    table = soup.find('table', {'id':'data-table'})
    trs = table.find_all('tr')[1:]
    for row in trs:
        row_data = row.find_all('td')
        row = [td.text for td in row_data]
        length = len(df)
        df.loc[length] = row
    try:
        #grabs the url of the next page
        next_page = soup.find('a', class_ = 'page-link').get('href')
        next_page = 'https://www.fplanalytics.com/history1213.html'+next_page
#        driver.get(next_page)
    except:
        break
    
    #Imports the next pages HTML into python
    page = requests.get(next_page)
    soup = BeautifulSoup(page.text, 'lxml')