Python 使用selenium抓取网页文本和下载音频

#！\usr\bin\env python
# -*- coding: utf-8 -*-

'一个自动从https://podcast.duolingo.com/spanish中下载音频并且爬取文本的程序'
'需要配置下载以下所需库，并且配置好webdriver.Chrome()，否则报错'

from selenium import webdriver
import requests
import re
import os
import shelve
                   
def mainProc():
    '主进程'
    db = openDb()
    get_pages(db)
    get_episodes(db)
    db.close() 

def openDb():
    '打开data文件，如果当前路径不存在，则新建文件并初始化'
    filename = "data.dat"
    if not os.path.exists(filename):
        db = shelve.open("data", writeback=True)
        db["pages"] = []
        db["episodes"] = []
    else:
        db = shelve.open("data", writeback=True)

    return db

def get_pages(db):
    '遍历获取所有页面的网址并保存到shelve文件中'
    # 主页面
    main = 'https://podcast.duolingo.com/spanish'  

    # 循环遍历获取所有页面的网址
    # 第一页则为主页面，不需要在main末尾添加i
    #'https://podcast.duolingo.com/spanish2' 以此类推"
    # 如果页面没有在文件中存在，则尝试访问页面，如果200成功，写入文本
    
    for i in range(1, 100):   
        page = main if i == 1 else main + str(i)
        if not page in db["pages"]:                                 
            r = requests.get(page)
            print(f'{page} with status code {r.status_code}.') 
            if r.status_code != 200:                                
                break 
            db["pages"].append(page)
            # 获取页面所有节目链接并补全连接
            episodes = re.findall('entry-title">\s*<a href="(.*)" rel', r.text)
            for episode in episodes:
                episode = str(main[:-7]) + str(episode[2:])
                db["episodes"].append(episode)
                        
def get_episodes(db):
    '在每一页中遍历所有的单集网址'
    for episode in db["episodes"]:
        r = requests.get(episode)
        print(f'{episode} with status code {r.status_code}.') 
        if r.status_code != 200:
            continue
        # 将页面的文本写入文件中并下载音频
        get_transcript(episode)
        get_audios(r, episode)

def get_transcript(episode):
    # 获取节目单集网址中的文本
    filename = 'transcript/' + episode.split('/')[-1] + '.txt'
    if os.path.exists(filename):
        print(filename, 'existed!')
    else:
        req = requests.get(episode)
        print('{episode} with status code {status}.'.format(episode=episode, status=req.status_code))
        if not os.path.exists('transcript'):
            os.mkdir('transcript')
        with open(filename, 'w+', encoding="utf-8") as fp:
            for lines in re.findall('strong>(.*)</strong>(.*)</p>', req.text):
                for line in lines:
                    fp.write(line)
                fp.write('\n\n')
            print(filename, 'added!')

def get_audios(r, episode):
    audio = "https:" + re.findall('<iframe .* src="(.*)" height', r.text)[0]
    # 自定义下载配置
    chromeOptions = webdriver.ChromeOptions()
    chromeOptions.add_argument("--ignore-certificate-errors")
    prefs = {"download.default_directory":r"E:\Python\code\project\duolingo\audio"}
    chromeOptions.add_experimental_option("prefs", prefs)
    # 下载文件
    print(audio)
    browser = webdriver.Chrome(chrome_options=chromeOptions)
    browser.get(audio)
    if not os.path.exists("audio"):
        os.mkdir("audio")
    browser.find_element_by_id('download-player').click()
    download_status = False
    while not download_status:
        download_status = True
        for i in os.listdir('audio'):
            if i.endswith(".crdownload"):
                download_status = False
                time.sleep(5)
    browser.close()

if __name__ == "__main__":
    mainProc()