import re
import requests
from bs4 import BeautifulSoup
import argparse
import os
script = argparse.ArgumentParser(prog='豆瓣爬虫',description='这个脚本可以帮助你爬出豆瓣信息顺便存储本地',epilog='祝你爬取成功 直供学习交流')
script.add_argument('-u','--url',type=str,help='爬取的url链接',required=True)
script.add_argument('-p','--page',type=str,help='爬取的总页数',required=True)
script.add_argument('-r','--path',type=str,help='输入你的路径',required=False)
a = script.parse_args()
print(a)
if __name__ == "__main__":
    def Song(url='https://music.douban.com/tag/%E7%BE%8E%E5%9B%BD',count=0):
        # 我爬取的是音乐流行曲 学习交流
        n = 0
        header = {
            'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36 Edg/96.0.1054.62"
        }
        parse = {
            'start':count,
            'type':'T'
        }
        Song = {
            'name': [],
            'discription': [],
            'eval': []
        }
        while n < count:
            result = requests.get(url=url, headers=header,params=parse)
            soup = BeautifulSoup(result.text, "lxml")
            # 歌曲数量
            SongNum = soup.findAll('div', class_='pl2')
            SongTitle = []
            for item in SongNum:
                item = str(item.find('a').getText())
                # 头部 缩短
                item = item.replace("n            ", "")
                # 中部
                item = item.replace("    ", "  ")
                # 尾部
                item = item.replace("n", "")
                # 单描述
                item = item.replace("     ", "")
                SongTitle.append(item)
            #     每列的歌曲
            Song['name'].append(SongTitle)
            SongDis = []
            # discription
            dis = soup.findAll('p', class_='pl')
            Disreg = re.compile('<p class="pl">(.*)</p>')
            for item in dis:
                item = Disreg.match(str(item)).group(1)
                SongDis.append(item)
            Song['discription'].append(SongDis)
            Songeval = []
            evaReg = re.compile('.*?(d+人评价).*')
            eva = soup.findAll('span', class_='pl')
            for item in eva:
                item = str(item).replace("n", "")
                item = item.replace("                ", "")
                item = item.replace("    ", "")
                item = evaReg.match(item)
                Songeval.append(item.group(1))
            Song['eval'].append(Songeval)
            if result.status_code == 200:
                n += 1
            print('正在采集....{}次 请稍等后....'.format(n))
            pass
        # 转化为json
        S = []
        for i in range(n):
            for index in range(20):
                Son = {}
                Son["name"] = Song["name"][i][index]
                Son["discription"] = Song["discription"][i][index]
                Son["eval"] = Song["eval"][i][index]
                S.append(Son)
        import json
        Song = json.dumps(S, ensure_ascii=False)
        # print("转化后的JSON: {}".format(Song))
        try:
            if os.path.exists(a.path):
                with open(a.path,"w",encoding='utf-8') as f:
                    f.write(Song)
            else:
                with open("inks.txt","w",encoding='utf-8') as f:
                    f.write(Song)
        except Exception as q:
            print('保存失败 可能是保存位置出错')
        else:
            print('保存到了：{}'.format(a.path))
    Song(a.url,int(a.page))