import requests
from bs4 import BeautifulSoup
from pandas.core.frame import DataFrame
import re
import time


class Graduate:
    def __init__(self, province, category):
        self.head = {
            "User-Agent":"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKi"
                         "t/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36"

        }
        self.data = []
        self.province = province
        self.category = category

    def get_list_fun(self, url, name):
        """获取提交表单代码"""
        response = requests.get(url, headers=self.head)
        province = response.json()
        with open("{}.txt".format(name), "w") as f:
            for x in province:
                f.write(str(x))
                f.write("\n")

    def get_list(self):
        """
        分别获取省，学科门类，专业编号数据
        写入txt文件
        """
        self.get_list_fun("http://yz.chsi.com.cn/zsml/pages/getSs.jsp", "province")
        self.get_list_fun('http://yz.chsi.com.cn/zsml/pages/getMl.jsp', "category")
        self.get_list_fun('http://yz.chsi.com.cn/zsml/pages/getZy.jsp', 'major')

    def get_school_url(self):
        """
        输入省份，
        发送post请求，获取数据
        提取数据
        必填省份，学科门类，专业可选填
        返回学校网址

        """
        url = "http://yz.chsi.com.cn/zsml/queryAction.do"
        data = {
            "ssdm": self.province,
            "yjxkdm": self.category,
        }
        response = requests.post(url, data=data, headers=self.head)
        html = response.text
        reg = re.compile(r'(<tr>.*? </tr>)', re.S)
        content = re.findall(reg, html)
        schools_url = re.findall('<a href="(.*?)" target="_blank">.*?</a>',str(content))
        return schools_url

    def get_college_data(self, url):
        """返回一个学校所有学院数据"""
        response = requests.get(url, headers=self.head)
        html = response.text
        colleges_url = re.findall('<td class="ch-table-center"><a href="(.*?)" target="_blank">查看</a>',html)
        return colleges_url

    def get_final_data(self, url):
        """输出一个学校一个学院一个专业的数据"""
        temp = []
        response = requests.get(url, headers=self.head)
        html = response.text
        soup = BeautifulSoup(html, features='lxml')
        summary = soup.find_all('td',{"class":"zsml-summary"})
        for x in summary:
            temp.append(x.get_text())
        self.data.append(temp)

    def get_schools_data(self):
        """获取所有学校的数据"""
        url = "http://yz.chsi.com.cn"
        schools_url = self.get_school_url()
        amount = len(schools_url)
        i = 0
        for school_url in schools_url:
            i +=1
            url_ = url + school_url
            # 找到一个学校对应所有满足学院网址
            colleges_url = self.get_college_data(url_)
            print("已完成第"+str(i)+"/"+ str(amount)+ "学院爬取")
            time.sleep(1)
            for college_url in colleges_url:
                _url = url + college_url
                self.get_final_data(_url)


    def get_data_frame(self):
        """将列表形数据转化为数据框格式"""
        data = DataFrame(self.data)
        data.to_csv("查询招生信息.csv", encoding="utf_8_sig")

if __name__ == '__main__':
    province = input("请输入查询学校省份编号:")
    category = input("请输入查询专业代码:")
    #province = "11"
    #category = "0812"
    spyder = Graduate(province, category)
    spyder.get_schools_data()
    spyder.get_data_frame()