"""
台北市不動產估價師公會名單抓取工具

資料來源：
- 會員名單：http://www.reaa.org.tw/member_namelist.php
- 助理員名單：http://www.reaa.org.tw/member_namelist_02.php
- 理監事名單：http://www.reaa.org.tw/supervisor_namelist.php

使用方式：
    python -X utf8 scraper.py

輸出檔案：
    - taipei_appraisers.csv    估價師會員名單
    - taipei_assistants.csv    估價助理員名單
    - taipei_supervisors.csv   理監事名單（標記為現任）

建議每月執行一次更新。
"""

import sys
sys.stdout.reconfigure(encoding='utf-8')

import requests
import urllib3
from bs4 import BeautifulSoup
import csv
import time
import os
from datetime import datetime

# 停用 SSL 警告（該網站 SSL 憑證有問題）
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# 設定
BASE_URLS = {
    'members': 'http://www.reaa.org.tw/member_namelist.php',
    'assistants': 'http://www.reaa.org.tw/member_namelist_02.php',
    'supervisors': 'http://www.reaa.org.tw/supervisor_namelist.php'
}

OUTPUT_DIR = os.path.dirname(os.path.abspath(__file__))


def parse_member_table(table):
    """解析單個會員的表格，回傳 dict"""
    rows = table.find_all('tr')
    member = {}
    for row in rows:
        cells = row.find_all('td')
        for i in range(0, len(cells)-1, 2):
            key = cells[i].get_text(strip=True)
            value = cells[i+1].get_text(strip=True)
            if key:
                # 統一欄位名稱
                if key == '開業事務所':
                    key = '事務所'
                member[key] = value
    return member


def fetch_list_page(base_url, page_num):
    """抓取指定頁面的名單"""
    url = f'{base_url}?page={page_num}'
    response = requests.get(url, verify=False, timeout=30)
    response.encoding = 'utf-8'
    soup = BeautifulSoup(response.text, 'html.parser')
    tables = soup.find_all('table', class_='list_form')
    members = []
    for table in tables:
        member = parse_member_table(table)
        if member:
            members.append(member)
    return members


def fetch_all_pages(base_url, list_name):
    """抓取所有頁面的名單"""
    all_members = []
    page = 1
    while page <= 100:  # 最多 100 頁
        print(f'抓取{list_name}第 {page} 頁...')
        members = fetch_list_page(base_url, page)
        if not members:
            print(f'  第 {page} 頁沒有資料，結束')
            break
        all_members.extend(members)
        print(f'  取得 {len(members)} 筆，累計 {len(all_members)} 筆')
        if len(members) < 10:  # 不滿 10 筆表示最後一頁
            break
        page += 1
        time.sleep(0.3)  # 避免請求過快
    return all_members


def save_to_csv(data, filename, fieldnames, update_time):
    """儲存資料至 CSV"""
    # 加入更新時間欄位
    for item in data:
        item['更新時間'] = update_time

    fieldnames_with_time = fieldnames + ['更新時間']

    filepath = os.path.join(OUTPUT_DIR, filename)
    with open(filepath, 'w', newline='', encoding='utf-8-sig') as f:
        writer = csv.DictWriter(f, fieldnames=fieldnames_with_time, extrasaction='ignore')
        writer.writeheader()
        writer.writerows(data)
    print(f'已儲存至 {filepath}')
    return filepath


def main():
    """主程式"""
    update_time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    print(f'開始抓取台北市不動產估價師公會名單')
    print(f'更新時間：{update_time}')
    print('=' * 60)

    results = {}

    # === 1. 會員名單 ===
    print('\n【估價師會員名單】')
    members = fetch_all_pages(BASE_URLS['members'], '會員')
    results['members'] = len(members)
    save_to_csv(
        members,
        'taipei_appraisers.csv',
        ['姓名', '會員編號', '聯絡電話', '傳真', '電子信箱', '事務所', '地址'],
        update_time
    )

    # === 2. 助理員名單 ===
    print('\n【估價助理員名單】')
    assistants = fetch_all_pages(BASE_URLS['assistants'], '助理員')
    results['assistants'] = len(assistants)
    save_to_csv(
        assistants,
        'taipei_assistants.csv',
        ['姓名', '聯絡電話', '事務所', '地址'],
        update_time
    )

    # === 3. 理監事名單 ===
    print('\n【理監事名單】')
    url = BASE_URLS['supervisors']
    response = requests.get(url, verify=False, timeout=30)
    response.encoding = 'utf-8'
    soup = BeautifulSoup(response.text, 'html.parser')
    tables = soup.find_all('table', class_='list_form')

    supervisors = []
    for table in tables:
        member = parse_member_table(table)
        if member:
            member['屆別'] = '現任'
            supervisors.append(member)
    results['supervisors'] = len(supervisors)
    print(f'取得 {len(supervisors)} 位理監事')
    save_to_csv(
        supervisors,
        'taipei_supervisors.csv',
        ['姓名', '職稱', '聯絡電話', '傳真', '電子信箱', '事務所', '地址', '屆別'],
        update_time
    )

    # === 摘要 ===
    print('\n' + '=' * 60)
    print('抓取完成！')
    print('=' * 60)
    print(f'估價師會員：{results["members"]} 位')
    print(f'估價助理員：{results["assistants"]} 位')
    print(f'理監事：{results["supervisors"]} 位')
    print(f'\n更新時間：{update_time}')

    return results


if __name__ == '__main__':
    main()