"""
SEC Form 4 Insider Trading Crawler DAG
"""

from airflow import DAG
from airflow.operators.python import PythonOperator
from datetime import datetime, timedelta
import feedparser
import requests
from bs4 import BeautifulSoup
import logging
import time

default_args = {
    'owner': 'airflow',
    'start_date': datetime(2025, 1, 1),
    'retries': 2,
    'retry_delay': timedelta(minutes=5),
}

dag = DAG(
    'sec_form4_crawler',
    default_args=default_args,
    description='Crawl SEC Form 4 insider trading filings',
    schedule_interval='*/30 * * * *',  # 30분마다 실행
    catchup=False,
    tags=['sec', 'insider-trading'],
)


def fetch_sec_rss(**context):
    """SEC EDGAR RSS 피드에서 최신 Form 4 가져오기"""
    
    logger = logging.getLogger(__name__)
    
    # SEC 요구사항: User-Agent 필수
    headers = {
        'User-Agent': 'InsiderTradingAgent/1.0 (your-email@example.com)',
        'Accept-Encoding': 'gzip, deflate',
        'Host': 'www.sec.gov'
    }
    
    rss_url = (
        "https://www.sec.gov/cgi-bin/browse-edgar"
        "?action=getcurrent&type=4&count=100&output=atom"
    )
    
    logger.info(f"Fetching RSS from: {rss_url}")
    
    # RSS 요청
    response = requests.get(rss_url, headers=headers, timeout=30)
    response.raise_for_status()
    
    logger.info(f"Response status: {response.status_code}")
    logger.info(f"Response length: {len(response.content)} bytes")
    
    # feedparser로 파싱
    feed = feedparser.parse(response.content)
    
    logger.info(f"Feed entries: {len(feed.entries)}")
    
    filings = []
    for entry in feed.entries:
        filing = {
            'accession_number': entry.id.split('/')[-1],
            'filing_url': entry.link,
            'title': entry.title,
            'company': entry.title.split('-')[0].strip(),
            'filed_at': entry.get('published', ''),
            'summary': entry.get('summary', ''),
        }
        filings.append(filing)
    
    logger.info(f"✅ Found {len(filings)} Form 4 filings")
    
    # XCom에 저장 (다음 Task로 전달)
    context['task_instance'].xcom_push(key='filings', value=filings)
    
    return len(filings)


def parse_form4_details(**context):
    """Form 4 상세 정보 파싱"""
    
    logger = logging.getLogger(__name__)
    
    # 이전 Task에서 데이터 가져오기
    filings = context['task_instance'].xcom_pull(
        task_ids='fetch_sec_rss',
        key='filings'
    )
    
    if not filings:
        logger.warning("No filings to process")
        return 0
    
    logger.info(f"Processing {len(filings)} filings")
    
    parsed_filings = []
    headers = {'User-Agent': 'InsiderTradingAgent/1.0'}
    
    # 처음 5개만 파싱 (테스트)
    for i, filing in enumerate(filings[:5]):
        try:
            logger.info(f"[{i+1}/5] Parsing: {filing['accession_number']}")
            
            # Rate limiting (SEC: 10 requests/sec)
            time.sleep(0.2)
            
            response = requests.get(
                filing['filing_url'],
                headers=headers,
                timeout=15
            )
            response.raise_for_status()
            
            soup = BeautifulSoup(response.content, 'lxml')
            
            # 간단한 정보만 추출 (향후 개선 예정)
            parsed = {
                'accession_number': filing['accession_number'],
                'company': filing['company'],
                'html_length': len(response.text),
                'parsed_at': datetime.now().isoformat(),
            }
            
            parsed_filings.append(parsed)
            logger.info(f"✅ Parsed: {filing['company']}")
            
        except Exception as e:
            logger.error(f"Error: {e}")
            continue
    
    logger.info(f"✅ Successfully parsed {len(parsed_filings)} filings")
    
    context['task_instance'].xcom_push(
        key='parsed_filings',
        value=parsed_filings
    )
    
    return len(parsed_filings)


def send_to_kafka(**context):
    """파싱된 데이터를 Kafka로 전송 (향후 구현)"""
    
    logger = logging.getLogger(__name__)
    
    parsed_filings = context['task_instance'].xcom_pull(
        task_ids='parse_form4_details',
        key='parsed_filings'
    )
    
    if not parsed_filings:
        logger.warning("No parsed filings to send")
        return 0
    
    # TODO: 실제 Kafka Producer 구현
    for filing in parsed_filings:
        logger.info(f"📤 [KAFKA] Would send: {filing['company']}")
    
    return len(parsed_filings)


# Task 정의
task_fetch = PythonOperator(
    task_id='fetch_sec_rss',
    python_callable=fetch_sec_rss,
    dag=dag,
)

task_parse = PythonOperator(
    task_id='parse_form4_details',
    python_callable=parse_form4_details,
    dag=dag,
)

task_kafka = PythonOperator(
    task_id='send_to_kafka',
    python_callable=send_to_kafka,
    dag=dag,
)

# Task 의존성 설정
task_fetch >> task_parse >> task_kafka

headers = {
    'User-Agent': 'InsiderTradingAgent/1.0 (your-email@example.com)',
    'Host': 'www.sec.gov'
}
response = requests.get(rss_url, headers=headers)