preprocessing.py

"""Data preprocessing and feature engineering"""

import pandas as pd
import numpy as np
from miner_specs import MINER_SPECS, ELECTRICITY_RATES
from fetch_blockchain_data import get_days_since_halving
from electricity_prices import get_electricity_rate


def engineer_features(blockchain_df):
    """Engineer features from blockchain data - keep it simple"""
    
    df = blockchain_df.copy().sort_values('date').reset_index(drop=True)
    df['date'] = pd.to_datetime(df['date'])
    
    # Just return as is, we'll select features later
    return df


def prepare_miner_features(blockchain_df, miner_name, miner_price, region='texas'):
    """Add miner-specific features - EXACTLY 14 features"""
    
    df = blockchain_df.copy()
    specs = MINER_SPECS[miner_name]
    
    # Keep only these columns from blockchain data
    df = df[['date', 'bitcoin_price', 'difficulty', 'fees', 'hashrate', 'revenue', 'block_reward']].copy()
    df['date'] = pd.to_datetime(df['date'])
    
    # Add miner features
    df['machine_price'] = miner_price
    df['machine_hashrate'] = specs['hashrate']
    df['power'] = specs['power']
    df['efficiency'] = specs['efficiency']
    
    # Calculate age_days (days since miner was released)
    release_date = pd.to_datetime(specs['release_date'])
    df['age_days'] = (df['date'] - release_date).dt.days
    
    # Days since halving
    df['days_since_halving'] = df['date'].apply(get_days_since_halving)
    
    # Revenue potential
    hashrate_hs = df['machine_hashrate'] * 1e12
    btc_per_day = (hashrate_hs * 86400) / (df['difficulty'] * (2**32)) * (df['block_reward'] + (df['fees']/144))
    df['Revenue_Potential'] = btc_per_day * df['bitcoin_price']
    
    # Electricity rate
    # df['electricity_rate'] = ELECTRICITY_RATES.get(region, 0.10)
    df['electricity_rate'] = df['date'].dt.date.apply(
        lambda day: get_electricity_rate(region, day)
    )
    
    return df


def get_latest_sequence(blockchain_df, miner_name, miner_price, region='texas', window_size=30):
    """Get the most recent sequence for prediction - EXACTLY 14 features in CORRECT ORDER"""
    
    df_features = engineer_features(blockchain_df)
    df_miner = prepare_miner_features(df_features, miner_name, miner_price, region)
    
    # CRITICAL: This order MUST match your training data CSV exactly!
    # Your training CSV: bitcoin_price,difficulty,fees,hashrate,revenue,machine_price,machine_hashrate,power,efficiency,block_reward,age_days,days_since_halving,Revenue_Potential,electricity_rate
    feature_cols = [
        'bitcoin_price',      # 1
        'difficulty',         # 2
        'fees',              # 3
        'hashrate',          # 4
        'revenue',           # 5
        'machine_price',     # 6
        'machine_hashrate',  # 7
        'power',             # 8
        'efficiency',        # 9
        'block_reward',      # 10
        'age_days',          # 11
        'days_since_halving',# 12
        'Revenue_Potential', # 13
        'electricity_rate'   # 14
    ]
    
    df_miner = df_miner.dropna().reset_index(drop=True)
    
    if len(df_miner) < window_size:
        raise ValueError(f"Not enough data: need {window_size} days, have {len(df_miner)}")
    
    # Get last window_size days with exactly 14 features
    sequence = df_miner[feature_cols].values[-window_size:]
    latest_date = df_miner['date'].iloc[-1]
    
    # Verify shape
    if sequence.shape[1] != 14:
        raise ValueError(f"Expected 14 features, got {sequence.shape[1]}")
    
    return sequence, feature_cols, latest_date



if __name__ == "__main__":
    from fetch_blockchain_data import get_latest_blockchain_data
    
    print("\n" + "="*80)
    print("TESTING PREPROCESSING PIPELINE")
    print("="*80 + "\n")
    
    # Fetch blockchain data
    print("📡 Fetching blockchain data...")
    blockchain_df = get_latest_blockchain_data(days=90)
    
    if blockchain_df is None:
        print("❌ Failed to fetch blockchain data")
        exit(1)
    
    print(f"✅ Fetched {len(blockchain_df)} days of data\n")
    
    # Test configuration
    miner_name = 's19pro'
    miner_price = 2500
    region = 'texas'
    window_size = 30
    
    print("⚙️  Test Configuration:")
    print(f"   Miner: {MINER_SPECS[miner_name]['full_name']}")
    print(f"   Price: ${miner_price:,}")
    print(f"   Region: {region.title()}")
    print(f"   Window: {window_size} days")
    print(f"   Electricity: ${ELECTRICITY_RATES[region]}/kWh\n")
    
    # Step 1: Engineer features
    print("="*80)
    print("STEP 1: ENGINEER FEATURES")
    print("="*80)
    df_engineered = engineer_features(blockchain_df)
    print(f"✅ Engineered features")
    print(f"   Shape: {df_engineered.shape}")
    print(f"   Columns: {list(df_engineered.columns)}\n")
    print("First 3 rows:")
    print(df_engineered.head(3))
    print("\nLast 3 rows:")
    print(df_engineered.tail(3))
    
    # Step 2: Prepare miner features
    print("\n" + "="*80)
    print("STEP 2: PREPARE MINER FEATURES")
    print("="*80)
    df_miner = prepare_miner_features(df_engineered, miner_name, miner_price, region)
    print(f"✅ Added miner-specific features")
    print(f"   Shape: {df_miner.shape}")
    print(f"   Columns: {list(df_miner.columns)}\n")
    
    print("Miner-specific values (constant across all days):")
    print(f"   machine_hashrate: {df_miner['machine_hashrate'].iloc[0]} TH/s")
    print(f"   power: {df_miner['power'].iloc[0]} W")
    print(f"   efficiency: {df_miner['efficiency'].iloc[0]} W/TH")
    print(f"   machine_price: ${df_miner['machine_price'].iloc[0]:,.2f}")
    print(f"   electricity_rate: ${df_miner['electricity_rate'].iloc[0]:.4f}/kWh")
    
    print("\nDynamic values (change over time):")
    print(f"   age_days: {df_miner['age_days'].iloc[0]} → {df_miner['age_days'].iloc[-1]} days")
    print(f"   days_since_halving: {df_miner['days_since_halving'].iloc[0]} → {df_miner['days_since_halving'].iloc[-1]} days")
    print(f"   Revenue_Potential: ${df_miner['Revenue_Potential'].iloc[0]:.2f} → ${df_miner['Revenue_Potential'].iloc[-1]:.2f}/day")
    
    print("\nFirst 3 rows:")
    print(df_miner.head(3))
    print("\nLast 3 rows:")
    print(df_miner.tail(3))
    
    # Step 3: Get latest sequence
    print("\n" + "="*80)
    print("STEP 3: GET LATEST SEQUENCE")
    print("="*80)
    sequence, feature_cols, latest_date = get_latest_sequence(blockchain_df, miner_name, miner_price, region, window_size)
    
    print(f"✅ Created sequence for model")
    print(f"   Shape: {sequence.shape}")
    print(f"   Expected: ({window_size}, 14)")
    print(f"   Latest date: {latest_date.strftime('%Y-%m-%d')}\n")
    
    print("14 Features (in order):")
    for i, col in enumerate(feature_cols, 1):
        print(f"   {i:2d}. {col:25s} → First: {sequence[0, i-1]:>15.2f}  Last: {sequence[-1, i-1]:>15.2f}")
    
    print("\n" + "="*80)
    print("SEQUENCE STATISTICS")
    print("="*80)
    print("\nFirst day in sequence:")
    for i, col in enumerate(feature_cols):
        print(f"   {col:25s} = {sequence[0, i]:>15.2f}")
    
    print(f"\nLast day in sequence (for prediction on {latest_date.strftime('%Y-%m-%d')}):")
    for i, col in enumerate(feature_cols):
        print(f"   {col:25s} = {sequence[-1, i]:>15.2f}")
    
    # Show some statistics
    print("\n" + "="*80)
    print("FEATURE RANGES")
    print("="*80)
    for i, col in enumerate(feature_cols):
        min_val = sequence[:, i].min()
        max_val = sequence[:, i].max()
        mean_val = sequence[:, i].mean()
        print(f"{col:25s} → Min: {min_val:>12.2f}  Max: {max_val:>12.2f}  Mean: {mean_val:>12.2f}")
    
    print("\n" + "="*80)
    print("✅ PREPROCESSING PIPELINE TEST COMPLETE")
    print("="*80 + "\n")