# Complex demo: Load California Housing, preprocess (scale, select features), regress, evaluate, visualize
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.datasets import fetch_california_housing
from sklearn.preprocessing import StandardScaler
from sklearn.feature_selection import SelectKBest, f_regression
import matplotlib.pyplot as plt

def advanced_housing_preprocessing():
    # Load and clean
    housing = fetch_california_housing()
    df = pd.DataFrame(housing.data, columns=housing.feature_names)
    df['Target'] = housing.target
    df = df.fillna(df.mean(numeric_only=True))
    
    # Preprocessing: Feature engineering
    df['RoomsPerHouse'] = df['AveRooms'] / (df['HouseAge'] + 1e-5)
    
    # Scale features
    scaler = StandardScaler()
    X_scaled = scaler.fit_transform(df.drop('Target', axis=1))
    df_scaled = pd.DataFrame(X_scaled, columns=df.drop('Target', axis=1).columns)
    df_scaled['Target'] = df['Target']
    
    # Feature selection
    selector = SelectKBest(score_func=f_regression, k=5)
    X_selected = selector.fit_transform(df_scaled.drop('Target', axis=1), df_scaled['Target'])
    selected_features = df_scaled.drop('Target', axis=1).columns[selector.get_support()].tolist()
    print("Selected Features:", selected_features)
    
    # Split
    X = X_selected
    y = df_scaled['Target']
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)
    
    # Train and predict
    model = LinearRegression()
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    
    # Evaluate
    mse = mean_squared_error(y_test, y_pred)
    r2 = r2_score(y_test, y_pred)
    print(f"Mean Squared Error: {mse:.2f}")
    print(f"R² Score: {r2:.2f}")
    
    # Visualize
    plt.scatter(y_test, y_pred, c='blue', alpha=0.5)
    plt.xlabel('True Values')
    plt.ylabel('Predictions')
    plt.title('Preprocessed Housing Regression')
    plt.annotate('Best Fit', xy=(y_test.mean(), y_pred.mean()), xytext=(y_test.mean() + 0.5, y_pred.mean() + 0.5), arrowprops=dict(facecolor='black', shrink=0.05))
    plt.show()

advanced_housing_preprocessing()