# Advanced demo: Handle multiple files (CSV/JSON), read/write, preprocess, model with Scikit-learn, evaluate, visualize
import pandas as pd
import json
from sklearn.datasets import fetch_california_housing
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.preprocessing import StandardScaler
import matplotlib.pyplot as plt
import numpy as np

def advanced_multi_file_demo():
    # Generate and write to CSV and JSON
    housing = fetch_california_housing()
    df = pd.DataFrame(housing.data, columns=housing.feature_names)
    df['Target'] = housing.target
    df['Engineered'] = df['AveRooms'] * df['Population']  # Complex feature
    df.to_csv('housing_data.csv', index=False)
    df.to_json('housing_data.json', orient='records')
    # Read multiple files, merge/preprocess
    df_csv = pd.read_csv('housing_data.csv')
    with open('housing_data.json', 'r') as f:
        df_json = pd.DataFrame(json.load(f))
    df_merged = pd.merge(df_csv, df_json, on=housing.feature_names + ['Target', 'Engineered'], how='inner')
    scaler = StandardScaler()
    X_scaled = scaler.fit_transform(df_merged.drop('Target', axis=1))
    y = df_merged['Target']
    # Split and model
    X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.25, random_state=42)
    model = LinearRegression()
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    mse = mean_squared_error(y_test, y_pred)
    r2 = r2_score(y_test, y_pred)
    print(f"MSE: {mse:.2f}, R2: {r2:.2f}")
    # Visualize
    plt.scatter(y_test, y_pred, c='blue', alpha=0.5)
    plt.title('Multi-File Housing Regression Predictions')
    plt.xlabel('True Values')
    plt.ylabel('Predictions')
    plt.annotate('Best Fit', xy=(y_test.mean(), y_pred.mean()), xytext=(y_test.mean() + 0.5, y_pred.mean() + 0.5), arrowprops=dict(facecolor='black', shrink=0.05))
    plt.show()
    # Write evaluated results to new JSON
    results = {'MSE': mse, 'R2': r2}
    with open('housing_results.json', 'w') as f:
        json.dump(results, f)
    print("Results saved to 'housing_results.json'")

advanced_multi_file_demo()