# Project Title: [Descriptive Analysis/Model Name]

**Author:** [Your Name]  
**Date:** [YYYY-MM-DD]  
**Version:** [1.0]  
**Environment:** Python [3.x], Jupyter [version]

---

## 📋 Project Overview

**Objective:** [Brief description of what this notebook accomplishes]

**Dataset:** [Data source and description]

**Key Questions:**
- Question 1
- Question 2
- Question 3

**Expected Outcomes:**
- Outcome 1
- Outcome 2

---

# =============================================================================
# SETUP & CONFIGURATION
# =============================================================================

# Standard data science imports
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from scipy import stats
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import classification_report, confusion_matrix

# Configuration
plt.style.use('seaborn-v0_8')
sns.set_palette("husl")
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', 100)
np.random.seed(42)

# Constants
DATA_PATH = '../data/'
FIGURE_SIZE = (12, 8)
RANDOM_STATE = 42

# Jupyter display settings
from IPython.display import display, HTML, Markdown
%matplotlib inline
%config InlineBackend.figure_format = 'retina'

print("✅ Setup complete")
print(f"📦 Pandas version: {pd.__version__}")
print(f"📦 NumPy version: {np.__version__}")

# =============================================================================
# DATA LOADING & INITIAL OVERVIEW
# =============================================================================

def load_and_validate_data(filepath, expected_cols=None):
    """
    Load data with basic validation and overview
    """
    try:
        df = pd.read_csv(filepath)
        print(f"✅ Data loaded successfully")
        print(f"📊 Shape: {df.shape}")

        if expected_cols and not all(col in df.columns for col in expected_cols):
            missing_cols = set(expected_cols) - set(df.columns)
            print(f"⚠️ Missing expected columns: {missing_cols}")

        return df
    except Exception as e:
        print(f"❌ Error loading data: {e}")
        return None

# Load dataset
df = load_and_validate_data(DATA_PATH + 'dataset.csv')

# Quick overview
if df is not None:
    display(HTML("<h3>📋 Dataset Overview</h3>"))
    print(f"Rows: {df.shape[0]:,} | Columns: {df.shape[1]}")
    print(f"Memory usage: {df.memory_usage(deep=True).sum() / 1024**2:.2f} MB")

    display(HTML("<h4>Sample Data</h4>"))
    display(df.head())

    display(HTML("<h4>Data Types & Missing Values</h4>"))
    info_df = pd.DataFrame({
        'DataType': df.dtypes,
        'Non_Null_Count': df.count(),
        'Null_Count': df.isnull().sum(),
        'Null_Percentage': (df.isnull().sum() / len(df) * 100).round(2)
    })
    display(info_df)

# =============================================================================
# EXPLORATORY DATA ANALYSIS
# =============================================================================

# Helper function for consistent plotting
def create_subplot_figure(nrows, ncols, figsize=None):
    if figsize is None:
        figsize = (5*ncols, 4*nrows)
    fig, axes = plt.subplots(nrows, ncols, figsize=figsize)
    if nrows == 1 and ncols == 1:
        axes = [axes]
    elif nrows == 1 or ncols == 1:
        axes = axes.flatten()
    return fig, axes

# Numerical variables analysis
numerical_cols = df.select_dtypes(include=[np.number]).columns.tolist()
if numerical_cols:
    display(HTML("<h3>📈 Numerical Variables Analysis</h3>"))
    display(df[numerical_cols].describe())

    # Distribution plots
    n_cols = min(3, len(numerical_cols))
    n_rows = (len(numerical_cols) + n_cols - 1) // n_cols

    fig, axes = create_subplot_figure(n_rows, n_cols)
    for i, col in enumerate(numerical_cols):
        row, col_idx = divmod(i, n_cols)
        ax = axes[row][col_idx] if n_rows > 1 else axes[i]

        df[col].hist(bins=30, ax=ax, alpha=0.7)
        ax.set_title(f'Distribution of {col}')
        ax.set_ylabel('Frequency')

    plt.tight_layout()
    plt.show()

# Categorical variables analysis
categorical_cols = df.select_dtypes(include=['object']).columns.tolist()
if categorical_cols:
    display(HTML("<h3>📊 Categorical Variables Analysis</h3>"))
    for col in categorical_cols:
        value_counts = df[col].value_counts()
        print(f"\n{col}: {len(value_counts)} unique values")
        display(value_counts.head(10))

---

## 🎯 Key Findings

### Data Quality Assessment
- **Missing Data:** [X]% of records have missing values
- **Data Types:** [Summary of data type issues]
- **Outliers:** [Number and percentage of outliers detected]

### Exploratory Analysis Results
1. **Finding 1:** [Description with supporting evidence]
2. **Finding 2:** [Description with supporting evidence]
3. **Finding 3:** [Description with supporting evidence]

### Statistical Summary
| Metric | Value | Interpretation |
|--------|-------|---------------|
| Sample Size | [N] | [Context] |
| Key Correlation | [r=X.XX] | [Meaning] |
| Primary Insight | [Value] | [Business Impact] |

---

## 📋 Conclusions

### Summary
[2-3 sentences summarizing the main outcomes]

### Recommendations
1. **Immediate Actions:** [What should be done first]
2. **Further Analysis:** [Additional questions to explore]
3. **Data Collection:** [What additional data would be valuable]

### Next Steps
- [ ] Action item 1
- [ ] Action item 2
- [ ] Action item 3

---

## 📚 References

- **Data Source:** [URL or description]
- **Documentation:** [Links to relevant docs]
- **Related Work:** [Citations or links]

# Add to setup cell for git integration
import subprocess
import datetime

# Get git info
try:
    git_hash = subprocess.check_output(['git', 'rev-parse', 'HEAD']).decode('ascii').strip()
    git_branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']).decode('ascii').strip()
    print(f"🔗 Git: {git_branch} ({git_hash[:7]})")
except:
    print("📝 Not in git repository")

print(f"⏰ Executed: {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")

# Add timing and memory monitoring
import time
import psutil
import functools

def monitor_performance(func):
    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        start_time = time.time()
        start_memory = psutil.Process().memory_info().rss / 1024 / 1024

        result = func(*args, **kwargs)

        end_time = time.time()
        end_memory = psutil.Process().memory_info().rss / 1024 / 1024

        print(f"⏱️ {func.__name__}: {end_time - start_time:.2f}s")
        print(f"💾 Memory change: {end_memory - start_memory:+.2f} MB")

        return result
    return wrapper

# Usage: @monitor_performance above any function

# Clean notebook for sharing
from nbconvert.preprocessors import ClearOutputPreprocessor
from nbconvert import NotebookExporter

def clean_notebook_output(notebook_path):
    """Remove all outputs from notebook for clean sharing"""
    clear_output = ClearOutputPreprocessor()
    exporter = NotebookExporter(preprocessors=[clear_output])

    with open(notebook_path) as f:
        nb = f.read()

    (body, resources) = exporter.from_filename(notebook_path)

    with open(notebook_path.replace('.ipynb', '_clean.ipynb'), 'w') as f:
        f.write(body)