Data Preparation for Machine Learning

import pandas as pd

# From CSV file
data = pd.read_csv('customer_data.csv')

# From Excel
data = pd.read_excel('sales_data.xlsx')

# From database (SQL)
import sqlite3
conn = sqlite3.connect('company.db')
data = pd.read_sql_query("SELECT * FROM customers", conn)

# From API (example: weather data)
import requests
response = requests.get('https://api.weather.com/data')
data = pd.DataFrame(response.json())

# First look at your data
print(data.head())  # First 5 rows
print(data.info())  # Column types and non-null counts
print(data.shape)   # (rows, columns)

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# Load data
data = pd.read_csv('data.csv')

# 1. Basic Info
print(data.head())           # First few rows
print(data.info())           # Data types, null counts
print(data.describe())       # Statistics (mean, std, min, max)

# 2. Check for Missing Values
print(data.isnull().sum())   # Count nulls per column

# 3. Visualize Distribution
data['age'].hist(bins=20)
plt.title('Age Distribution')
plt.show()

# 4. Box plot for outliers
data.boxplot(column='salary')
plt.show()

# 5. Correlation heatmap
correlation = data.corr()
sns.heatmap(correlation, annot=True, cmap='coolwarm')
plt.show()

# 6. Scatter plot for relationships
plt.scatter(data['experience'], data['salary'])
plt.xlabel('Years of Experience')
plt.ylabel('Salary')
plt.show()

import pandas as pd
import numpy as np

# Load data
df = pd.read_csv('messy_data.csv')

# 1. Check for missing values
print(df.isnull().sum())

# 2. Handle missing values
# Option A: Drop rows with any missing values
df_clean = df.dropna()

# Option B: Fill numeric columns with mean
df['age'].fillna(df['age'].mean(), inplace=True)
df['income'].fillna(df['income'].median(), inplace=True)

# Option C: Fill categorical with mode
df['city'].fillna(df['city'].mode()[0], inplace=True)

# 3. Remove duplicates
df = df.drop_duplicates()

# 4. Handle outliers using IQR method
Q1 = df['salary'].quantile(0.25)
Q3 = df['salary'].quantile(0.75)
IQR = Q3 - Q1

# Remove outliers
df = df[(df['salary'] >= Q1 - 1.5*IQR) &
        (df['salary'] <= Q3 + 1.5*IQR)]

# 5. Standardize text data
df['country'] = df['country'].str.lower().str.strip()
df['country'] = df['country'].replace({
    'u.s.a.': 'usa',
    'united states': 'usa'
})

# 6. Check data types
print(df.dtypes)
df['date'] = pd.to_datetime(df['date'])

print("Clean data shape:", df.shape)

import pandas as pd
import numpy as np

# 1. Mathematical transformations
df['bmi'] = df['weight'] / (df['height'] ** 2)
df['price_per_sqft'] = df['price'] / df['square_feet']
df['profit'] = df['revenue'] - df['cost']

# 2. Date/time features
df['date'] = pd.to_datetime(df['date'])
df['year'] = df['date'].dt.year
df['month'] = df['date'].dt.month
df['day_of_week'] = df['date'].dt.dayofweek
df['is_weekend'] = df['day_of_week'].isin([5, 6]).astype(int)
df['hour'] = df['date'].dt.hour

# 3. Text features
df['review_length'] = df['review'].str.len()
df['word_count'] = df['review'].str.split().str.len()
df['has_exclamation'] = df['review'].str.contains('!').astype(int)

# 4. Binning
df['age_group'] = pd.cut(df['age'],
                         bins=[0, 18, 35, 60, 100],
                         labels=['Child', 'Young Adult', 'Adult', 'Senior'])

# 5. Interaction features
df['location_time'] = df['city'] + '_' + df['time_of_day']

# 6. Aggregation features (customer-level)
customer_stats = df.groupby('customer_id').agg({
    'purchase_amount': ['sum', 'mean', 'count'],
    'date': lambda x: (pd.Timestamp.now() - x.max()).days
})
customer_stats.columns = ['total_spent', 'avg_purchase',
                          'num_purchases', 'days_since_last']

from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.model_selection import train_test_split
import pandas as pd

# Load clean data
df = pd.read_csv('clean_data.csv')

# Separate features and target
X = df.drop('target', axis=1)
y = df['target']

# 1. Encode categorical variables
categorical_cols = X.select_dtypes(include=['object']).columns
X_encoded = pd.get_dummies(X, columns=categorical_cols)

# 2. Split data BEFORE scaling (important!)
X_train, X_test, y_train, y_test = train_test_split(
    X_encoded, y, test_size=0.2, random_state=42
)

# 3. Scale features (fit only on training data!)
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)  # Only transform, don't fit!

# Now ready for modeling!
print("Training shape:", X_train_scaled.shape)
print("Test shape:", X_test_scaled.shape)

from sklearn.model_selection import train_test_split

# Basic split (80% train, 20% test)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# With stratification (maintains class balance)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, stratify=y, random_state=42
)

# Three-way split (60% train, 20% val, 20% test)
# First split: 80% train+val, 20% test
X_temp, X_test, y_temp, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)
# Second split: 75% of 80% = 60% train, 25% of 80% = 20% val
X_train, X_val, y_train, y_val = train_test_split(
    X_temp, y_temp, test_size=0.25, random_state=42
)

print(f"Train: {len(X_train)}, Val: {len(X_val)}, Test: {len(X_test)}")