ML Odyssey: Part 2 - Data Manipulation & Visualization with Python

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# Example: Pokemon dataset structure
import pandas as pd

# Sample Pokemon dataset
pokemon_data = pd.DataFrame({
    'name': ['Pikachu', 'Charizard', 'Bulbasaur'],      # Identifier (not typically a feature)
    'attack': [55, 84, 49],                             # Feature 1
    'defense': [40, 78, 49],                            # Feature 2  
    'speed': [90, 100, 45],                             # Feature 3
    'type': ['Electric', 'Fire', 'Grass'],              # Feature 4 (categorical)
    'is_legendary': [False, False, False]               # Label (what we want to predict)
})

print("Dataset shape:", pokemon_data.shape)  # (3 rows, 6 columns)
print("\nFeatures (input):", ['attack', 'defense', 'speed', 'type'])
print("Label (target):", 'is_legendary')

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import scipy.stats as stats  # Import scipy.stats as 'stats'

# Raw numerical data (renamed from 'stats' to 'raw_stats_data' to avoid conflict)
raw_stats_data = np.array([[55, 40], [84, 78], [49, 49]])

# Create a structured DataFrame
data = pd.DataFrame(
    raw_stats_data,
    columns=['Attack', 'Defense'], # Changed to PascalCase for consistency
    index=['Pikachu', 'Charizard', 'Bulbasaur']
)

# Add Pokemon types
data['Type'] = ['Electric', 'Fire/Flying', 'Grass/Poison'] # Changed to PascalCase

# Display the DataFrame
print("Full DataFrame:\n", data)

# --- Visualization: Attack Stats Bar Plot ---
sns.set_theme(style="whitegrid") # Apply a clean seaborn theme
plt.figure(figsize=(8, 5)) # Set a good figure size

# Create a bar plot for Attack Stats
# FIX: Added hue='index' and legend=False to address FutureWarning
sns.barplot(data=data.reset_index(), x='index', y='Attack', hue='index', legend=False, palette='viridis')
plt.title('Pokemon Attack Stats', fontsize=16) # Clearer title
plt.xlabel('Pokemon', fontsize=12) # Clearer label
plt.ylabel('Attack Stat', fontsize=12) # Clearer label
plt.xticks(rotation=0) # Keep labels horizontal
plt.tight_layout()
plt.show()

# --- Descriptive Statistics ---
print("\nAverage Stats:")
print(f"Mean Attack: {np.mean(data['Attack']):.1f}")
print(f"Standard Deviation Attack: {np.std(data['Attack']):.1f}") # Clarified for which stat

# --- Statistical Validation ---
# Example: Are attack and defense significantly correlated?
# 'stats.pearsonr' now correctly refers to the function from scipy.stats
correlation, p_value = stats.pearsonr(data['Attack'], data['Defense'])
print(f"\nStatistical Validation:")
print(f"Attack-Defense correlation: {correlation:.3f}")
print(f"P-value: {p_value:.3f} ({'Significant' if p_value < 0.05 else 'Not significant'} relationship)")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

import pandas as pd
import numpy as np

# Creating a Series (1D array with labels)
s = pd.Series([1, 3, 5, np.nan, 6, 8])
print("Series example:\n", s)

# Creating a DataFrame (2D table)
dates = pd.date_range('20250101', periods=6)
df = pd.DataFrame(np.random.randn(6, 4), 
         index=dates,
         columns=['A', 'B', 'C', 'D'])
print("\nDataFrame example:\n", df)

1
2
3
4
5
6
7
8

# Quick overview
print("First 5 rows:\n", df.head())
print("\nDataFrame info:\n", df.info())
print("\nQuick statistics:\n", df.describe())

# Index and columns
print("\nIndex:", df.index)
print("Columns:", df.columns)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14

# Getting a column
print("Column 'A':\n", df['A'])

# Selecting by position
print("First 3 rows:\n", df[:3])

# Selection by label
print("By labels:\n", df.loc['20250102':'20250104'])

# Selection by position
print("By position:\n", df.iloc[3:5, 0:2])

# Boolean indexing
print("Values > 0:\n", df[df > 0])

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# Statistics
print("Mean by column:\n", df.mean())
print("Mean by row:\n", df.mean(axis=1))

# Applying functions
print("\nApply custom function:")
print(df.apply(lambda x: x.max() - x.min()))

# Grouping
df['E'] = ['one', 'one', 'two', 'three', 'two', 'one']
grouped = df.groupby('E')
print("\nGroup sums:\n", grouped.sum())

1
2
3
4
5
6
7
8
9

# Create some missing data
df2 = df.copy()
df2.iloc[0:2, 0] = np.nan

# Drop rows with missing data
print("Drop NA rows:\n", df2.dropna())

# Fill missing data
print("Fill NA with 0:\n", df2.fillna(0))

1
2
3
4
5
6
7
8

# Concatenating
pieces = [df[:2], df[2:4], df[4:]]
print("Concatenated:\n", pd.concat(pieces))

# Merging
left = pd.DataFrame({'key': ['foo', 'bar'], 'lval': [1, 2]})
right = pd.DataFrame({'key': ['foo', 'bar'], 'rval': [4, 5]})
print("\nMerged:\n", pd.merge(left, right, on='key'))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

import pandas as pd
import numpy as np

# Create sample sales data - this is your "normal" table
np.random.seed(42)  # For consistent results
sales_data = pd.DataFrame({
    'date': pd.date_range('2024-01-01', periods=20, freq='D'),
    'product': np.random.choice(['Laptop', 'Phone', 'Tablet'], 20),
    'sales': np.random.randint(50, 150, 20),
    'region': np.random.choice(['North', 'South', 'East', 'West'], 20),
    'salesperson': np.random.choice(['Alice', 'Bob', 'Charlie'], 20)
})

print("📊 ORIGINAL DATA (first 10 rows):")
print(sales_data.head(10))
print(f"\nTotal rows: {len(sales_data)}")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# 🔥 PIVOT TABLE MAGIC: Transform rows into a summary
pivot_table = sales_data.pivot_table(
    values='sales',           # What to summarize
    index='product',          # Rows: Group by product
    columns='region',         # Columns: Split by region  
    aggfunc='mean',          # How to summarize: average
    margins=True             # Add totals
)

print("📈 PIVOT TABLE - Average Sales by Product and Region:")
print(pivot_table.round(1))

# Compare with manual grouping (much more verbose!)
print("\n🔄 Same result using traditional grouping (more complex):")
manual_summary = sales_data.groupby(['product', 'region'], observed=False)['sales'].mean().unstack(fill_value=0)
print(manual_summary.round(1))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22

# Example 1: Multiple metrics in one table
multi_metric_pivot = sales_data.pivot_table(
    values='sales',
    index='product',
    columns='region',
    aggfunc=['mean', 'sum', 'count'],  # Multiple aggregations!
    margins=True
)
print("📊 Multiple metrics (mean, sum, count):")
print(multi_metric_pivot)

# Example 2: Time-based analysis
sales_data['month'] = sales_data['date'].dt.month
time_pivot = sales_data.pivot_table(
    values='sales',
    index='month',
    columns='product',
    aggfunc='sum',
    fill_value=0
)
print("\n📅 Sales by month and product:")
print(time_pivot)

1
2
3
4
5
6
7
8
9

# 1. Rolling windows for trends
sales_data['rolling_avg'] = sales_data.groupby('product', observed=False)['sales'].rolling(7).mean().reset_index(0, drop=True)

# 2. String operations
sales_data['product_category'] = sales_data['product'].str.upper() + '_CATEGORY'

# 3. Advanced groupby with multiple functions
summary = sales_data.groupby('product', observed=False)['sales'].agg(['mean', 'std', 'count', 'sum'])
print("\nAdvanced groupby:\n", summary)

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

# Complete data exploration pipeline
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

# Load real data (seaborn includes several datasets)
tips = sns.load_dataset('tips')

# Quick exploration pipeline
print("Dataset shape:", tips.shape)
print("\nData types:\n", tips.dtypes)
print("\nMissing values:\n", tips.isnull().sum())
print("\nBasic statistics:\n", tips.describe())

# Create insights with pandas + seaborn
fig, axes = plt.subplots(2, 2, figsize=(12, 10))

# Distribution of tips
sns.histplot(data=tips, x='tip', ax=axes[0,0])
axes[0,0].set_title('Distribution of Tips')

# Relationship between total bill and tip
sns.scatterplot(data=tips, x='total_bill', y='tip', hue='time', ax=axes[0,1])
axes[0,1].set_title('Bill vs Tip by Time')

# Average tip by day and time
tip_summary = tips.groupby(['day', 'time'], observed=False)['tip'].mean().unstack()
sns.heatmap(tip_summary, annot=True, fmt='.2f', ax=axes[1,0])
axes[1,0].set_title('Average Tip by Day and Time')

# Box plot showing tip distribution by day
sns.boxplot(data=tips, x='day', y='tip', ax=axes[1,1])
axes[1,1].set_title('Tip Distribution by Day')

plt.tight_layout()
plt.show()

# Generate insights using pandas
print("\n🔍 Key Insights:")
print(f"• Average tip: ${tips['tip'].mean():.2f}")
print(f"• Best tipping day: {tips.groupby('day', observed=False)['tip'].mean().idxmax()}")
print(f"• Tip percentage: {(tips['tip'] / tips['total_bill'] * 100).mean():.1f}%")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

# Using seaborn for statistical insights
import seaborn as sns
import pandas as pd
import numpy as np

# Load a dataset with interesting relationships
iris = sns.load_dataset('iris')

# Create a comprehensive statistical overview
fig, axes = plt.subplots(2, 2, figsize=(15, 12))

# 1. Correlation heatmap
correlation_matrix = iris.select_dtypes(include=[np.number]).corr()
sns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', center=0, ax=axes[0,0])
axes[0,0].set_title('Feature Correlations')

# 2. Distribution comparison
sns.violinplot(data=iris, x='species', y='petal_length', ax=axes[0,1])
axes[0,1].set_title('Petal Length Distribution by Species')

# 3. Statistical regression
sns.regplot(data=iris, x='petal_length', y='petal_width', ax=axes[1,0])
axes[1,0].set_title('Petal Length vs Width (with regression)')

# 4. Categorical relationships
sns.boxplot(data=iris, x='species', y='sepal_width', ax=axes[1,1])
axes[1,1].set_title('Sepal Width by Species')

plt.tight_layout()
plt.show()

# The famous pairplot - shows all relationships at once
plt.figure(figsize=(10, 8))
sns.pairplot(iris, hue='species', height=2.5)
plt.suptitle('Iris Dataset: All Variable Relationships', y=1.02)
plt.show()

# Statistical insights from the analysis
print("🔍 Key insights from our analysis:")
print("• Strong correlation between petal length and width (r=0.96)")
print("• Clear species separation based on petal measurements")
print("• Setosa species has distinctly different characteristics")
print("• Petal features are more discriminative than sepal features")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

# Time series analysis with pandas
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np

# Create sample time series data
dates = pd.date_range('2023-01-01', '2024-12-31', freq='D')
ts_data = pd.DataFrame({
    'date': dates,
    'value': np.cumsum(np.random.randn(len(dates))) + 100,
    'category': np.random.choice(['A', 'B'], len(dates))
})

# Set date as index for time series operations
ts_data.set_index('date', inplace=True)

# Time series specific operations
ts_data['month'] = ts_data.index.month
ts_data['weekday'] = ts_data.index.day_name()
ts_data['rolling_7d'] = ts_data['value'].rolling(7).mean()
ts_data['rolling_30d'] = ts_data['value'].rolling(30).mean()

# Visualization
fig, axes = plt.subplots(2, 1, figsize=(12, 8))

# Time series plot with multiple rolling averages
ts_data[['value', 'rolling_7d', 'rolling_30d']].plot(ax=axes[0])
axes[0].set_title('Time Series with Rolling Averages')
axes[0].legend(['Original', '7-day Average', '30-day Average'])

# Monthly patterns
monthly_avg = ts_data.groupby('month')['value'].mean()
sns.barplot(x=monthly_avg.index, y=monthly_avg.values, ax=axes[1])
axes[1].set_title('Average Values by Month')
axes[1].set_xlabel('Month')

plt.tight_layout()
plt.show()

print("📅 Time series insights:")
print(f"• Data spans {len(ts_data)} days")
print(f"• Monthly peak: {monthly_avg.idxmax()}")
print(f"• Trend: {'Rising' if ts_data['value'].iloc[-30:].mean() > ts_data['value'].iloc[:30].mean() else 'Falling'}")