ML Odyssey: Part 4 - Exploratory Data Analysis with Pokemon Dataset I

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

import pandas as pd

# Load the Pokemon Team Rocket dataset directly from GitHub
dataset_url = "https://raw.githubusercontent.com/alvarolop/ml-odyssey/main/datasets/pokemon_team_rocket_dataset.csv"

print("Loading Pokemon Team Rocket dataset from GitHub...")
all_pokemon_df = pd.read_csv(dataset_url)
print(f"Dataset loaded successfully! Shape: {all_pokemon_df.shape}")

# Alternative: Download and save locally if you prefer
# pokemon_df.to_csv('pokemon_team_rocket_dataset.csv', index=False)
# print("Dataset also saved locally as 'pokemon_team_rocket_dataset.csv'")

1
2
3
4
5
6
7

# --- Split the dataset into training and prediction sets ---
pokemon_df = all_pokemon_df[all_pokemon_df['Team Rocket'].notnull()].copy()
predict_df = all_pokemon_df[all_pokemon_df['Team Rocket'].isnull()].copy()

# --- Confirm the dataset sizes ---
print(f'Size of the labeled dataset (To Train): {pokemon_df.shape}')
print(f'Size of the non-labeled dataset (To predict): {predict_df.shape}')

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

import matplotlib.pyplot as plt
import seaborn as sns
# --- Refined Basic Exploration ---
print(f"{'='*50}\n{'DATASET OVERVIEW':^50}\n{'='*50}")
print(f"Total Number of Pokemon: {len(pokemon_df)}\n")
print(f"Features (Columns) Available: {', '.join(pokemon_df.columns.tolist())}\n")

print(f"\n{'='*50}\n{'DATA TYPES AND NON-NULL COUNTS':^50}\n{'='*50}")
pokemon_df.info()

# --- Refined Basic Statistics ---
print(f"\n{'='*50}\n{'BASIC STATISTICAL SUMMARY':^50}\n{'='*50}")
# Transpose the describe output for better readability (features as rows)
print(pokemon_df.describe().T.to_string())

# --- Display the first few rows ---
print(f"\n{'='*50}\n{'FIRST 5 POKEMON RECORDS':^50}\n{'='*50}")
print(pokemon_df.head().to_string()) # Use .to_string() for better console formatting

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

# --- Review the objective labels proportionality and counts ---
print("Distribution of the 'Team Rocket' objective variable (proportion):")
print(pokemon_df['Team Rocket'].value_counts(normalize=True))
print("\nCount of each class in the 'Team Rocket' objective variable:")
print(pokemon_df['Team Rocket'].value_counts())

# --- Visualize the objective labels ---
sns.countplot(x='Team Rocket', hue='Team Rocket', data=pokemon_df, palette='viridis', legend=False)
plt.title('Distribution of the "Team Rocket" Objective Variable')
plt.ylabel('Count')
plt.xlabel('Does it belong to Team Rocket?')
plt.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18

# --- Step 1: Identify categorical variables ---
categorical_vars = pokemon_df.select_dtypes(include='object').columns

# --- Step 2: Analyze categorical variables ---
print(f"{'='*50}\n{'Number of categories per categorical variable':^50}\n{'='*50}")
for col in categorical_vars:
    num_categories = pokemon_df[col].nunique()
    print(f"{col}: {num_categories} categories")

print(f"{'='*50}\n{'Sample values for each categorical variable':^50}\n{'='*50}")
for col in categorical_vars:
    unique_values = pokemon_df[col].unique()
    # Show first 5 unique values for readability
    sample_values = unique_values[:5] if len(unique_values) > 5 else unique_values
    print(f"\n{col}:")
    print(f"  Sample values: {list(sample_values)}")
    if len(unique_values) > 5:
        print(f"  ... and {len(unique_values) - 5} more categories")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

from sklearn.preprocessing import LabelEncoder

# --- Step 3: Apply Label Encoding ---
print(f"{'='*50}\n{'Applying Label Encoding to categorical variables...':^50}\n{'='*50}")
label_encoders = {} # Dictionary to store label encoders for later use

# Apply label encoding to all categorical variables
for col in categorical_vars:
    if col != 'Team Rocket':
      le = LabelEncoder()
      pokemon_df[col] = le.fit_transform(pokemon_df[col].astype(str))
      predict_df[col] = le.fit_transform(predict_df[col].astype(str))
      label_encoders[col] = le
      
      print(f"\n✅ Encoded '{col}':")
      print(f"   Pokemon Dataset: {list(pokemon_df[col].unique()[:3])}")
      print(f"   Predict Dataset:  {list(predict_df[col].unique()[:3])}")

print(f"\n📊 Encoding complete! Dataset shape: {pokemon_df.shape}")
print(f"📋 All variables are now numerical and ready for ML algorithms!")

# Print the label_encoders dictionary to show the mapping for each column
print(f"\n🔑 Label Encoders mapping per column:")
for col, le in label_encoders.items():
    print(f"  {col}: {dict(zip(le.classes_, le.transform(le.classes_)))}")

# Display the first few rows to see the transformation
print(f"\n👀 First 3 rows of encoded dataset:")
print(pokemon_df.head(3))

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

from scipy.stats import chi2_contingency

# Group by Team Rocket status and Pokemon type, count occurrences
type_status_counts = pd.crosstab(
  pokemon_df['Most Used Pokemon Type'],
  pokemon_df['Team Rocket']
)

# Map the integer index back to the original type names for the x-axis
type_status_counts.index = label_encoders['Most Used Pokemon Type'].inverse_transform(type_status_counts.index)

# Plot stacked bar chart
type_status_counts.plot(
  kind='bar',
  stacked=True,
  figsize=(10, 6),
  colormap='plasma'
)
plt.title('Pokémon Type Distribution: Rocket vs Non-Rocket')
plt.xlabel('Pokémon Type')
plt.ylabel('Count')
plt.xticks(rotation=45, ha='right')
plt.legend(title='Team Rocket')
plt.tight_layout()
plt.show()

# Chi-square test for independence using scipy
chi2, p, dof, expected = chi2_contingency(type_status_counts)
print(f"Chi-square statistic: {chi2:.2f}")
print(f"p-value: {p:.4f}")
if p < 0.05:
  print("Result: 🏆 Significant association between Pokémon type and Team Rocket membership.")
else:
  print("Result: 🧐 No significant association found.")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30

from scipy.stats import f_oneway

# Box plot: Debt by Team Rocket status
plt.figure(figsize=(10, 6))
sns.boxplot(
  data=pokemon_df,
  x='Team Rocket',
  y='Debt to Kanto',
  hue='Team Rocket',
  palette='Set2',
  legend=False
)
plt.title('Debt Distribution by Team Rocket Status')
plt.xlabel('Team Rocket Member')
plt.ylabel('Debt to Kanto')
plt.show()

# ANOVA test for Debt to Kanto
debt_groups = [
  group['Debt to Kanto'].dropna()
  for _, group in pokemon_df.groupby('Team Rocket')
]
debt_anova = f_oneway(*debt_groups)
print(f"Debt to Kanto ANOVA F-statistic: {debt_anova.statistic:.2f}")
print(f"Debt to Kanto ANOVA p-value: {debt_anova.pvalue:.4f}")

if debt_anova.pvalue < 0.05:
  print("Result: 🏆 Significant debt differences found between Team Rocket and non-Team Rocket members.")
else:
  print("Result: 🧐 No significant debt differences found.")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

# Count plot: Economic Status by Team Rocket status
plt.figure(figsize=(10, 6))
sns.countplot(
  data=pokemon_df,
  x='Economic Status',
  hue='Team Rocket',
  palette='Set3'
)
plt.title('Economic Status Frequency by Team Rocket Status')
plt.xlabel('Economic Status')
plt.ylabel('Count')

# Replace numeric labels with original string labels on X-axis
ax = plt.gca()
numeric_labels = [int(label.get_text()) for label in ax.get_xticklabels()]
original_labels = label_encoders['Economic Status'].inverse_transform(numeric_labels)
ax.set_xticks(numeric_labels)  # Set tick positions first
ax.set_xticklabels(original_labels, rotation=15)  # Then set labels

plt.tight_layout()
plt.show()

# ANOVA test for Economic Status
# Economic Status is already encoded as numeric, so we can use it directly
econ_groups = [
  group['Economic Status'].dropna()
  for _, group in pokemon_df.groupby('Team Rocket')
]
econ_anova = f_oneway(*econ_groups)
print(f"Economic Status ANOVA F-statistic: {econ_anova.statistic:.2f}")
print(f"Economic Status ANOVA p-value: {econ_anova.pvalue:.4f}")

if econ_anova.pvalue < 0.05:
  print("Result: 🏆 Significant economic status differences found between Team Rocket and non-Team Rocket members.")
else:
  print("Result: 🧐 No significant economic status differences found.")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28

# --- Heatmap of PokéBall usage vs Team Rocket status ---
# Step 1: Create a cross-tab of counts
pokeball_ct = pd.crosstab(pokemon_df['PokéBall Usage'], pokemon_df['Team Rocket'])

# Step 2: Normalize by column (each column will sum to 1.0 or 100%)
pokeball_percent = pokeball_ct.div(pokeball_ct.sum(axis=0), axis=1).multiply(100)

pokeball_percent.index = label_encoders['PokéBall Usage'].inverse_transform(pokeball_percent.index)

# Step 3: Format labels with % sign and no decimals
# The fix is here: changed from "{x:.1f}%" to "{x:.0f}%"
labels = pokeball_percent.map(lambda x: f"{x:.0f}%")

# Step 4: Plot
sns.heatmap(pokeball_percent, annot=labels, fmt='', cmap='coolwarm')
plt.title('PokéBall Usage (% within Team Rocket groups)')
plt.xlabel('Team Rocket Status')
plt.ylabel('PokéBall Usage')
plt.show()

# Chi-square test for independence
chi2, p, _, _ = chi2_contingency(pokeball_ct)
print(f"Chi-square statistic: {chi2:.2f}")
print(f"p-value: {p:.4f}")
if p < 0.05:
    print("Result: 🏆 Significant association between PokéBall usage and Team Rocket membership.")
else:
    print("Result: 🧐 No significant association found.")

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26

from scipy.stats import ttest_ind

# Separate win ratio data by Team Rocket membership
win_ratio_rocket = pokemon_df[pokemon_df['Team Rocket'] == 'Yes']['Win Ratio'].dropna()
win_ratio_nonrocket = pokemon_df[pokemon_df['Team Rocket'] == 'No']['Win Ratio'].dropna()

# Calculate means for comparison
mean_rocket = win_ratio_rocket.mean()
mean_nonrocket = win_ratio_nonrocket.mean()
print(f"Mean Win Ratio - Team Rocket: {mean_rocket:.2f}, Non-Rocket: {mean_nonrocket:.2f}\n")

# Create KDE plot with mean indicators
plt.figure(figsize=(10, 6))
colors = sns.color_palette('mako', n_colors=2)
sns.kdeplot(data=pokemon_df, x='Win Ratio', hue='Team Rocket', fill=True, common_norm=False, palette=colors)

# Add mean lines
plt.axvline(x=mean_rocket, color=colors[1], linestyle='--', label=f"Rocket Mean: {mean_rocket:.2f}")
plt.axvline(x=mean_nonrocket, color=colors[0], linestyle='--', label=f"Non-Rocket Mean: {mean_nonrocket:.2f}")

plt.title('Win Ratio Distribution by Team Rocket Membership')
plt.xlabel('Win Ratio')
plt.ylabel('Density')
plt.legend()
plt.tight_layout()
plt.show()

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16

# --- T-test for Mean Difference ---
# Perform an independent t-test. The 'equal_var=False' parameter
# performs a Welch's t-test, which is generally more robust.
ttest_result = ttest_ind(
    win_ratio_rocket,
    win_ratio_nonrocket,
    equal_var=False
)

print(f"T-test statistic: {ttest_result.statistic:.2f}")
print(f"p-value: {ttest_result.pvalue:.4f}")

if ttest_result.pvalue < 0.05:
    print("Result: 🏆 Significant difference in mean Win Ratio between Team Rocket and non-Team Rocket members.")
else:
    print("Result: 🧐 No significant difference in mean Win Ratio between groups.")