Tree-Based Feature Importance [DRAFT]

import numpy as np
import pandas as pd
from sklearn.ensemble import RandomForestClassifier
from sklearn.datasets import make_classification
import matplotlib.pyplot as plt

# Generate synthetic dataset with 50 features (only 10 informative)
X, y = make_classification(
    n_samples=1000,
    n_features=50,
    n_informative=10,
    n_redundant=5,
    n_clusters_per_class=2,
    random_state=42
)

# Create feature names
feature_names = [f'feature_{i}' for i in range(50)]
df = pd.DataFrame(X, columns=feature_names)
df['target'] = y

print(f"Dataset shape: {df.shape}")
print(f"Features: {len(feature_names)}")
print(f"Truly informative: 10")

# Train Random Forest
rf = RandomForestClassifier(
    n_estimators=100,  # Number of trees
    max_depth=10,      # Limit depth to prevent overfitting
    random_state=42
)

rf.fit(df[feature_names], df['target'])

# Get feature importances
importances = rf.feature_importances_

# Create DataFrame for easy viewing
importance_df = pd.DataFrame({
    'feature': feature_names,
    'importance': importances
}).sort_values('importance', ascending=False)

print("\nTop 15 most important features:")
print(importance_df.head(15))

# Plot top 20 features
plt.figure(figsize=(10, 6))
top_20 = importance_df.head(20)
plt.barh(range(len(top_20)), top_20['importance'])
plt.yticks(range(len(top_20)), top_20['feature'])
plt.xlabel('Importance Score')
plt.title('Top 20 Feature Importances (Random Forest)')
plt.gca().invert_yaxis()
plt.tight_layout()
plt.show()

# Keep top K features (e.g., top 20)
K = 20
top_features = importance_df.head(K)['feature'].tolist()

print(f"\nSelected {K} features:")
print(top_features)

# Create reduced dataset
X_reduced = df[top_features]
print(f"\nReduced dataset shape: {X_reduced.shape}")

import xgboost as xgb
from sklearn.datasets import make_regression

# Generate regression dataset
X, y = make_regression(
    n_samples=1000,
    n_features=50,
    n_informative=15,
    random_state=42
)

feature_names = [f'feature_{i}' for i in range(50)]

# Train XGBoost
xgb_model = xgb.XGBRegressor(
    n_estimators=100,
    max_depth=6,
    learning_rate=0.1,
    random_state=42
)

xgb_model.fit(X, y)

# Get importances (default: weight = number of times feature is used for split)
importances = xgb_model.feature_importances_

# Create DataFrame
importance_df = pd.DataFrame({
    'feature': feature_names,
    'importance': importances
}).sort_values('importance', ascending=False)

print("Top 15 features (XGBoost):")
print(importance_df.head(15))

from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler

def select_features_with_rf(df, target_col, n_features=50, random_state=42):
    """
    Select top N features using Random Forest importance.

    Args:
        df: DataFrame with features and target
        target_col: Name of target column
        n_features: Number of features to select
        random_state: Random seed

    Returns:
        selected_features: List of selected feature names
        importance_df: DataFrame with all features and their importance scores
    """
    # Separate features and target
    feature_cols = [col for col in df.columns if col != target_col]
    X = df[feature_cols]
    y = df[target_col]

    # Split data (only train on training set to avoid leakage)
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=random_state
    )

    # Train Random Forest on training set only
    rf = RandomForestClassifier(
        n_estimators=100,
        max_depth=10,
        random_state=random_state,
        n_jobs=-1  # Use all CPU cores
    )

    rf.fit(X_train, y_train)

    # Get importances
    importance_df = pd.DataFrame({
        'feature': feature_cols,
        'importance': rf.feature_importances_
    }).sort_values('importance', ascending=False)

    # Select top N
    selected_features = importance_df.head(n_features)['feature'].tolist()

    # Report
    print(f"Selected {n_features} features from {len(feature_cols)} total")
    print(f"Top 10: {selected_features[:10]}")

    # Validation: Check if selected features actually improve model
    rf_reduced = RandomForestClassifier(
        n_estimators=100,
        max_depth=10,
        random_state=random_state
    )

    # Train on full features
    rf.fit(X_train, y_train)
    score_full = rf.score(X_test, y_test)

    # Train on selected features only
    rf_reduced.fit(X_train[selected_features], y_train)
    score_reduced = rf_reduced.score(X_test[selected_features], y_test)

    print(f"\nValidation accuracy:")
    print(f"  All features ({len(feature_cols)}): {score_full:.3f}")
    print(f"  Selected features ({n_features}): {score_reduced:.3f}")
    print(f"  Difference: {score_reduced - score_full:.3f}")

    return selected_features, importance_df

# Example usage
# selected_features, importance_df = select_features_with_rf(
#     df, target_col='label', n_features=50
# )

# Assume you have labeled data (0=normal, 1=anomaly)
# This might come from historical incidents, SOC analyst labels, etc.

import pandas as pd
from sklearn.ensemble import RandomForestClassifier

# Load OCSF data with labels
# df = pd.read_csv('ocsf_logs_labeled.csv')

# Example feature columns (after flattening nested JSON)
ocsf_features = [
    'severity_id', 'activity_id', 'status_id',
    'actor_user_uid', 'src_endpoint_ip_subnet',
    'dst_endpoint_port', 'http_request_http_method',
    'bytes_per_second', 'failed_login_count_1h',
    'unique_ip_count_24h', 'hour_of_day', 'day_of_week',
    # ... 288 more features
]

# Select top 50 features
selected_features, importance_df = select_features_with_rf(
    df,
    target_col='is_anomaly',
    n_features=50
)

# Top 10 features for security anomaly detection:
# 1. failed_login_count_1h (0.085) - Brute force indicator
# 2. bytes_per_second (0.072) - Data exfiltration
# 3. unique_ip_count_24h (0.068) - Account compromise
# 4. dst_endpoint_port (0.051) - Unusual ports
# 5. hour_of_day (0.047) - Off-hours activity
# ...

print("Top features for anomaly detection:")
print(importance_df.head(10))