diff options
author | Dennis Kobert <dennis@kobert.dev> | 2025-03-11 00:15:45 +0100 |
---|---|---|
committer | Dennis Kobert <dennis@kobert.dev> | 2025-03-25 15:52:04 +0100 |
commit | 7f49625ee1cfadaf523da9e2c247fded90d82665 (patch) | |
tree | 290fb6c6b84c817010b915dcf058371ebb831803 | |
parent | 776dee32aca73ded5c0720afbe25ef2aae7b67f3 (diff) |
Select counters
-rw-r--r-- | best_counters.py | 315 | ||||
-rw-r--r-- | power_predictor.py | 26 | ||||
-rw-r--r-- | src/benchmark.rs | 252 |
3 files changed, 389 insertions, 204 deletions
diff --git a/best_counters.py b/best_counters.py new file mode 100644 index 0000000..feb5900 --- /dev/null +++ b/best_counters.py @@ -0,0 +1,315 @@ +import pandas as pd +import numpy as np +import matplotlib.pyplot as plt +from sklearn.model_selection import train_test_split, cross_val_score +from sklearn.ensemble import RandomForestRegressor +from sklearn.feature_selection import RFE, mutual_info_regression +from sklearn.preprocessing import StandardScaler +from sklearn.impute import KNNImputer +from sklearn.metrics import r2_score, mean_squared_error +from itertools import combinations + +def load_and_preprocess_data(csv_path): + # Load the CSV file + df = pd.read_csv(csv_path) + + # Extract target (power usage) + y = df['package_power_j'].values + + # Extract features (CPU frequency and performance counters) + X = df.iloc[:, 3:] # Skip timestamp, duration_ms, and power + + # Replace empty strings with NaN + X = X.replace('', np.nan) + + # Convert all values to float + X = X.astype(float) + + # Impute missing values using KNN + imputer = KNNImputer(n_neighbors=5) + X_imputed = imputer.fit_transform(X) + X = pd.DataFrame(X_imputed, columns=X.columns) + + return X, y + +def calculate_mutual_information(X, y): + # Calculate mutual information for each feature + mi_scores = mutual_info_regression(X, y) + + # Create a DataFrame of features and their importance scores + feature_importance = pd.DataFrame({ + 'Feature': X.columns, + 'Importance': mi_scores + }) + + # Sort by importance + feature_importance = feature_importance.sort_values('Importance', ascending=False) + + # Print top features + print("\nFeature importance by mutual information:") + for i, (_, row) in enumerate(feature_importance.head(10).iterrows(), 1): + print(f"{i}. {row['Feature']} - importance: {row['Importance']:.4f}") + + # Visualize feature importance + plt.figure(figsize=(10, 6)) + plt.barh(feature_importance['Feature'], feature_importance['Importance']) + plt.xlabel('Mutual Information Score') + plt.title('Feature Importance') + plt.tight_layout() + plt.savefig('feature_importance.png') + + return feature_importance + +def find_best_features_rfe(X, y, n_features=5): + print("\nRunning Recursive Feature Elimination...") + model = RandomForestRegressor(n_estimators=100, random_state=42) + selector = RFE(estimator=model, n_features_to_select=n_features, step=1) + selector.fit(X, y) + + # Get the selected features + selected_features = X.columns[selector.support_].tolist() + print(f"Selected features (RFE): {selected_features}") + return selected_features + +def select_uncorrelated_features(X, feature_importances, top_n=5, correlation_threshold=0.75): + print(f"\nSelecting {top_n} uncorrelated features...") + + # Convert feature importances to dictionary for easy lookup + importance_dict = {row['Feature']: row['Importance'] + for _, row in feature_importances.iterrows()} + + # Calculate correlation matrix + corr_matrix = X.corr().abs() + + # Start with the most important feature + top_feature = feature_importances.iloc[0]['Feature'] + selected = [top_feature] + print(f"Starting with top feature: {top_feature}") + + candidates = X.columns.tolist() + candidates.remove(top_feature) + + # Greedy selection of uncorrelated features + while len(selected) < top_n and candidates: + best_feature = None + max_importance = -np.inf + + for feature in candidates: + # Check correlation with already selected features + correlations = [corr_matrix.loc[feature, sel] for sel in selected] + if max(correlations) < correlation_threshold: + # Use mutual information score as importance + importance = importance_dict[feature] + if importance > max_importance: + max_importance = importance + best_feature = feature + + if best_feature: + selected.append(best_feature) + candidates.remove(best_feature) + print(f"Added {best_feature} (correlation with selected: {max([corr_matrix.loc[best_feature, sel] for sel in selected[:-1]])})") + else: + # If no feature satisfies correlation threshold, relax the threshold + old_threshold = correlation_threshold + correlation_threshold += 0.05 + print(f"No features satisfy threshold {old_threshold}, relaxing to {correlation_threshold}") + + print(f"Selected uncorrelated features: {selected}") + return selected + +def evaluate_feature_combination(X, y, feature_combo): + X_subset = X[list(feature_combo)] + model = RandomForestRegressor(n_estimators=100, random_state=42) + scores = cross_val_score(model, X_subset, y, cv=5, scoring='r2') + return scores.mean(), scores.std() + +def find_best_feature_combination(X, y, n_features=5, top_k=10): + print(f"\nFinding the best combination of {n_features} features...") + + # Always include frequency if available + freq_col = 'cpu_frequency_mhz' if 'cpu_frequency_mhz' in X.columns else None + + # For efficiency, limit the search to the top_k most important features + feature_importances = calculate_mutual_information(X, y) + top_features = feature_importances.head(top_k)['Feature'].tolist() + + if freq_col and freq_col not in top_features: + top_features = [feat for feat in top_features if feat != freq_col] + top_features = top_features[:n_features-1] # Make room for frequency + else: + top_features = top_features[:n_features] + + best_score = -np.inf + best_combo = None + best_std = 0 + + # Try all combinations + total_combos = sum(1 for _ in combinations(top_features, n_features if not freq_col else n_features-1)) + print(f"Testing {total_combos} combinations...") + + for i, combo in enumerate(combinations(top_features, n_features if not freq_col else n_features-1)): + if i % 10 == 0: + print(f"Evaluating combination {i+1}/{total_combos}") + + features = list(combo) + if freq_col and freq_col not in features: + features.append(freq_col) + + score, std = evaluate_feature_combination(X, y, features) + + if score > best_score: + best_score = score + best_std = std + best_combo = features + + print(f"Best feature combination (R²={best_score:.4f}±{best_std:.4f}): {best_combo}") + return best_combo, best_score, best_std + +def compare_feature_subsets(X, y, n_features=5): + print("\nComparing different feature selection methods...") + + # Always include frequency if available + freq_col = 'cpu_frequency_mhz' if 'cpu_frequency_mhz' in X.columns else None + + # Get feature importances + feature_importances = calculate_mutual_information(X, y) + + # Method 1: Top features by mutual information + if freq_col: + mi_top = feature_importances['Feature'].tolist() + if freq_col in mi_top: + mi_top.remove(freq_col) + mi_top = mi_top[:n_features-1] + [freq_col] + else: + mi_top = feature_importances['Feature'].tolist()[:n_features] + + # Method 2: Recursive Feature Elimination + rfe_selected = find_best_features_rfe(X, y, n_features if not freq_col else n_features-1) + if freq_col and freq_col not in rfe_selected: + rfe_selected.append(freq_col) + + # Method 3: Uncorrelated features + uncorrelated = select_uncorrelated_features(X, feature_importances, + n_features if not freq_col else n_features-1) + if freq_col and freq_col not in uncorrelated: + uncorrelated.append(freq_col) + + # Method 4: Best combination (limited search) + best_combo, _, _ = find_best_feature_combination(X, y, n_features, top_k=10) + + # Define subsets to test + subsets = { + 'mutual_info_top': mi_top, + 'rfe_selected': rfe_selected, + 'uncorrelated': uncorrelated, + 'best_combination': best_combo + } + + # Create a visual representation of the correlation matrix + plt.figure(figsize=(12, 10)) + corr_matrix = X.corr() + plt.imshow(corr_matrix, cmap='coolwarm', interpolation='none', vmin=-1, vmax=1) + plt.colorbar() + plt.xticks(range(len(X.columns)), X.columns, rotation=90) + plt.yticks(range(len(X.columns)), X.columns) + plt.title('Feature Correlation Matrix') + plt.tight_layout() + plt.savefig('correlation_matrix.png') + + # Evaluate each subset with a RandomForestRegressor + results = {} + for name, features in subsets.items(): + X_subset = X[features] + scores = cross_val_score(RandomForestRegressor(n_estimators=100, random_state=42), + X_subset, y, cv=5, scoring='r2') + + results[name] = { + 'r2_mean': scores.mean(), + 'r2_std': scores.std(), + 'features': features + } + + # Print results + print("\nResults for different feature subsets:") + for name, res in results.items(): + print(f"{name}: R²={res['r2_mean']:.4f}±{res['r2_std']:.4f}, Features: {res['features']}") + + # Create bar chart of results + plt.figure(figsize=(10, 6)) + methods = list(results.keys()) + scores = [results[m]['r2_mean'] for m in methods] + errors = [results[m]['r2_std'] for m in methods] + + plt.bar(methods, scores, yerr=errors, capsize=5) + plt.ylabel('R² Score') + plt.title('Performance of Different Feature Selection Methods') + plt.ylim(0.90, 1.0) # Adjust as needed + plt.tight_layout() + plt.savefig('feature_selection_comparison.png') + + # Find the best method + best_method = max(results.items(), key=lambda x: x[1]['r2_mean']) + print(f"\nBest method: {best_method[0]} with R²={best_method[1]['r2_mean']:.4f}") + print(f"Final recommended feature set: {best_method[1]['features']}") + + return results, best_method[1]['features'] + +def final_model_evaluation(X, y, selected_features): + print("\nEvaluating final model with selected features...") + + # Prepare data + X_selected = X[selected_features] + X_train, X_test, y_train, y_test = train_test_split( + X_selected, y, test_size=0.2, random_state=42) + + # Train model + model = RandomForestRegressor(n_estimators=100, random_state=42) + model.fit(X_train, y_train) + + # Evaluate + y_pred = model.predict(X_test) + r2 = r2_score(y_test, y_pred) + rmse = np.sqrt(mean_squared_error(y_test, y_pred)) + + print(f"Final model performance: R²={r2:.4f}, RMSE={rmse:.6f}") + + # Plot actual vs predicted + plt.figure(figsize=(8, 8)) + plt.scatter(y_test, y_pred, alpha=0.5) + plt.plot([min(y_test), max(y_test)], [min(y_test), max(y_test)], 'r--') + plt.xlabel('Actual Power Usage (J)') + plt.ylabel('Predicted Power Usage (J)') + plt.title(f'Actual vs Predicted Power Usage with Selected Features\nR²={r2:.4f}, RMSE={rmse:.6f}') + plt.tight_layout() + plt.savefig('final_model_performance.png') + + # Feature importances in the final model + importances = model.feature_importances_ + indices = np.argsort(importances)[::-1] + + plt.figure(figsize=(10, 6)) + plt.title('Feature Importances in Final Model') + plt.bar(range(X_selected.shape[1]), importances[indices]) + plt.xticks(range(X_selected.shape[1]), np.array(selected_features)[indices], rotation=90) + plt.tight_layout() + plt.savefig('final_model_feature_importances.png') + + return model, r2, rmse + +def main(): + # Load and preprocess data + print("Loading and preprocessing data...") + X, y = load_and_preprocess_data('logs.csv') + + # Compare different feature selection methods + results, best_features = compare_feature_subsets(X, y, n_features=5) + + # Evaluate final model with selected features + model, r2, rmse = final_model_evaluation(X, y, best_features) + + print("\nAnalysis complete! The optimal set of 5 performance counters has been identified.") + print(f"Final selected features: {best_features}") + print(f"Model performance: R²={r2:.4f}, RMSE={rmse:.6f}") + +if __name__ == "__main__": + main() diff --git a/power_predictor.py b/power_predictor.py index 67c73ae..49e49fe 100644 --- a/power_predictor.py +++ b/power_predictor.py @@ -72,7 +72,7 @@ def analyze_feature_importance(X, y): plt.savefig('feature_importance.png') print("\nTop 5 most important features:") - for i, (_, row) in enumerate(feature_importance.head(5).iterrows(), 1): + for i, (_, row) in enumerate(feature_importance.head(15).iterrows(), 1): print(f"{i}. {row['Feature']} - importance: {row['Importance']:.4f}") return feature_importance @@ -205,6 +205,26 @@ def evaluate_model(model, X, y, scaler): return mse, rmse, mae, r2 +def remove_outliers(X, y, threshold=3): + # Scale features for distance calculations + scaler = StandardScaler() + X_scaled = scaler.fit_transform(X) + + # Calculate z-scores for target values + z_scores = np.abs((y - np.mean(y)) / np.std(y)) + + # Identify inliers + inliers = z_scores < threshold + + # Remove outliers + X_clean = X.iloc[inliers] + y_clean = y[inliers] + + removed = len(y) - len(y_clean) + print(f"Removed {removed} outliers ({removed/len(y)*100:.2f}% of data)") + + return X_clean, y_clean + # Step 7: Main function def main(): csv_path = 'logs.csv' @@ -212,6 +232,8 @@ def main(): # Load and prepare data print("Loading data...") X, y = load_data(csv_path) + + X, y = remove_outliers(X, y) # Impute missing values print("\nImputing missing values...") @@ -223,7 +245,7 @@ def main(): # Train the model print("\nTraining model...") - model, scaler = train_model(X_imputed, y, batch_size=8, epochs=200) + model, scaler = train_model(X_imputed, y, batch_size=8, epochs=100) # Evaluate the model print("\nEvaluating model...") diff --git a/src/benchmark.rs b/src/benchmark.rs index 606f29b..6368fa8 100644 --- a/src/benchmark.rs +++ b/src/benchmark.rs @@ -17,8 +17,8 @@ use std::thread; use std::time::{Duration, Instant}; const SLICE_US: u64 = 50000; -const LOG_INTERVAL_MS: u64 = 10; // Log every 1 second - // const RESHUFFLE_ROUNDS: usize = 5; // Number of rounds before reshuffling counters +const LOG_INTERVAL_MS: u64 = 5; // Log every 1 second + // const RESHUFFLE_ROUNDS: usize = 5; // Number of rounds before reshuffling counters const RESHUFFLE_ROUNDS: usize = 1; // Number of rounds before reshuffling counters const MAX_COUNTERS_AT_ONCE: usize = 5; @@ -379,284 +379,132 @@ fn define_available_events() -> Vec<(String, Event)> { "cpu_cycles".to_string(), Event::Hardware(Hardware::CPU_CYCLES), ), - ( - "instructions".to_string(), - Event::Hardware(Hardware::INSTRUCTIONS), - ), + // ( + // "instructions".to_string(), + // Event::Hardware(Hardware::INSTRUCTIONS), + // ), ( "cache_references".to_string(), Event::Hardware(Hardware::CACHE_REFERENCES), ), - ( - "cache_misses".to_string(), - Event::Hardware(Hardware::CACHE_MISSES), - ), - ( - "branch_instructions".to_string(), - Event::Hardware(Hardware::BRANCH_INSTRUCTIONS), - ), - ( - "branch_misses".to_string(), - Event::Hardware(Hardware::BRANCH_MISSES), - ), // ( - // "bus_cycles".to_string(), - // Event::Hardware(Hardware::BUS_CYCLES), + // "cache_misses".to_string(), + // Event::Hardware(Hardware::CACHE_MISSES), // ), // ( - // "stalled_cycles_frontend".to_string(), - // Event::Hardware(Hardware::STALLED_CYCLES_FRONTEND), - // ), - // ( - // "stalled_cycles_backend".to_string(), - // Event::Hardware(Hardware::STALLED_CYCLES_BACKEND), + // "branch_instructions".to_string(), + // Event::Hardware(Hardware::BRANCH_INSTRUCTIONS), // ), ( - "ref_cpu_cycles".to_string(), - Event::Hardware(Hardware::REF_CPU_CYCLES), - ), - ]); - - // Software events - events.extend([ - ( - "sw_cpu_clock".to_string(), - Event::Software(Software::CPU_CLOCK), - ), - ( - "sw_task_clock".to_string(), - Event::Software(Software::TASK_CLOCK), - ), - ( - "sw_page_faults".to_string(), - Event::Software(Software::PAGE_FAULTS), - ), - ( - "sw_context_switches".to_string(), - Event::Software(Software::CONTEXT_SWITCHES), - ), - ( - "sw_cpu_migrations".to_string(), - Event::Software(Software::CPU_MIGRATIONS), - ), - ( - "sw_page_faults_min".to_string(), - Event::Software(Software::PAGE_FAULTS_MIN), - ), - ( - "sw_page_faults_maj".to_string(), - Event::Software(Software::PAGE_FAULTS_MAJ), - ), - ( - "sw_alignment_faults".to_string(), - Event::Software(Software::ALIGNMENT_FAULTS), + "branch_misses".to_string(), + Event::Hardware(Hardware::BRANCH_MISSES), ), ( - "sw_emulation_faults".to_string(), - Event::Software(Software::EMULATION_FAULTS), + "ref_cpu_cycles".to_string(), + Event::Hardware(Hardware::REF_CPU_CYCLES), ), ]); // L1 Data Cache events events.extend([ - ( - "l1d_read_access".to_string(), - Event::Cache(Cache { - which: WhichCache::L1D, - operation: CacheOp::READ, - result: CacheResult::ACCESS, - }), - ), - ( - "l1d_read_miss".to_string(), - Event::Cache(Cache { - which: WhichCache::L1D, - operation: CacheOp::READ, - result: CacheResult::MISS, - }), - ), - // ( - // "l1d_write_access".to_string(), - // Event::Cache(Cache { - // which: WhichCache::L1D, - // operation: CacheOp::WRITE, - // result: CacheResult::ACCESS, - // }), - // ), - // ( - // "l1d_write_miss".to_string(), - // Event::Cache(Cache { - // which: WhichCache::L1D, - // operation: CacheOp::WRITE, - // result: CacheResult::MISS, - // }), - // ), // ( - // "l1d_prefetch_access".to_string(), + // "l1d_read_access".to_string(), // Event::Cache(Cache { // which: WhichCache::L1D, - // operation: CacheOp::PREFETCH, + // operation: CacheOp::READ, // result: CacheResult::ACCESS, // }), // ), - // ( - // "l1d_prefetch_miss".to_string(), - // Event::Cache(Cache { - // which: WhichCache::L1D, - // operation: CacheOp::PREFETCH, - // result: CacheResult::MISS, - // }), - // ), - ]); - - // L1 Instruction Cache events - events.extend([ ( - "l1i_read_access".to_string(), - Event::Cache(Cache { - which: WhichCache::L1I, - operation: CacheOp::READ, - result: CacheResult::ACCESS, - }), - ), - ( - "l1i_read_miss".to_string(), + "l1d_read_miss".to_string(), Event::Cache(Cache { - which: WhichCache::L1I, + which: WhichCache::L1D, operation: CacheOp::READ, result: CacheResult::MISS, }), ), ]); - // Last Level Cache events + // L1 Instruction Cache events events.extend([ // ( - // "llc_read_access".to_string(), + // "l1i_read_access".to_string(), // Event::Cache(Cache { - // which: WhichCache::LL, + // which: WhichCache::L1I, // operation: CacheOp::READ, // result: CacheResult::ACCESS, // }), // ), // ( - // "llc_read_miss".to_string(), + // "l1i_read_miss".to_string(), // Event::Cache(Cache { - // which: WhichCache::LL, + // which: WhichCache::L1I, // operation: CacheOp::READ, // result: CacheResult::MISS, // }), // ), + ]); + + // Data TLB events + events.extend([ // ( - // "llc_write_access".to_string(), + // "dtlb_read_access".to_string(), // Event::Cache(Cache { - // which: WhichCache::LL, - // operation: CacheOp::WRITE, + // which: WhichCache::DTLB, + // operation: CacheOp::READ, // result: CacheResult::ACCESS, // }), // ), // ( - // "llc_write_miss".to_string(), + // "dtlb_read_miss".to_string(), // Event::Cache(Cache { - // which: WhichCache::LL, - // operation: CacheOp::WRITE, + // which: WhichCache::DTLB, + // operation: CacheOp::READ, // result: CacheResult::MISS, // }), // ), + ]); + + // Instruction TLB events + events.extend([ // ( - // "llc_prefetch_access".to_string(), + // "itlb_read_access".to_string(), // Event::Cache(Cache { - // which: WhichCache::LL, - // operation: CacheOp::PREFETCH, + // which: WhichCache::ITLB, + // operation: CacheOp::READ, // result: CacheResult::ACCESS, // }), // ), // ( - // "llc_prefetch_miss".to_string(), + // "itlb_read_miss".to_string(), // Event::Cache(Cache { - // which: WhichCache::LL, - // operation: CacheOp::PREFETCH, + // which: WhichCache::ITLB, + // operation: CacheOp::READ, // result: CacheResult::MISS, // }), // ), ]); - // Data TLB events + // Branch Prediction Unit events events.extend([ - ( - "dtlb_read_access".to_string(), - Event::Cache(Cache { - which: WhichCache::DTLB, - operation: CacheOp::READ, - result: CacheResult::ACCESS, - }), - ), - ( - "dtlb_read_miss".to_string(), - Event::Cache(Cache { - which: WhichCache::DTLB, - operation: CacheOp::READ, - result: CacheResult::MISS, - }), - ), // ( - // "dtlb_write_access".to_string(), + // "bpu_read_access".to_string(), // Event::Cache(Cache { - // which: WhichCache::DTLB, - // operation: CacheOp::WRITE, + // which: WhichCache::BPU, + // operation: CacheOp::READ, // result: CacheResult::ACCESS, // }), // ), // ( - // "dtlb_write_miss".to_string(), + // "bpu_read_miss".to_string(), // Event::Cache(Cache { - // which: WhichCache::DTLB, - // operation: CacheOp::WRITE, + // which: WhichCache::BPU, + // operation: CacheOp::READ, // result: CacheResult::MISS, // }), // ), ]); - // Instruction TLB events - events.extend([ - ( - "itlb_read_access".to_string(), - Event::Cache(Cache { - which: WhichCache::ITLB, - operation: CacheOp::READ, - result: CacheResult::ACCESS, - }), - ), - ( - "itlb_read_miss".to_string(), - Event::Cache(Cache { - which: WhichCache::ITLB, - operation: CacheOp::READ, - result: CacheResult::MISS, - }), - ), - ]); - - // Branch Prediction Unit events - events.extend([ - ( - "bpu_read_access".to_string(), - Event::Cache(Cache { - which: WhichCache::BPU, - operation: CacheOp::READ, - result: CacheResult::ACCESS, - }), - ), - ( - "bpu_read_miss".to_string(), - Event::Cache(Cache { - which: WhichCache::BPU, - operation: CacheOp::READ, - result: CacheResult::MISS, - }), - ), - ]); - // Sort events by name for consistent ordering events.sort_unstable_by_key(|(name, _)| name.clone()); |