diff options
Diffstat (limited to 'drivers/gpu/drm/amd/pm')
| -rw-r--r-- | drivers/gpu/drm/amd/pm/inc/smu11_driver_if_cyan_skillfish.h | 86 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/pm/inc/smu_types.h | 5 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/pm/inc/smu_v11_8_ppsmc.h | 9 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/pm/powerplay/si_dpm.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/pm/swsmu/smu11/cyan_skillfish_ppt.c | 481 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c | 28 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c | 16 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c | 12 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c | 21 | ||||
| -rw-r--r-- | drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h | 15 | 
15 files changed, 627 insertions, 74 deletions
diff --git a/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_cyan_skillfish.h b/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_cyan_skillfish.h index 8a08ecc34c69..4884a4e1f261 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_cyan_skillfish.h +++ b/drivers/gpu/drm/amd/pm/inc/smu11_driver_if_cyan_skillfish.h @@ -33,63 +33,47 @@  #define TABLE_PMSTATUSLOG        3 // Called by Tools for Agm logging  #define TABLE_DPMCLOCKS          4 // Called by Driver; defined here, but not used, for backward compatible  #define TABLE_MOMENTARY_PM       5 // Called by Tools; defined here, but not used, for backward compatible -#define TABLE_COUNT              6 +#define TABLE_SMU_METRICS        6 // Called by Driver +#define TABLE_COUNT              7 -#define NUM_DSPCLK_LEVELS		8 -#define NUM_SOCCLK_DPM_LEVELS	8 -#define NUM_DCEFCLK_DPM_LEVELS	4 -#define NUM_FCLK_DPM_LEVELS		4 -#define NUM_MEMCLK_DPM_LEVELS	4 +typedef struct SmuMetricsTable_t { +	//CPU status +	uint16_t CoreFrequency[6];              //[MHz] +	uint32_t CorePower[6];                  //[mW] +	uint16_t CoreTemperature[6];            //[centi-Celsius] +	uint16_t L3Frequency[2];                //[MHz] +	uint16_t L3Temperature[2];              //[centi-Celsius] +	uint16_t C0Residency[6];                //Percentage -#define NUMBER_OF_PSTATES		8 -#define NUMBER_OF_CORES			8 +	// GFX status +	uint16_t GfxclkFrequency;               //[MHz] +	uint16_t GfxTemperature;                //[centi-Celsius] -typedef enum { -	S3_TYPE_ENTRY, -	S5_TYPE_ENTRY, -} Sleep_Type_e; +	// SOC IP info +	uint16_t SocclkFrequency;               //[MHz] +	uint16_t VclkFrequency;                 //[MHz] +	uint16_t DclkFrequency;                 //[MHz] +	uint16_t MemclkFrequency;               //[MHz] -typedef enum { -	GFX_OFF = 0, -	GFX_ON  = 1, -} GFX_Mode_e; +	// power, VF info for CPU/GFX telemetry rails, and then socket power total +	uint32_t Voltage[2];                    //[mV] indices: VDDCR_VDD, VDDCR_GFX +	uint32_t Current[2];                    //[mA] indices: VDDCR_VDD, VDDCR_GFX +	uint32_t Power[2];                      //[mW] indices: VDDCR_VDD, VDDCR_GFX +	uint32_t CurrentSocketPower;            //[mW] -typedef enum { -	CPU_P0 = 0, -	CPU_P1, -	CPU_P2, -	CPU_P3, -	CPU_P4, -	CPU_P5, -	CPU_P6, -	CPU_P7 -} CPU_PState_e; +	uint16_t SocTemperature;                //[centi-Celsius] +	uint16_t EdgeTemperature; +	uint16_t ThrottlerStatus; +	uint16_t Spare; -typedef enum { -	CPU_CORE0 = 0, -	CPU_CORE1, -	CPU_CORE2, -	CPU_CORE3, -	CPU_CORE4, -	CPU_CORE5, -	CPU_CORE6, -	CPU_CORE7 -} CORE_ID_e; +} SmuMetricsTable_t; -typedef enum { -	DF_DPM0 = 0, -	DF_DPM1, -	DF_DPM2, -	DF_DPM3, -	DF_PState_Count -} DF_PState_e; - -typedef enum { -	GFX_DPM0 = 0, -	GFX_DPM1, -	GFX_DPM2, -	GFX_DPM3, -	GFX_PState_Count -} GFX_PState_e; +typedef struct SmuMetrics_t { +	SmuMetricsTable_t Current; +	SmuMetricsTable_t Average; +	uint32_t SampleStartTime; +	uint32_t SampleStopTime; +	uint32_t Accnt; +} SmuMetrics_t;  #endif diff --git a/drivers/gpu/drm/amd/pm/inc/smu_types.h b/drivers/gpu/drm/amd/pm/inc/smu_types.h index 6f1b1b50d527..18b862a90fbe 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_types.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_types.h @@ -226,7 +226,10 @@  	__SMU_DUMMY_MAP(SetUclkDpmMode),		\  	__SMU_DUMMY_MAP(LightSBR),			\  	__SMU_DUMMY_MAP(GfxDriverResetRecovery),	\ -	__SMU_DUMMY_MAP(BoardPowerCalibration), +	__SMU_DUMMY_MAP(BoardPowerCalibration),   \ +	__SMU_DUMMY_MAP(RequestGfxclk),           \ +	__SMU_DUMMY_MAP(ForceGfxVid),             \ +	__SMU_DUMMY_MAP(UnforceGfxVid),  #undef __SMU_DUMMY_MAP  #define __SMU_DUMMY_MAP(type)	SMU_MSG_##type diff --git a/drivers/gpu/drm/amd/pm/inc/smu_v11_8_ppsmc.h b/drivers/gpu/drm/amd/pm/inc/smu_v11_8_ppsmc.h index 6e6088760b18..909a86aa60f3 100644 --- a/drivers/gpu/drm/amd/pm/inc/smu_v11_8_ppsmc.h +++ b/drivers/gpu/drm/amd/pm/inc/smu_v11_8_ppsmc.h @@ -65,6 +65,13 @@  #define PPSMC_MSG_SetDriverTableVMID                    0x34  #define PPSMC_MSG_SetSoftMinCclk                        0x35  #define PPSMC_MSG_SetSoftMaxCclk                        0x36 -#define PPSMC_Message_Count                             0x37 +#define PPSMC_MSG_GetGfxFrequency                       0x37 +#define PPSMC_MSG_GetGfxVid                             0x38 +#define PPSMC_MSG_ForceGfxFreq                          0x39 +#define PPSMC_MSG_UnForceGfxFreq                        0x3A +#define PPSMC_MSG_ForceGfxVid                           0x3B +#define PPSMC_MSG_UnforceGfxVid                         0x3C +#define PPSMC_MSG_GetEnabledSmuFeatures                 0x3D +#define PPSMC_Message_Count                             0x3E  #endif diff --git a/drivers/gpu/drm/amd/pm/powerplay/si_dpm.c b/drivers/gpu/drm/amd/pm/powerplay/si_dpm.c index bdbbeb959c68..81f82aa05ec2 100644 --- a/drivers/gpu/drm/amd/pm/powerplay/si_dpm.c +++ b/drivers/gpu/drm/amd/pm/powerplay/si_dpm.c @@ -6867,6 +6867,8 @@ static int si_dpm_enable(struct amdgpu_device *adev)  	si_enable_auto_throttle_source(adev, AMDGPU_DPM_AUTO_THROTTLE_SRC_THERMAL, true);  	si_thermal_start_thermal_controller(adev); +	ni_update_current_ps(adev, boot_ps); +  	return 0;  } diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index 3ab1ce4d3419..04863a797115 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -1404,7 +1404,7 @@ static int smu_disable_dpms(struct smu_context *smu)  	 */  	if (smu->uploading_custom_pp_table &&  	    (adev->asic_type >= CHIP_NAVI10) && -	    (adev->asic_type <= CHIP_DIMGREY_CAVEFISH)) +	    (adev->asic_type <= CHIP_BEIGE_GOBY))  		return smu_disable_all_features_with_exception(smu,  							       true,  							       SMU_FEATURE_COUNT); diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c index e343cc218990..082f01893f3d 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/arcturus_ppt.c @@ -771,8 +771,12 @@ static int arcturus_print_clk_levels(struct smu_context *smu,  	struct smu_11_0_dpm_context *dpm_context = NULL;  	uint32_t gen_speed, lane_width; -	if (amdgpu_ras_intr_triggered()) -		return sysfs_emit(buf, "unavailable\n"); +	smu_cmn_get_sysfs_buf(&buf, &size); + +	if (amdgpu_ras_intr_triggered()) { +		size += sysfs_emit_at(buf, size, "unavailable\n"); +		return size; +	}  	dpm_context = smu_dpm->dpm_context; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/cyan_skillfish_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/cyan_skillfish_ppt.c index b05f9541accc..3d4c65bc29dc 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/cyan_skillfish_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/cyan_skillfish_ppt.c @@ -44,6 +44,27 @@  #undef pr_info  #undef pr_debug +/* unit: MHz */ +#define CYAN_SKILLFISH_SCLK_MIN			1000 +#define CYAN_SKILLFISH_SCLK_MAX			2000 +#define CYAN_SKILLFISH_SCLK_DEFAULT			1800 + +/* unit: mV */ +#define CYAN_SKILLFISH_VDDC_MIN			700 +#define CYAN_SKILLFISH_VDDC_MAX			1129 +#define CYAN_SKILLFISH_VDDC_MAGIC			5118 // 0x13fe + +static struct gfx_user_settings { +	uint32_t sclk; +	uint32_t vddc; +} cyan_skillfish_user_settings; + +#define FEATURE_MASK(feature) (1ULL << feature) +#define SMC_DPM_FEATURE ( \ +	FEATURE_MASK(FEATURE_FCLK_DPM_BIT)	|	\ +	FEATURE_MASK(FEATURE_SOC_DPM_BIT)	|	\ +	FEATURE_MASK(FEATURE_GFX_DPM_BIT)) +  static struct cmn2asic_msg_mapping cyan_skillfish_message_map[SMU_MSG_MAX_COUNT] = {  	MSG_MAP(TestMessage,                    PPSMC_MSG_TestMessage,			0),  	MSG_MAP(GetSmuVersion,                  PPSMC_MSG_GetSmuVersion,		0), @@ -52,14 +73,473 @@ static struct cmn2asic_msg_mapping cyan_skillfish_message_map[SMU_MSG_MAX_COUNT]  	MSG_MAP(SetDriverDramAddrLow,           PPSMC_MSG_SetDriverTableDramAddrLow,	0),  	MSG_MAP(TransferTableSmu2Dram,          PPSMC_MSG_TransferTableSmu2Dram,	0),  	MSG_MAP(TransferTableDram2Smu,          PPSMC_MSG_TransferTableDram2Smu,	0), +	MSG_MAP(GetEnabledSmuFeatures,          PPSMC_MSG_GetEnabledSmuFeatures,	0), +	MSG_MAP(RequestGfxclk,                  PPSMC_MSG_RequestGfxclk,		0), +	MSG_MAP(ForceGfxVid,                    PPSMC_MSG_ForceGfxVid,			0), +	MSG_MAP(UnforceGfxVid,                  PPSMC_MSG_UnforceGfxVid,		0), +}; + +static struct cmn2asic_mapping cyan_skillfish_table_map[SMU_TABLE_COUNT] = { +	TAB_MAP_VALID(SMU_METRICS),  }; +static int cyan_skillfish_tables_init(struct smu_context *smu) +{ +	struct smu_table_context *smu_table = &smu->smu_table; +	struct smu_table *tables = smu_table->tables; + +	SMU_TABLE_INIT(tables, SMU_TABLE_SMU_METRICS, +				sizeof(SmuMetrics_t), +				PAGE_SIZE, +				AMDGPU_GEM_DOMAIN_VRAM); + +	smu_table->metrics_table = kzalloc(sizeof(SmuMetrics_t), GFP_KERNEL); +	if (!smu_table->metrics_table) +		goto err0_out; + +	smu_table->gpu_metrics_table_size = sizeof(struct gpu_metrics_v2_2); +	smu_table->gpu_metrics_table = kzalloc(smu_table->gpu_metrics_table_size, GFP_KERNEL); +	if (!smu_table->gpu_metrics_table) +		goto err1_out; + +	smu_table->metrics_time = 0; + +	return 0; + +err1_out: +	smu_table->gpu_metrics_table_size = 0; +	kfree(smu_table->metrics_table); +err0_out: +	return -ENOMEM; +} + +static int cyan_skillfish_init_smc_tables(struct smu_context *smu) +{ +	int ret = 0; + +	ret = cyan_skillfish_tables_init(smu); +	if (ret) +		return ret; + +	return smu_v11_0_init_smc_tables(smu); +} + +static int cyan_skillfish_finit_smc_tables(struct smu_context *smu) +{ +	struct smu_table_context *smu_table = &smu->smu_table; + +	kfree(smu_table->metrics_table); +	smu_table->metrics_table = NULL; + +	kfree(smu_table->gpu_metrics_table); +	smu_table->gpu_metrics_table = NULL; +	smu_table->gpu_metrics_table_size = 0; + +	smu_table->metrics_time = 0; + +	return 0; +} + +static int +cyan_skillfish_get_smu_metrics_data(struct smu_context *smu, +					MetricsMember_t member, +					uint32_t *value) +{ +	struct smu_table_context *smu_table = &smu->smu_table; +	SmuMetrics_t *metrics = (SmuMetrics_t *)smu_table->metrics_table; +	int ret = 0; + +	mutex_lock(&smu->metrics_lock); + +	ret = smu_cmn_get_metrics_table_locked(smu, NULL, false); +	if (ret) { +		mutex_unlock(&smu->metrics_lock); +		return ret; +	} + +	switch (member) { +	case METRICS_CURR_GFXCLK: +		*value = metrics->Current.GfxclkFrequency; +		break; +	case METRICS_CURR_SOCCLK: +		*value = metrics->Current.SocclkFrequency; +		break; +	case METRICS_CURR_VCLK: +		*value = metrics->Current.VclkFrequency; +		break; +	case METRICS_CURR_DCLK: +		*value = metrics->Current.DclkFrequency; +		break; +	case METRICS_CURR_UCLK: +		*value = metrics->Current.MemclkFrequency; +		break; +	case METRICS_AVERAGE_SOCKETPOWER: +		*value = (metrics->Current.CurrentSocketPower << 8) / +				1000; +		break; +	case METRICS_TEMPERATURE_EDGE: +		*value = metrics->Current.GfxTemperature / 100 * +				SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; +		break; +	case METRICS_TEMPERATURE_HOTSPOT: +		*value = metrics->Current.SocTemperature / 100 * +				SMU_TEMPERATURE_UNITS_PER_CENTIGRADES; +		break; +	case METRICS_VOLTAGE_VDDSOC: +		*value = metrics->Current.Voltage[0]; +		break; +	case METRICS_VOLTAGE_VDDGFX: +		*value = metrics->Current.Voltage[1]; +		break; +	case METRICS_THROTTLER_STATUS: +		*value = metrics->Current.ThrottlerStatus; +		break; +	default: +		*value = UINT_MAX; +		break; +	} + +	mutex_unlock(&smu->metrics_lock); + +	return ret; +} + +static int cyan_skillfish_read_sensor(struct smu_context *smu, +					enum amd_pp_sensors sensor, +					void *data, +					uint32_t *size) +{ +	int ret = 0; + +	if (!data || !size) +		return -EINVAL; + +	mutex_lock(&smu->sensor_lock); + +	switch (sensor) { +	case AMDGPU_PP_SENSOR_GFX_SCLK: +		ret = cyan_skillfish_get_smu_metrics_data(smu, +						   METRICS_CURR_GFXCLK, +						   (uint32_t *)data); +		*(uint32_t *)data *= 100; +		*size = 4; +		break; +	case AMDGPU_PP_SENSOR_GFX_MCLK: +		ret = cyan_skillfish_get_smu_metrics_data(smu, +						   METRICS_CURR_UCLK, +						   (uint32_t *)data); +		*(uint32_t *)data *= 100; +		*size = 4; +		break; +	case AMDGPU_PP_SENSOR_GPU_POWER: +		ret = cyan_skillfish_get_smu_metrics_data(smu, +						   METRICS_AVERAGE_SOCKETPOWER, +						   (uint32_t *)data); +		*size = 4; +		break; +	case AMDGPU_PP_SENSOR_HOTSPOT_TEMP: +		ret = cyan_skillfish_get_smu_metrics_data(smu, +						   METRICS_TEMPERATURE_HOTSPOT, +						   (uint32_t *)data); +		*size = 4; +		break; +	case AMDGPU_PP_SENSOR_EDGE_TEMP: +		ret = cyan_skillfish_get_smu_metrics_data(smu, +						   METRICS_TEMPERATURE_EDGE, +						   (uint32_t *)data); +		*size = 4; +		break; +	case AMDGPU_PP_SENSOR_VDDNB: +		ret = cyan_skillfish_get_smu_metrics_data(smu, +						   METRICS_VOLTAGE_VDDSOC, +						   (uint32_t *)data); +		*size = 4; +		break; +	case AMDGPU_PP_SENSOR_VDDGFX: +		ret = cyan_skillfish_get_smu_metrics_data(smu, +						   METRICS_VOLTAGE_VDDGFX, +						   (uint32_t *)data); +		*size = 4; +		break; +	default: +		ret = -EOPNOTSUPP; +		break; +	} + +	mutex_unlock(&smu->sensor_lock); + +	return ret; +} + +static int cyan_skillfish_get_current_clk_freq(struct smu_context *smu, +						enum smu_clk_type clk_type, +						uint32_t *value) +{ +	MetricsMember_t member_type; + +	switch (clk_type) { +	case SMU_GFXCLK: +	case SMU_SCLK: +		member_type = METRICS_CURR_GFXCLK; +		break; +	case SMU_FCLK: +	case SMU_MCLK: +		member_type = METRICS_CURR_UCLK; +		break; +	case SMU_SOCCLK: +		member_type = METRICS_CURR_SOCCLK; +		break; +	case SMU_VCLK: +		member_type = METRICS_CURR_VCLK; +		break; +	case SMU_DCLK: +		member_type = METRICS_CURR_DCLK; +		break; +	default: +		return -EINVAL; +	} + +	return cyan_skillfish_get_smu_metrics_data(smu, member_type, value); +} + +static int cyan_skillfish_print_clk_levels(struct smu_context *smu, +					enum smu_clk_type clk_type, +					char *buf) +{ +	int ret = 0, size = 0; +	uint32_t cur_value = 0; + +	smu_cmn_get_sysfs_buf(&buf, &size); + +	switch (clk_type) { +	case SMU_OD_SCLK: +		ret  = cyan_skillfish_get_smu_metrics_data(smu, METRICS_CURR_GFXCLK, &cur_value); +		if (ret) +			return ret; +		size += sysfs_emit_at(buf, size,"%s:\n", "OD_SCLK"); +		size += sysfs_emit_at(buf, size, "0: %uMhz *\n", cur_value); +		break; +	case SMU_OD_VDDC_CURVE: +		ret  = cyan_skillfish_get_smu_metrics_data(smu, METRICS_VOLTAGE_VDDGFX, &cur_value); +		if (ret) +			return ret; +		size += sysfs_emit_at(buf, size,"%s:\n", "OD_VDDC"); +		size += sysfs_emit_at(buf, size, "0: %umV *\n", cur_value); +		break; +	case SMU_OD_RANGE: +		size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE"); +		size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n", +						CYAN_SKILLFISH_SCLK_MIN, CYAN_SKILLFISH_SCLK_MAX); +		size += sysfs_emit_at(buf, size, "VDDC: %7umV  %10umV\n", +						CYAN_SKILLFISH_VDDC_MIN, CYAN_SKILLFISH_VDDC_MAX); +		break; +	case SMU_GFXCLK: +	case SMU_SCLK: +	case SMU_FCLK: +	case SMU_MCLK: +	case SMU_SOCCLK: +	case SMU_VCLK: +	case SMU_DCLK: +		ret = cyan_skillfish_get_current_clk_freq(smu, clk_type, &cur_value); +		if (ret) +			return ret; +		size += sysfs_emit_at(buf, size, "0: %uMhz *\n", cur_value); +		break; +	default: +		dev_warn(smu->adev->dev, "Unsupported clock type\n"); +		return ret; +	} + +	return size; +} + +static bool cyan_skillfish_is_dpm_running(struct smu_context *smu) +{ +	struct amdgpu_device *adev = smu->adev; +	int ret = 0; +	uint32_t feature_mask[2]; +	uint64_t feature_enabled; + +	/* we need to re-init after suspend so return false */ +	if (adev->in_suspend) +		return false; + +	ret = smu_cmn_get_enabled_32_bits_mask(smu, feature_mask, 2); + +	if (ret) +		return false; + +	feature_enabled = (uint64_t)feature_mask[0] | +				((uint64_t)feature_mask[1] << 32); + +	return !!(feature_enabled & SMC_DPM_FEATURE); +} + +static ssize_t cyan_skillfish_get_gpu_metrics(struct smu_context *smu, +						void **table) +{ +	struct smu_table_context *smu_table = &smu->smu_table; +	struct gpu_metrics_v2_2 *gpu_metrics = +		(struct gpu_metrics_v2_2 *)smu_table->gpu_metrics_table; +	SmuMetrics_t metrics; +	int i, ret = 0; + +	ret = smu_cmn_get_metrics_table(smu, &metrics, true); +	if (ret) +		return ret; + +	smu_cmn_init_soft_gpu_metrics(gpu_metrics, 2, 2); + +	gpu_metrics->temperature_gfx = metrics.Current.GfxTemperature; +	gpu_metrics->temperature_soc = metrics.Current.SocTemperature; + +	gpu_metrics->average_socket_power = metrics.Current.CurrentSocketPower; +	gpu_metrics->average_soc_power = metrics.Current.Power[0]; +	gpu_metrics->average_gfx_power = metrics.Current.Power[1]; + +	gpu_metrics->average_gfxclk_frequency = metrics.Average.GfxclkFrequency; +	gpu_metrics->average_socclk_frequency = metrics.Average.SocclkFrequency; +	gpu_metrics->average_uclk_frequency = metrics.Average.MemclkFrequency; +	gpu_metrics->average_fclk_frequency = metrics.Average.MemclkFrequency; +	gpu_metrics->average_vclk_frequency = metrics.Average.VclkFrequency; +	gpu_metrics->average_dclk_frequency = metrics.Average.DclkFrequency; + +	gpu_metrics->current_gfxclk = metrics.Current.GfxclkFrequency; +	gpu_metrics->current_socclk = metrics.Current.SocclkFrequency; +	gpu_metrics->current_uclk = metrics.Current.MemclkFrequency; +	gpu_metrics->current_fclk = metrics.Current.MemclkFrequency; +	gpu_metrics->current_vclk = metrics.Current.VclkFrequency; +	gpu_metrics->current_dclk = metrics.Current.DclkFrequency; + +	for (i = 0; i < 6; i++) { +		gpu_metrics->temperature_core[i] = metrics.Current.CoreTemperature[i]; +		gpu_metrics->average_core_power[i] = metrics.Average.CorePower[i]; +		gpu_metrics->current_coreclk[i] = metrics.Current.CoreFrequency[i]; +	} + +	for (i = 0; i < 2; i++) { +		gpu_metrics->temperature_l3[i] = metrics.Current.L3Temperature[i]; +		gpu_metrics->current_l3clk[i] = metrics.Current.L3Frequency[i]; +	} + +	gpu_metrics->throttle_status = metrics.Current.ThrottlerStatus; +	gpu_metrics->system_clock_counter = ktime_get_boottime_ns(); + +	*table = (void *)gpu_metrics; + +	return sizeof(struct gpu_metrics_v2_2); +} + +static int cyan_skillfish_od_edit_dpm_table(struct smu_context *smu, +					enum PP_OD_DPM_TABLE_COMMAND type, +					long input[], uint32_t size) +{ +	int ret = 0; +	uint32_t vid; + +	switch (type) { +	case PP_OD_EDIT_VDDC_CURVE: +		if (size != 3 || input[0] != 0) { +			dev_err(smu->adev->dev, "Invalid parameter!\n"); +			return -EINVAL; +		} + +		if (input[1] <= CYAN_SKILLFISH_SCLK_MIN || +			input[1] > CYAN_SKILLFISH_SCLK_MAX) { +			dev_err(smu->adev->dev, "Invalid sclk! Valid sclk range: %uMHz - %uMhz\n", +					CYAN_SKILLFISH_SCLK_MIN, CYAN_SKILLFISH_SCLK_MAX); +			return -EINVAL; +		} + +		if (input[2] <= CYAN_SKILLFISH_VDDC_MIN || +			input[2] > CYAN_SKILLFISH_VDDC_MAX) { +			dev_err(smu->adev->dev, "Invalid vddc! Valid vddc range: %umV - %umV\n", +					CYAN_SKILLFISH_VDDC_MIN, CYAN_SKILLFISH_VDDC_MAX); +			return -EINVAL; +		} + +		cyan_skillfish_user_settings.sclk = input[1]; +		cyan_skillfish_user_settings.vddc = input[2]; + +		break; +	case PP_OD_RESTORE_DEFAULT_TABLE: +		if (size != 0) { +			dev_err(smu->adev->dev, "Invalid parameter!\n"); +			return -EINVAL; +		} + +		cyan_skillfish_user_settings.sclk = CYAN_SKILLFISH_SCLK_DEFAULT; +		cyan_skillfish_user_settings.vddc = CYAN_SKILLFISH_VDDC_MAGIC; + +		break; +	case PP_OD_COMMIT_DPM_TABLE: +		if (size != 0) { +			dev_err(smu->adev->dev, "Invalid parameter!\n"); +			return -EINVAL; +		} + +		if (cyan_skillfish_user_settings.sclk < CYAN_SKILLFISH_SCLK_MIN || +		    cyan_skillfish_user_settings.sclk > CYAN_SKILLFISH_SCLK_MAX) { +			dev_err(smu->adev->dev, "Invalid sclk! Valid sclk range: %uMHz - %uMhz\n", +					CYAN_SKILLFISH_SCLK_MIN, CYAN_SKILLFISH_SCLK_MAX); +			return -EINVAL; +		} + +		if ((cyan_skillfish_user_settings.vddc != CYAN_SKILLFISH_VDDC_MAGIC) && +			(cyan_skillfish_user_settings.vddc < CYAN_SKILLFISH_VDDC_MIN || +			cyan_skillfish_user_settings.vddc > CYAN_SKILLFISH_VDDC_MAX)) { +			dev_err(smu->adev->dev, "Invalid vddc! Valid vddc range: %umV - %umV\n", +					CYAN_SKILLFISH_VDDC_MIN, CYAN_SKILLFISH_VDDC_MAX); +			return -EINVAL; +		} + +		ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_RequestGfxclk, +					cyan_skillfish_user_settings.sclk, NULL); +		if (ret) { +			dev_err(smu->adev->dev, "Set sclk failed!\n"); +			return ret; +		} + +		if (cyan_skillfish_user_settings.vddc == CYAN_SKILLFISH_VDDC_MAGIC) { +			ret = smu_cmn_send_smc_msg(smu, SMU_MSG_UnforceGfxVid, NULL); +			if (ret) { +				dev_err(smu->adev->dev, "Unforce vddc failed!\n"); +				return ret; +			} +		} else { +			/* +			 * PMFW accepts SVI2 VID code, convert voltage to VID: +			 * vid = (uint32_t)((1.55 - voltage) * 160.0 + 0.00001) +			 */ +			vid = (1550 - cyan_skillfish_user_settings.vddc) * 160 / 1000; +			ret = smu_cmn_send_smc_msg_with_param(smu, SMU_MSG_ForceGfxVid, vid, NULL); +			if (ret) { +				dev_err(smu->adev->dev, "Force vddc failed!\n"); +				return ret; +			} +		} + +		break; +	default: +		return -EOPNOTSUPP; +	} + +	return ret; +} +  static const struct pptable_funcs cyan_skillfish_ppt_funcs = {  	.check_fw_status = smu_v11_0_check_fw_status,  	.check_fw_version = smu_v11_0_check_fw_version,  	.init_power = smu_v11_0_init_power,  	.fini_power = smu_v11_0_fini_power, +	.init_smc_tables = cyan_skillfish_init_smc_tables, +	.fini_smc_tables = cyan_skillfish_finit_smc_tables, +	.read_sensor = cyan_skillfish_read_sensor, +	.print_clk_levels = cyan_skillfish_print_clk_levels, +	.is_dpm_running = cyan_skillfish_is_dpm_running, +	.get_gpu_metrics = cyan_skillfish_get_gpu_metrics, +	.od_edit_dpm_table = cyan_skillfish_od_edit_dpm_table,  	.register_irq_handler = smu_v11_0_register_irq_handler,  	.notify_memory_pool_location = smu_v11_0_notify_memory_pool_location,  	.send_smc_msg_with_param = smu_cmn_send_smc_msg_with_param, @@ -72,5 +552,6 @@ void cyan_skillfish_set_ppt_funcs(struct smu_context *smu)  {  	smu->ppt_funcs = &cyan_skillfish_ppt_funcs;  	smu->message_map = cyan_skillfish_message_map; +	smu->table_map = cyan_skillfish_table_map;  	smu->is_apu = true;  } diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c index a5fc5d7cb6c7..b1ad451af06b 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/navi10_ppt.c @@ -1279,6 +1279,8 @@ static int navi10_print_clk_levels(struct smu_context *smu,  	struct smu_11_0_overdrive_table *od_settings = smu->od_settings;  	uint32_t min_value, max_value; +	smu_cmn_get_sysfs_buf(&buf, &size); +  	switch (clk_type) {  	case SMU_GFXCLK:  	case SMU_SCLK: @@ -1392,7 +1394,7 @@ static int navi10_print_clk_levels(struct smu_context *smu,  	case SMU_OD_RANGE:  		if (!smu->od_enabled || !od_table || !od_settings)  			break; -		size = sysfs_emit(buf, "%s:\n", "OD_RANGE"); +		size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE");  		if (navi10_od_feature_is_supported(od_settings, SMU_11_0_ODCAP_GFXCLK_LIMITS)) {  			navi10_od_setting_get_range(od_settings, SMU_11_0_ODSETTING_GFXCLKFMIN, @@ -2272,7 +2274,27 @@ static int navi10_baco_enter(struct smu_context *smu)  {  	struct amdgpu_device *adev = smu->adev; -	if (adev->in_runpm) +	/* +	 * This aims the case below: +	 *   amdgpu driver loaded -> runpm suspend kicked -> sound driver loaded +	 * +	 * For NAVI10 and later ASICs, we rely on PMFW to handle the runpm. To +	 * make that possible, PMFW needs to acknowledge the dstate transition +	 * process for both gfx(function 0) and audio(function 1) function of +	 * the ASIC. +	 * +	 * The PCI device's initial runpm status is RUNPM_SUSPENDED. So as the +	 * device representing the audio function of the ASIC. And that means +	 * even if the sound driver(snd_hda_intel) was not loaded yet, it's still +	 * possible runpm suspend kicked on the ASIC. However without the dstate +	 * transition notification from audio function, pmfw cannot handle the +	 * BACO in/exit correctly. And that will cause driver hang on runpm +	 * resuming. +	 * +	 * To address this, we revert to legacy message way(driver masters the +	 * timing for BACO in/exit) on sound driver missing. +	 */ +	if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev))  		return smu_v11_0_baco_set_armd3_sequence(smu, BACO_SEQ_BACO);  	else  		return smu_v11_0_baco_enter(smu); @@ -2282,7 +2304,7 @@ static int navi10_baco_exit(struct smu_context *smu)  {  	struct amdgpu_device *adev = smu->adev; -	if (adev->in_runpm) { +	if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) {  		/* Wait for PMFW handling for the Dstate change */  		msleep(10);  		return smu_v11_0_baco_set_armd3_sequence(smu, BACO_SEQ_ULPS); diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c index 5e292c3f5050..ca57221e3962 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/sienna_cichlid_ppt.c @@ -1058,6 +1058,8 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu,  	uint32_t min_value, max_value;  	uint32_t smu_version; +	smu_cmn_get_sysfs_buf(&buf, &size); +  	switch (clk_type) {  	case SMU_GFXCLK:  	case SMU_SCLK: @@ -1180,7 +1182,7 @@ static int sienna_cichlid_print_clk_levels(struct smu_context *smu,  		if (!smu->od_enabled || !od_table || !od_settings)  			break; -		size = sysfs_emit(buf, "%s:\n", "OD_RANGE"); +		size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE");  		if (sienna_cichlid_is_od_feature_supported(od_settings, SMU_11_0_7_ODCAP_GFXCLK_LIMITS)) {  			sienna_cichlid_get_od_setting_range(od_settings, SMU_11_0_7_ODSETTING_GFXCLKFMIN, @@ -2187,7 +2189,7 @@ static int sienna_cichlid_baco_enter(struct smu_context *smu)  {  	struct amdgpu_device *adev = smu->adev; -	if (adev->in_runpm) +	if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev))  		return smu_v11_0_baco_set_armd3_sequence(smu, BACO_SEQ_BACO);  	else  		return smu_v11_0_baco_enter(smu); @@ -2197,7 +2199,7 @@ static int sienna_cichlid_baco_exit(struct smu_context *smu)  {  	struct amdgpu_device *adev = smu->adev; -	if (adev->in_runpm) { +	if (adev->in_runpm && smu_cmn_is_audio_func_enabled(adev)) {  		/* Wait for PMFW handling for the Dstate change */  		msleep(10);  		return smu_v11_0_baco_set_armd3_sequence(smu, BACO_SEQ_ULPS); diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c index 3a3421452e57..f6ef0ce6e9e2 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu11/vangogh_ppt.c @@ -589,10 +589,12 @@ static int vangogh_print_legacy_clk_levels(struct smu_context *smu,  	if (ret)  		return ret; +	smu_cmn_get_sysfs_buf(&buf, &size); +  	switch (clk_type) {  	case SMU_OD_SCLK:  		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { -			size = sysfs_emit(buf, "%s:\n", "OD_SCLK"); +			size += sysfs_emit_at(buf, size, "%s:\n", "OD_SCLK");  			size += sysfs_emit_at(buf, size, "0: %10uMhz\n",  			(smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq);  			size += sysfs_emit_at(buf, size, "1: %10uMhz\n", @@ -601,7 +603,7 @@ static int vangogh_print_legacy_clk_levels(struct smu_context *smu,  		break;  	case SMU_OD_CCLK:  		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { -			size = sysfs_emit(buf, "CCLK_RANGE in Core%d:\n",  smu->cpu_core_id_select); +			size += sysfs_emit_at(buf, size, "CCLK_RANGE in Core%d:\n",  smu->cpu_core_id_select);  			size += sysfs_emit_at(buf, size, "0: %10uMhz\n",  			(smu->cpu_actual_soft_min_freq > 0) ? smu->cpu_actual_soft_min_freq : smu->cpu_default_soft_min_freq);  			size += sysfs_emit_at(buf, size, "1: %10uMhz\n", @@ -610,7 +612,7 @@ static int vangogh_print_legacy_clk_levels(struct smu_context *smu,  		break;  	case SMU_OD_RANGE:  		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { -			size = sysfs_emit(buf, "%s:\n", "OD_RANGE"); +			size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE");  			size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n",  				smu->gfx_default_hard_min_freq, smu->gfx_default_soft_max_freq);  			size += sysfs_emit_at(buf, size, "CCLK: %7uMhz %10uMhz\n", @@ -688,10 +690,12 @@ static int vangogh_print_clk_levels(struct smu_context *smu,  	if (ret)  		return ret; +	smu_cmn_get_sysfs_buf(&buf, &size); +  	switch (clk_type) {  	case SMU_OD_SCLK:  		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { -			size = sysfs_emit(buf, "%s:\n", "OD_SCLK"); +			size += sysfs_emit_at(buf, size, "%s:\n", "OD_SCLK");  			size += sysfs_emit_at(buf, size, "0: %10uMhz\n",  			(smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq);  			size += sysfs_emit_at(buf, size, "1: %10uMhz\n", @@ -700,7 +704,7 @@ static int vangogh_print_clk_levels(struct smu_context *smu,  		break;  	case SMU_OD_CCLK:  		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { -			size = sysfs_emit(buf, "CCLK_RANGE in Core%d:\n",  smu->cpu_core_id_select); +			size += sysfs_emit_at(buf, size, "CCLK_RANGE in Core%d:\n",  smu->cpu_core_id_select);  			size += sysfs_emit_at(buf, size, "0: %10uMhz\n",  			(smu->cpu_actual_soft_min_freq > 0) ? smu->cpu_actual_soft_min_freq : smu->cpu_default_soft_min_freq);  			size += sysfs_emit_at(buf, size, "1: %10uMhz\n", @@ -709,7 +713,7 @@ static int vangogh_print_clk_levels(struct smu_context *smu,  		break;  	case SMU_OD_RANGE:  		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { -			size = sysfs_emit(buf, "%s:\n", "OD_RANGE"); +			size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE");  			size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n",  				smu->gfx_default_hard_min_freq, smu->gfx_default_soft_max_freq);  			size += sysfs_emit_at(buf, size, "CCLK: %7uMhz %10uMhz\n", diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c index 5aa175e12a78..145f13b8c977 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu12/renoir_ppt.c @@ -497,6 +497,8 @@ static int renoir_print_clk_levels(struct smu_context *smu,  	if (ret)  		return ret; +	smu_cmn_get_sysfs_buf(&buf, &size); +  	switch (clk_type) {  	case SMU_OD_RANGE:  		if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c index ab652028e003..5019903db492 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/aldebaran_ppt.c @@ -733,15 +733,19 @@ static int aldebaran_print_clk_levels(struct smu_context *smu,  	uint32_t freq_values[3] = {0};  	uint32_t min_clk, max_clk; -	if (amdgpu_ras_intr_triggered()) -		return sysfs_emit(buf, "unavailable\n"); +	smu_cmn_get_sysfs_buf(&buf, &size); + +	if (amdgpu_ras_intr_triggered()) { +		size += sysfs_emit_at(buf, size, "unavailable\n"); +		return size; +	}  	dpm_context = smu_dpm->dpm_context;  	switch (type) {  	case SMU_OD_SCLK: -		size = sysfs_emit(buf, "%s:\n", "GFXCLK"); +		size += sysfs_emit_at(buf, size, "%s:\n", "GFXCLK");  		fallthrough;  	case SMU_SCLK:  		ret = aldebaran_get_current_clk_freq_by_table(smu, SMU_GFXCLK, &now); @@ -795,7 +799,7 @@ static int aldebaran_print_clk_levels(struct smu_context *smu,  		break;  	case SMU_OD_MCLK: -		size = sysfs_emit(buf, "%s:\n", "MCLK"); +		size += sysfs_emit_at(buf, size, "%s:\n", "MCLK");  		fallthrough;  	case SMU_MCLK:  		ret = aldebaran_get_current_clk_freq_by_table(smu, SMU_UCLK, &now); diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c b/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c index 627ba2eec7fd..a403657151ba 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu13/yellow_carp_ppt.c @@ -1052,16 +1052,18 @@ static int yellow_carp_print_clk_levels(struct smu_context *smu,  	int i, size = 0, ret = 0;  	uint32_t cur_value = 0, value = 0, count = 0; +	smu_cmn_get_sysfs_buf(&buf, &size); +  	switch (clk_type) {  	case SMU_OD_SCLK: -		size = sysfs_emit(buf, "%s:\n", "OD_SCLK"); +		size += sysfs_emit_at(buf, size, "%s:\n", "OD_SCLK");  		size += sysfs_emit_at(buf, size, "0: %10uMhz\n",  		(smu->gfx_actual_hard_min_freq > 0) ? smu->gfx_actual_hard_min_freq : smu->gfx_default_hard_min_freq);  		size += sysfs_emit_at(buf, size, "1: %10uMhz\n",  		(smu->gfx_actual_soft_max_freq > 0) ? smu->gfx_actual_soft_max_freq : smu->gfx_default_soft_max_freq);  		break;  	case SMU_OD_RANGE: -		size = sysfs_emit(buf, "%s:\n", "OD_RANGE"); +		size += sysfs_emit_at(buf, size, "%s:\n", "OD_RANGE");  		size += sysfs_emit_at(buf, size, "SCLK: %7uMhz %10uMhz\n",  						smu->gfx_default_hard_min_freq, smu->gfx_default_soft_max_freq);  		break; diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c index 66711ab24c15..843d2cbfc71d 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.c @@ -1053,3 +1053,24 @@ int smu_cmn_set_mp1_state(struct smu_context *smu,  	return ret;  } + +bool smu_cmn_is_audio_func_enabled(struct amdgpu_device *adev) +{ +	struct pci_dev *p = NULL; +	bool snd_driver_loaded; + +	/* +	 * If the ASIC comes with no audio function, we always assume +	 * it is "enabled". +	 */ +	p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus), +			adev->pdev->bus->number, 1); +	if (!p) +		return true; + +	snd_driver_loaded = pci_is_enabled(p) ? true : false; + +	pci_dev_put(p); + +	return snd_driver_loaded; +} diff --git a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h index 16993daa2ae0..beea03810bca 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h +++ b/drivers/gpu/drm/amd/pm/swsmu/smu_cmn.h @@ -110,5 +110,20 @@ void smu_cmn_init_soft_gpu_metrics(void *table, uint8_t frev, uint8_t crev);  int smu_cmn_set_mp1_state(struct smu_context *smu,  			  enum pp_mp1_state mp1_state); +/* + * Helper function to make sysfs_emit_at() happy. Align buf to + * the current page boundary and record the offset. + */ +static inline void smu_cmn_get_sysfs_buf(char **buf, int *offset) +{ +	if (!*buf || !offset) +		return; + +	*offset = offset_in_page(*buf); +	*buf -= *offset; +} + +bool smu_cmn_is_audio_func_enabled(struct amdgpu_device *adev); +  #endif  #endif  | 
