1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
|
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2020-2024 Intel Corporation
*/
#ifndef __IVPU_DRV_H__
#define __IVPU_DRV_H__
#include <drm/drm_device.h>
#include <drm/drm_drv.h>
#include <drm/drm_managed.h>
#include <drm/drm_mm.h>
#include <drm/drm_print.h>
#include <linux/pci.h>
#include <linux/xarray.h>
#include <uapi/drm/ivpu_accel.h>
#include "ivpu_mmu_context.h"
#include "ivpu_ipc.h"
#define DRIVER_NAME "intel_vpu"
#define DRIVER_DESC "Driver for Intel NPU (Neural Processing Unit)"
#define PCI_DEVICE_ID_MTL 0x7d1d
#define PCI_DEVICE_ID_ARL 0xad1d
#define PCI_DEVICE_ID_LNL 0x643e
#define IVPU_HW_IP_37XX 37
#define IVPU_HW_IP_40XX 40
#define IVPU_HW_IP_50XX 50
#define IVPU_HW_IP_60XX 60
#define IVPU_HW_IP_REV_LNL_B0 4
#define IVPU_HW_BTRS_MTL 1
#define IVPU_HW_BTRS_LNL 2
#define IVPU_GLOBAL_CONTEXT_MMU_SSID 0
/* SSID 1 is used by the VPU to represent reserved context */
#define IVPU_RESERVED_CONTEXT_MMU_SSID 1
#define IVPU_USER_CONTEXT_MIN_SSID 2
#define IVPU_USER_CONTEXT_MAX_SSID (IVPU_USER_CONTEXT_MIN_SSID + 63)
#define IVPU_MIN_DB 1
#define IVPU_MAX_DB 255
#define IVPU_JOB_ID_JOB_MASK GENMASK(7, 0)
#define IVPU_JOB_ID_CONTEXT_MASK GENMASK(31, 8)
#define IVPU_NUM_ENGINES 2
#define IVPU_NUM_PRIORITIES 4
#define IVPU_NUM_CMDQS_PER_CTX (IVPU_NUM_ENGINES * IVPU_NUM_PRIORITIES)
#define IVPU_CMDQ_INDEX(engine, priority) ((engine) * IVPU_NUM_PRIORITIES + (priority))
#define IVPU_PLATFORM_SILICON 0
#define IVPU_PLATFORM_SIMICS 2
#define IVPU_PLATFORM_FPGA 3
#define IVPU_PLATFORM_INVALID 8
#define IVPU_SCHED_MODE_AUTO -1
#define IVPU_DBG_REG BIT(0)
#define IVPU_DBG_IRQ BIT(1)
#define IVPU_DBG_MMU BIT(2)
#define IVPU_DBG_FILE BIT(3)
#define IVPU_DBG_MISC BIT(4)
#define IVPU_DBG_FW_BOOT BIT(5)
#define IVPU_DBG_PM BIT(6)
#define IVPU_DBG_IPC BIT(7)
#define IVPU_DBG_BO BIT(8)
#define IVPU_DBG_JOB BIT(9)
#define IVPU_DBG_JSM BIT(10)
#define IVPU_DBG_KREF BIT(11)
#define IVPU_DBG_RPM BIT(12)
#define IVPU_DBG_MMU_MAP BIT(13)
#define ivpu_err(vdev, fmt, ...) \
drm_err(&(vdev)->drm, "%s(): " fmt, __func__, ##__VA_ARGS__)
#define ivpu_err_ratelimited(vdev, fmt, ...) \
drm_err_ratelimited(&(vdev)->drm, "%s(): " fmt, __func__, ##__VA_ARGS__)
#define ivpu_warn(vdev, fmt, ...) \
drm_warn(&(vdev)->drm, "%s(): " fmt, __func__, ##__VA_ARGS__)
#define ivpu_warn_ratelimited(vdev, fmt, ...) \
drm_err_ratelimited(&(vdev)->drm, "%s(): " fmt, __func__, ##__VA_ARGS__)
#define ivpu_info(vdev, fmt, ...) drm_info(&(vdev)->drm, fmt, ##__VA_ARGS__)
#define ivpu_dbg(vdev, type, fmt, args...) do { \
if (unlikely(IVPU_DBG_##type & ivpu_dbg_mask)) \
dev_dbg((vdev)->drm.dev, "[%s] " fmt, #type, ##args); \
} while (0)
#define IVPU_WA(wa_name) (vdev->wa.wa_name)
#define IVPU_PRINT_WA(wa_name) do { \
if (IVPU_WA(wa_name)) \
ivpu_dbg(vdev, MISC, "Using WA: " #wa_name "\n"); \
} while (0)
struct ivpu_wa_table {
bool punit_disabled;
bool clear_runtime_mem;
bool interrupt_clear_with_0;
bool disable_clock_relinquish;
bool disable_d0i3_msg;
bool wp0_during_power_up;
};
struct ivpu_hw_info;
struct ivpu_mmu_info;
struct ivpu_fw_info;
struct ivpu_ipc_info;
struct ivpu_pm_info;
struct ivpu_device {
struct drm_device drm;
void __iomem *regb;
void __iomem *regv;
u32 platform;
u32 irq;
struct ivpu_wa_table wa;
struct ivpu_hw_info *hw;
struct ivpu_mmu_info *mmu;
struct ivpu_fw_info *fw;
struct ivpu_ipc_info *ipc;
struct ivpu_pm_info *pm;
struct ivpu_mmu_context gctx;
struct ivpu_mmu_context rctx;
struct mutex context_list_lock; /* Protects user context addition/removal */
struct xarray context_xa;
struct xa_limit context_xa_limit;
struct xarray db_xa;
struct xa_limit db_limit;
struct xa_limit default_db_limit;
struct mutex bo_list_lock; /* Protects bo_list */
struct list_head bo_list;
struct xarray submitted_jobs_xa;
struct ivpu_ipc_consumer job_done_consumer;
atomic64_t unique_id_counter;
ktime_t busy_start_ts;
ktime_t busy_time;
struct {
int boot;
int jsm;
int tdr;
int autosuspend;
int d0i3_entry_msg;
int state_dump_msg;
} timeout;
};
/*
* file_priv has its own refcount (ref) that allows user space to close the fd
* without blocking even if VPU is still processing some jobs.
*/
struct ivpu_file_priv {
struct kref ref;
struct ivpu_device *vdev;
struct mutex lock; /* Protects cmdq */
struct ivpu_cmdq *cmdq[IVPU_NUM_CMDQS_PER_CTX];
struct ivpu_mmu_context ctx;
struct mutex ms_lock; /* Protects ms_instance_list, ms_info_bo */
struct list_head ms_instance_list;
struct ivpu_bo *ms_info_bo;
struct xa_limit job_limit;
struct xa_limit default_job_limit;
bool has_mmu_faults;
bool bound;
bool aborted;
};
extern int ivpu_dbg_mask;
extern u8 ivpu_pll_min_ratio;
extern u8 ivpu_pll_max_ratio;
extern int ivpu_sched_mode;
extern bool ivpu_disable_mmu_cont_pages;
extern bool ivpu_force_snoop;
#define IVPU_TEST_MODE_FW_TEST BIT(0)
#define IVPU_TEST_MODE_NULL_HW BIT(1)
#define IVPU_TEST_MODE_NULL_SUBMISSION BIT(2)
#define IVPU_TEST_MODE_D0I3_MSG_DISABLE BIT(4)
#define IVPU_TEST_MODE_D0I3_MSG_ENABLE BIT(5)
#define IVPU_TEST_MODE_PREEMPTION_DISABLE BIT(6)
#define IVPU_TEST_MODE_DISABLE_TIMEOUTS BIT(8)
#define IVPU_TEST_MODE_TURBO BIT(9)
extern int ivpu_test_mode;
struct ivpu_file_priv *ivpu_file_priv_get(struct ivpu_file_priv *file_priv);
void ivpu_file_priv_put(struct ivpu_file_priv **link);
int ivpu_boot(struct ivpu_device *vdev);
int ivpu_shutdown(struct ivpu_device *vdev);
void ivpu_prepare_for_reset(struct ivpu_device *vdev);
static inline u8 ivpu_revision(struct ivpu_device *vdev)
{
return to_pci_dev(vdev->drm.dev)->revision;
}
static inline u16 ivpu_device_id(struct ivpu_device *vdev)
{
return to_pci_dev(vdev->drm.dev)->device;
}
static inline int ivpu_hw_ip_gen(struct ivpu_device *vdev)
{
switch (ivpu_device_id(vdev)) {
case PCI_DEVICE_ID_MTL:
case PCI_DEVICE_ID_ARL:
return IVPU_HW_IP_37XX;
case PCI_DEVICE_ID_LNL:
return IVPU_HW_IP_40XX;
default:
dump_stack();
ivpu_err(vdev, "Unknown NPU IP generation\n");
return 0;
}
}
static inline int ivpu_hw_btrs_gen(struct ivpu_device *vdev)
{
switch (ivpu_device_id(vdev)) {
case PCI_DEVICE_ID_MTL:
case PCI_DEVICE_ID_ARL:
return IVPU_HW_BTRS_MTL;
case PCI_DEVICE_ID_LNL:
return IVPU_HW_BTRS_LNL;
default:
dump_stack();
ivpu_err(vdev, "Unknown buttress generation\n");
return 0;
}
}
static inline struct ivpu_device *to_ivpu_device(struct drm_device *dev)
{
return container_of(dev, struct ivpu_device, drm);
}
static inline u32 ivpu_get_context_count(struct ivpu_device *vdev)
{
struct xa_limit ctx_limit = vdev->context_xa_limit;
return (ctx_limit.max - ctx_limit.min + 1);
}
static inline u32 ivpu_get_platform(struct ivpu_device *vdev)
{
WARN_ON_ONCE(vdev->platform == IVPU_PLATFORM_INVALID);
return vdev->platform;
}
static inline bool ivpu_is_silicon(struct ivpu_device *vdev)
{
return ivpu_get_platform(vdev) == IVPU_PLATFORM_SILICON;
}
static inline bool ivpu_is_simics(struct ivpu_device *vdev)
{
return ivpu_get_platform(vdev) == IVPU_PLATFORM_SIMICS;
}
static inline bool ivpu_is_fpga(struct ivpu_device *vdev)
{
return ivpu_get_platform(vdev) == IVPU_PLATFORM_FPGA;
}
static inline bool ivpu_is_force_snoop_enabled(struct ivpu_device *vdev)
{
return ivpu_force_snoop;
}
#endif /* __IVPU_DRV_H__ */
|
