1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47#include <linux/slab.h>
48
49#include "adf_accel_devices.h"
50#include "adf_common_drv.h"
51#include "icp_qat_hal.h"
52#include "icp_qat_uclo.h"
53
54#define BAD_REGADDR 0xffff
55#define MAX_RETRY_TIMES 10000
56#define INIT_CTX_ARB_VALUE 0x0
57#define INIT_CTX_ENABLE_VALUE 0x0
58#define INIT_PC_VALUE 0x0
59#define INIT_WAKEUP_EVENTS_VALUE 0x1
60#define INIT_SIG_EVENTS_VALUE 0x1
61#define INIT_CCENABLE_VALUE 0x2000
62#define RST_CSR_QAT_LSB 20
63#define RST_CSR_AE_LSB 0
64#define MC_TIMESTAMP_ENABLE (0x1 << 7)
65
66#define IGNORE_W1C_MASK ((~(1 << CE_BREAKPOINT_BITPOS)) & \
67 (~(1 << CE_CNTL_STORE_PARITY_ERROR_BITPOS)) & \
68 (~(1 << CE_REG_PAR_ERR_BITPOS)))
69#define INSERT_IMMED_GPRA_CONST(inst, const_val) \
70 (inst = ((inst & 0xFFFF00C03FFull) | \
71 ((((const_val) << 12) & 0x0FF00000ull) | \
72 (((const_val) << 10) & 0x0003FC00ull))))
73#define INSERT_IMMED_GPRB_CONST(inst, const_val) \
74 (inst = ((inst & 0xFFFF00FFF00ull) | \
75 ((((const_val) << 12) & 0x0FF00000ull) | \
76 (((const_val) << 0) & 0x000000FFull))))
77
78#define AE(handle, ae) handle->hal_handle->aes[ae]
79
80static const uint64_t inst_4b[] = {
81 0x0F0400C0000ull, 0x0F4400C0000ull, 0x0F040000300ull, 0x0F440000300ull,
82 0x0FC066C0000ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
83 0x0A021000000ull
84};
85
86static const uint64_t inst[] = {
87 0x0F0000C0000ull, 0x0F000000380ull, 0x0D805000011ull, 0x0FC082C0300ull,
88 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
89 0x0A0643C0000ull, 0x0BAC0000301ull, 0x0D802000101ull, 0x0F0000C0001ull,
90 0x0FC066C0001ull, 0x0F0000C0300ull, 0x0F0000C0300ull, 0x0F0000C0300ull,
91 0x0F000400300ull, 0x0A0610C0000ull, 0x0BAC0000301ull, 0x0D804400101ull,
92 0x0A0580C0000ull, 0x0A0581C0000ull, 0x0A0582C0000ull, 0x0A0583C0000ull,
93 0x0A0584C0000ull, 0x0A0585C0000ull, 0x0A0586C0000ull, 0x0A0587C0000ull,
94 0x0A0588C0000ull, 0x0A0589C0000ull, 0x0A058AC0000ull, 0x0A058BC0000ull,
95 0x0A058CC0000ull, 0x0A058DC0000ull, 0x0A058EC0000ull, 0x0A058FC0000ull,
96 0x0A05C0C0000ull, 0x0A05C1C0000ull, 0x0A05C2C0000ull, 0x0A05C3C0000ull,
97 0x0A05C4C0000ull, 0x0A05C5C0000ull, 0x0A05C6C0000ull, 0x0A05C7C0000ull,
98 0x0A05C8C0000ull, 0x0A05C9C0000ull, 0x0A05CAC0000ull, 0x0A05CBC0000ull,
99 0x0A05CCC0000ull, 0x0A05CDC0000ull, 0x0A05CEC0000ull, 0x0A05CFC0000ull,
100 0x0A0400C0000ull, 0x0B0400C0000ull, 0x0A0401C0000ull, 0x0B0401C0000ull,
101 0x0A0402C0000ull, 0x0B0402C0000ull, 0x0A0403C0000ull, 0x0B0403C0000ull,
102 0x0A0404C0000ull, 0x0B0404C0000ull, 0x0A0405C0000ull, 0x0B0405C0000ull,
103 0x0A0406C0000ull, 0x0B0406C0000ull, 0x0A0407C0000ull, 0x0B0407C0000ull,
104 0x0A0408C0000ull, 0x0B0408C0000ull, 0x0A0409C0000ull, 0x0B0409C0000ull,
105 0x0A040AC0000ull, 0x0B040AC0000ull, 0x0A040BC0000ull, 0x0B040BC0000ull,
106 0x0A040CC0000ull, 0x0B040CC0000ull, 0x0A040DC0000ull, 0x0B040DC0000ull,
107 0x0A040EC0000ull, 0x0B040EC0000ull, 0x0A040FC0000ull, 0x0B040FC0000ull,
108 0x0D81581C010ull, 0x0E000010000ull, 0x0E000010000ull,
109};
110
111void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle,
112 unsigned char ae, unsigned int ctx_mask)
113{
114 AE(handle, ae).live_ctx_mask = ctx_mask;
115}
116
117#define CSR_RETRY_TIMES 500
118static int qat_hal_rd_ae_csr(struct icp_qat_fw_loader_handle *handle,
119 unsigned char ae, unsigned int csr,
120 unsigned int *value)
121{
122 unsigned int iterations = CSR_RETRY_TIMES;
123
124 do {
125 *value = GET_AE_CSR(handle, ae, csr);
126 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
127 return 0;
128 } while (iterations--);
129
130 pr_err("QAT: Read CSR timeout\n");
131 return -EFAULT;
132}
133
134static int qat_hal_wr_ae_csr(struct icp_qat_fw_loader_handle *handle,
135 unsigned char ae, unsigned int csr,
136 unsigned int value)
137{
138 unsigned int iterations = CSR_RETRY_TIMES;
139
140 do {
141 SET_AE_CSR(handle, ae, csr, value);
142 if (!(GET_AE_CSR(handle, ae, LOCAL_CSR_STATUS) & LCS_STATUS))
143 return 0;
144 } while (iterations--);
145
146 pr_err("QAT: Write CSR Timeout\n");
147 return -EFAULT;
148}
149
150static void qat_hal_get_wakeup_event(struct icp_qat_fw_loader_handle *handle,
151 unsigned char ae, unsigned char ctx,
152 unsigned int *events)
153{
154 unsigned int cur_ctx;
155
156 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
157 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
158 qat_hal_rd_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT, events);
159 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
160}
161
162static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle,
163 unsigned char ae, unsigned int cycles,
164 int chk_inactive)
165{
166 unsigned int base_cnt = 0, cur_cnt = 0;
167 unsigned int csr = (1 << ACS_ABO_BITPOS);
168 int times = MAX_RETRY_TIMES;
169 int elapsed_cycles = 0;
170
171 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &base_cnt);
172 base_cnt &= 0xffff;
173 while ((int)cycles > elapsed_cycles && times--) {
174 if (chk_inactive)
175 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &csr);
176
177 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, &cur_cnt);
178 cur_cnt &= 0xffff;
179 elapsed_cycles = cur_cnt - base_cnt;
180
181 if (elapsed_cycles < 0)
182 elapsed_cycles += 0x10000;
183
184
185 if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS)))
186 return 0;
187 }
188 if (!times) {
189 pr_err("QAT: wait_num_cycles time out\n");
190 return -EFAULT;
191 }
192 return 0;
193}
194
195#define CLR_BIT(wrd, bit) (wrd & ~(1 << bit))
196#define SET_BIT(wrd, bit) (wrd | 1 << bit)
197
198int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle,
199 unsigned char ae, unsigned char mode)
200{
201 unsigned int csr, new_csr;
202
203 if ((mode != 4) && (mode != 8)) {
204 pr_err("QAT: bad ctx mode=%d\n", mode);
205 return -EINVAL;
206 }
207
208
209 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
210 csr = IGNORE_W1C_MASK & csr;
211 new_csr = (mode == 4) ?
212 SET_BIT(csr, CE_INUSE_CONTEXTS_BITPOS) :
213 CLR_BIT(csr, CE_INUSE_CONTEXTS_BITPOS);
214 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
215 return 0;
216}
217
218int qat_hal_set_ae_nn_mode(struct icp_qat_fw_loader_handle *handle,
219 unsigned char ae, unsigned char mode)
220{
221 unsigned int csr, new_csr;
222
223 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
224 csr &= IGNORE_W1C_MASK;
225
226 new_csr = (mode) ?
227 SET_BIT(csr, CE_NN_MODE_BITPOS) :
228 CLR_BIT(csr, CE_NN_MODE_BITPOS);
229
230 if (new_csr != csr)
231 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
232
233 return 0;
234}
235
236int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle,
237 unsigned char ae, enum icp_qat_uof_regtype lm_type,
238 unsigned char mode)
239{
240 unsigned int csr, new_csr;
241
242 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr);
243 csr &= IGNORE_W1C_MASK;
244 switch (lm_type) {
245 case ICP_LMEM0:
246 new_csr = (mode) ?
247 SET_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS) :
248 CLR_BIT(csr, CE_LMADDR_0_GLOBAL_BITPOS);
249 break;
250 case ICP_LMEM1:
251 new_csr = (mode) ?
252 SET_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS) :
253 CLR_BIT(csr, CE_LMADDR_1_GLOBAL_BITPOS);
254 break;
255 default:
256 pr_err("QAT: lmType = 0x%x\n", lm_type);
257 return -EINVAL;
258 }
259
260 if (new_csr != csr)
261 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, new_csr);
262 return 0;
263}
264
265static unsigned short qat_hal_get_reg_addr(unsigned int type,
266 unsigned short reg_num)
267{
268 unsigned short reg_addr;
269
270 switch (type) {
271 case ICP_GPA_ABS:
272 case ICP_GPB_ABS:
273 reg_addr = 0x80 | (reg_num & 0x7f);
274 break;
275 case ICP_GPA_REL:
276 case ICP_GPB_REL:
277 reg_addr = reg_num & 0x1f;
278 break;
279 case ICP_SR_RD_REL:
280 case ICP_SR_WR_REL:
281 case ICP_SR_REL:
282 reg_addr = 0x180 | (reg_num & 0x1f);
283 break;
284 case ICP_SR_ABS:
285 reg_addr = 0x140 | ((reg_num & 0x3) << 1);
286 break;
287 case ICP_DR_RD_REL:
288 case ICP_DR_WR_REL:
289 case ICP_DR_REL:
290 reg_addr = 0x1c0 | (reg_num & 0x1f);
291 break;
292 case ICP_DR_ABS:
293 reg_addr = 0x100 | ((reg_num & 0x3) << 1);
294 break;
295 case ICP_NEIGH_REL:
296 reg_addr = 0x280 | (reg_num & 0x1f);
297 break;
298 case ICP_LMEM0:
299 reg_addr = 0x200;
300 break;
301 case ICP_LMEM1:
302 reg_addr = 0x220;
303 break;
304 case ICP_NO_DEST:
305 reg_addr = 0x300 | (reg_num & 0xff);
306 break;
307 default:
308 reg_addr = BAD_REGADDR;
309 break;
310 }
311 return reg_addr;
312}
313
314void qat_hal_reset(struct icp_qat_fw_loader_handle *handle)
315{
316 unsigned int ae_reset_csr;
317
318 ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
319 ae_reset_csr |= handle->hal_handle->ae_mask << RST_CSR_AE_LSB;
320 ae_reset_csr |= handle->hal_handle->slice_mask << RST_CSR_QAT_LSB;
321 SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
322}
323
324static void qat_hal_wr_indr_csr(struct icp_qat_fw_loader_handle *handle,
325 unsigned char ae, unsigned int ctx_mask,
326 unsigned int ae_csr, unsigned int csr_val)
327{
328 unsigned int ctx, cur_ctx;
329
330 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
331
332 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
333 if (!(ctx_mask & (1 << ctx)))
334 continue;
335 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
336 qat_hal_wr_ae_csr(handle, ae, ae_csr, csr_val);
337 }
338
339 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
340}
341
342static void qat_hal_rd_indr_csr(struct icp_qat_fw_loader_handle *handle,
343 unsigned char ae, unsigned char ctx,
344 unsigned int ae_csr, unsigned int *csr_val)
345{
346 unsigned int cur_ctx;
347
348 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
349 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
350 qat_hal_rd_ae_csr(handle, ae, ae_csr, csr_val);
351 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
352}
353
354static void qat_hal_put_sig_event(struct icp_qat_fw_loader_handle *handle,
355 unsigned char ae, unsigned int ctx_mask,
356 unsigned int events)
357{
358 unsigned int ctx, cur_ctx;
359
360 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
361 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
362 if (!(ctx_mask & (1 << ctx)))
363 continue;
364 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
365 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_INDIRECT, events);
366 }
367 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
368}
369
370static void qat_hal_put_wakeup_event(struct icp_qat_fw_loader_handle *handle,
371 unsigned char ae, unsigned int ctx_mask,
372 unsigned int events)
373{
374 unsigned int ctx, cur_ctx;
375
376 qat_hal_rd_ae_csr(handle, ae, CSR_CTX_POINTER, &cur_ctx);
377 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
378 if (!(ctx_mask & (1 << ctx)))
379 continue;
380 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, ctx);
381 qat_hal_wr_ae_csr(handle, ae, CTX_WAKEUP_EVENTS_INDIRECT,
382 events);
383 }
384 qat_hal_wr_ae_csr(handle, ae, CSR_CTX_POINTER, cur_ctx);
385}
386
387static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle)
388{
389 unsigned int base_cnt, cur_cnt;
390 unsigned char ae;
391 unsigned int times = MAX_RETRY_TIMES;
392
393 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
394 if (!(handle->hal_handle->ae_mask & (1 << ae)))
395 continue;
396
397 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT,
398 (unsigned int *)&base_cnt);
399 base_cnt &= 0xffff;
400
401 do {
402 qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT,
403 (unsigned int *)&cur_cnt);
404 cur_cnt &= 0xffff;
405 } while (times-- && (cur_cnt == base_cnt));
406
407 if (!times) {
408 pr_err("QAT: AE%d is inactive!!\n", ae);
409 return -EFAULT;
410 }
411 }
412
413 return 0;
414}
415
416static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle)
417{
418 unsigned int misc_ctl;
419 unsigned char ae;
420
421
422 misc_ctl = GET_GLB_CSR(handle, MISC_CONTROL);
423 if (misc_ctl & MC_TIMESTAMP_ENABLE)
424 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl &
425 (~MC_TIMESTAMP_ENABLE));
426
427 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
428 if (!(handle->hal_handle->ae_mask & (1 << ae)))
429 continue;
430 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0);
431 qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0);
432 }
433
434 SET_GLB_CSR(handle, MISC_CONTROL, misc_ctl | MC_TIMESTAMP_ENABLE);
435}
436
437#define ESRAM_AUTO_TINIT (1<<2)
438#define ESRAM_AUTO_TINIT_DONE (1<<3)
439#define ESRAM_AUTO_INIT_USED_CYCLES (1640)
440#define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C
441static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle)
442{
443 void __iomem *csr_addr = handle->hal_ep_csr_addr_v +
444 ESRAM_AUTO_INIT_CSR_OFFSET;
445 unsigned int csr_val, times = 30;
446
447 csr_val = ADF_CSR_RD(csr_addr, 0);
448 if ((csr_val & ESRAM_AUTO_TINIT) && (csr_val & ESRAM_AUTO_TINIT_DONE))
449 return 0;
450
451 csr_val = ADF_CSR_RD(csr_addr, 0);
452 csr_val |= ESRAM_AUTO_TINIT;
453 ADF_CSR_WR(csr_addr, 0, csr_val);
454
455 do {
456 qat_hal_wait_cycles(handle, 0, ESRAM_AUTO_INIT_USED_CYCLES, 0);
457 csr_val = ADF_CSR_RD(csr_addr, 0);
458 } while (!(csr_val & ESRAM_AUTO_TINIT_DONE) && times--);
459 if ((!times)) {
460 pr_err("QAT: Fail to init eSram!\n");
461 return -EFAULT;
462 }
463 return 0;
464}
465
466#define SHRAM_INIT_CYCLES 2060
467int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle)
468{
469 unsigned int ae_reset_csr;
470 unsigned char ae;
471 unsigned int clk_csr;
472 unsigned int times = 100;
473 unsigned int csr;
474
475
476 ae_reset_csr = GET_GLB_CSR(handle, ICP_RESET);
477 ae_reset_csr &= ~(handle->hal_handle->ae_mask << RST_CSR_AE_LSB);
478 ae_reset_csr &= ~(handle->hal_handle->slice_mask << RST_CSR_QAT_LSB);
479 do {
480 SET_GLB_CSR(handle, ICP_RESET, ae_reset_csr);
481 if (!(times--))
482 goto out_err;
483 csr = GET_GLB_CSR(handle, ICP_RESET);
484 } while ((handle->hal_handle->ae_mask |
485 (handle->hal_handle->slice_mask << RST_CSR_QAT_LSB)) & csr);
486
487 clk_csr = GET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE);
488 clk_csr |= handle->hal_handle->ae_mask << 0;
489 clk_csr |= handle->hal_handle->slice_mask << 20;
490 SET_GLB_CSR(handle, ICP_GLOBAL_CLK_ENABLE, clk_csr);
491 if (qat_hal_check_ae_alive(handle))
492 goto out_err;
493
494
495 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
496 if (!(handle->hal_handle->ae_mask & (1 << ae)))
497 continue;
498 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
499 INIT_CTX_ENABLE_VALUE);
500 qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX,
501 CTX_STS_INDIRECT,
502 handle->hal_handle->upc_mask &
503 INIT_PC_VALUE);
504 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
505 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
506 qat_hal_put_wakeup_event(handle, ae,
507 ICP_QAT_UCLO_AE_ALL_CTX,
508 INIT_WAKEUP_EVENTS_VALUE);
509 qat_hal_put_sig_event(handle, ae,
510 ICP_QAT_UCLO_AE_ALL_CTX,
511 INIT_SIG_EVENTS_VALUE);
512 }
513 if (qat_hal_init_esram(handle))
514 goto out_err;
515 if (qat_hal_wait_cycles(handle, 0, SHRAM_INIT_CYCLES, 0))
516 goto out_err;
517 qat_hal_reset_timestamp(handle);
518
519 return 0;
520out_err:
521 pr_err("QAT: failed to get device out of reset\n");
522 return -EFAULT;
523}
524
525static void qat_hal_disable_ctx(struct icp_qat_fw_loader_handle *handle,
526 unsigned char ae, unsigned int ctx_mask)
527{
528 unsigned int ctx;
529
530 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx);
531 ctx &= IGNORE_W1C_MASK &
532 (~((ctx_mask & ICP_QAT_UCLO_AE_ALL_CTX) << CE_ENABLE_BITPOS));
533 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
534}
535
536static uint64_t qat_hal_parity_64bit(uint64_t word)
537{
538 word ^= word >> 1;
539 word ^= word >> 2;
540 word ^= word >> 4;
541 word ^= word >> 8;
542 word ^= word >> 16;
543 word ^= word >> 32;
544 return word & 1;
545}
546
547static uint64_t qat_hal_set_uword_ecc(uint64_t uword)
548{
549 uint64_t bit0_mask = 0xff800007fffULL, bit1_mask = 0x1f801ff801fULL,
550 bit2_mask = 0xe387e0781e1ULL, bit3_mask = 0x7cb8e388e22ULL,
551 bit4_mask = 0xaf5b2c93244ULL, bit5_mask = 0xf56d5525488ULL,
552 bit6_mask = 0xdaf69a46910ULL;
553
554
555 uword &= ~(0x7fULL << 0x2C);
556 uword |= qat_hal_parity_64bit(bit0_mask & uword) << 0x2C;
557 uword |= qat_hal_parity_64bit(bit1_mask & uword) << 0x2D;
558 uword |= qat_hal_parity_64bit(bit2_mask & uword) << 0x2E;
559 uword |= qat_hal_parity_64bit(bit3_mask & uword) << 0x2F;
560 uword |= qat_hal_parity_64bit(bit4_mask & uword) << 0x30;
561 uword |= qat_hal_parity_64bit(bit5_mask & uword) << 0x31;
562 uword |= qat_hal_parity_64bit(bit6_mask & uword) << 0x32;
563 return uword;
564}
565
566void qat_hal_wr_uwords(struct icp_qat_fw_loader_handle *handle,
567 unsigned char ae, unsigned int uaddr,
568 unsigned int words_num, uint64_t *uword)
569{
570 unsigned int ustore_addr;
571 unsigned int i;
572
573 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
574 uaddr |= UA_ECS;
575 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
576 for (i = 0; i < words_num; i++) {
577 unsigned int uwrd_lo, uwrd_hi;
578 uint64_t tmp;
579
580 tmp = qat_hal_set_uword_ecc(uword[i]);
581 uwrd_lo = (unsigned int)(tmp & 0xffffffff);
582 uwrd_hi = (unsigned int)(tmp >> 0x20);
583 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
584 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
585 }
586 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
587}
588
589static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle,
590 unsigned char ae, unsigned int ctx_mask)
591{
592 unsigned int ctx;
593
594 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx);
595 ctx &= IGNORE_W1C_MASK;
596 ctx_mask &= (ctx & CE_INUSE_CONTEXTS) ? 0x55 : 0xFF;
597 ctx |= (ctx_mask << CE_ENABLE_BITPOS);
598 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx);
599}
600
601static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle)
602{
603 unsigned char ae;
604 unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX;
605 int times = MAX_RETRY_TIMES;
606 unsigned int csr_val = 0;
607 unsigned short reg;
608 unsigned int savctx = 0;
609 int ret = 0;
610
611 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
612 if (!(handle->hal_handle->ae_mask & (1 << ae)))
613 continue;
614 for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) {
615 qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS,
616 reg, 0);
617 qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS,
618 reg, 0);
619 }
620 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
621 csr_val &= ~(1 << MMC_SHARE_CS_BITPOS);
622 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val);
623 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &csr_val);
624 csr_val &= IGNORE_W1C_MASK;
625 csr_val |= CE_NN_MODE;
626 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, csr_val);
627 qat_hal_wr_uwords(handle, ae, 0, ARRAY_SIZE(inst),
628 (uint64_t *)inst);
629 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
630 handle->hal_handle->upc_mask &
631 INIT_PC_VALUE);
632 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
633 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, 0);
634 qat_hal_put_wakeup_event(handle, ae, ctx_mask, XCWE_VOLUNTARY);
635 qat_hal_wr_indr_csr(handle, ae, ctx_mask,
636 CTX_SIG_EVENTS_INDIRECT, 0);
637 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
638 qat_hal_enable_ctx(handle, ae, ctx_mask);
639 }
640 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
641 if (!(handle->hal_handle->ae_mask & (1 << ae)))
642 continue;
643
644 do {
645 ret = qat_hal_wait_cycles(handle, ae, 20, 1);
646 } while (ret && times--);
647
648 if (!times) {
649 pr_err("QAT: clear GPR of AE %d failed", ae);
650 return -EINVAL;
651 }
652 qat_hal_disable_ctx(handle, ae, ctx_mask);
653 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
654 savctx & ACS_ACNO);
655 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES,
656 INIT_CTX_ENABLE_VALUE);
657 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
658 handle->hal_handle->upc_mask &
659 INIT_PC_VALUE);
660 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, INIT_CTX_ARB_VALUE);
661 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, INIT_CCENABLE_VALUE);
662 qat_hal_put_wakeup_event(handle, ae, ctx_mask,
663 INIT_WAKEUP_EVENTS_VALUE);
664 qat_hal_put_sig_event(handle, ae, ctx_mask,
665 INIT_SIG_EVENTS_VALUE);
666 }
667 return 0;
668}
669
670#define ICP_DH895XCC_AE_OFFSET 0x20000
671#define ICP_DH895XCC_CAP_OFFSET (ICP_DH895XCC_AE_OFFSET + 0x10000)
672#define LOCAL_TO_XFER_REG_OFFSET 0x800
673#define ICP_DH895XCC_EP_OFFSET 0x3a000
674#define ICP_DH895XCC_PMISC_BAR 1
675int qat_hal_init(struct adf_accel_dev *accel_dev)
676{
677 unsigned char ae;
678 unsigned int max_en_ae_id = 0;
679 struct icp_qat_fw_loader_handle *handle;
680 struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
681 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
682 struct adf_bar *bar =
683 &pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)];
684
685 handle = kzalloc(sizeof(*handle), GFP_KERNEL);
686 if (!handle)
687 return -ENOMEM;
688
689 handle->hal_cap_g_ctl_csr_addr_v = bar->virt_addr +
690 ICP_DH895XCC_CAP_OFFSET;
691 handle->hal_cap_ae_xfer_csr_addr_v = bar->virt_addr +
692 ICP_DH895XCC_AE_OFFSET;
693 handle->hal_ep_csr_addr_v = bar->virt_addr + ICP_DH895XCC_EP_OFFSET;
694 handle->hal_cap_ae_local_csr_addr_v =
695 handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET;
696
697 handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL);
698 if (!handle->hal_handle)
699 goto out_hal_handle;
700 handle->hal_handle->revision_id = accel_dev->accel_pci_dev.revid;
701 handle->hal_handle->ae_mask = hw_data->ae_mask;
702 handle->hal_handle->slice_mask = hw_data->accel_mask;
703
704 handle->hal_handle->upc_mask = 0x1ffff;
705 handle->hal_handle->max_ustore = 0x4000;
706 for (ae = 0; ae < ICP_QAT_UCLO_MAX_AE; ae++) {
707 if (!(hw_data->ae_mask & (1 << ae)))
708 continue;
709 handle->hal_handle->aes[ae].free_addr = 0;
710 handle->hal_handle->aes[ae].free_size =
711 handle->hal_handle->max_ustore;
712 handle->hal_handle->aes[ae].ustore_size =
713 handle->hal_handle->max_ustore;
714 handle->hal_handle->aes[ae].live_ctx_mask =
715 ICP_QAT_UCLO_AE_ALL_CTX;
716 max_en_ae_id = ae;
717 }
718 handle->hal_handle->ae_max_num = max_en_ae_id + 1;
719
720 if (qat_hal_clr_reset(handle)) {
721 pr_err("QAT: qat_hal_clr_reset error\n");
722 goto out_err;
723 }
724 if (qat_hal_clear_gpr(handle))
725 goto out_err;
726
727 for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) {
728 unsigned int csr_val = 0;
729
730 if (!(hw_data->ae_mask & (1 << ae)))
731 continue;
732 qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE, &csr_val);
733 csr_val |= 0x1;
734 qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val);
735 }
736 accel_dev->fw_loader->fw_loader = handle;
737 return 0;
738
739out_err:
740 kfree(handle->hal_handle);
741out_hal_handle:
742 kfree(handle);
743 return -EFAULT;
744}
745
746void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle)
747{
748 if (!handle)
749 return;
750 kfree(handle->hal_handle);
751 kfree(handle);
752}
753
754void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
755 unsigned int ctx_mask)
756{
757 qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) &
758 ICP_QAT_UCLO_AE_ALL_CTX, 0x10000);
759 qat_hal_enable_ctx(handle, ae, ctx_mask);
760}
761
762void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae,
763 unsigned int ctx_mask)
764{
765 qat_hal_disable_ctx(handle, ae, ctx_mask);
766}
767
768void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle,
769 unsigned char ae, unsigned int ctx_mask, unsigned int upc)
770{
771 qat_hal_wr_indr_csr(handle, ae, ctx_mask, CTX_STS_INDIRECT,
772 handle->hal_handle->upc_mask & upc);
773}
774
775static void qat_hal_get_uwords(struct icp_qat_fw_loader_handle *handle,
776 unsigned char ae, unsigned int uaddr,
777 unsigned int words_num, uint64_t *uword)
778{
779 unsigned int i, uwrd_lo, uwrd_hi;
780 unsigned int ustore_addr, misc_control;
781
782 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &misc_control);
783 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL,
784 misc_control & 0xfffffffb);
785 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
786 uaddr |= UA_ECS;
787 for (i = 0; i < words_num; i++) {
788 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
789 uaddr++;
790 qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_LOWER, &uwrd_lo);
791 qat_hal_rd_ae_csr(handle, ae, USTORE_DATA_UPPER, &uwrd_hi);
792 uword[i] = uwrd_hi;
793 uword[i] = (uword[i] << 0x20) | uwrd_lo;
794 }
795 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, misc_control);
796 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
797}
798
799void qat_hal_wr_umem(struct icp_qat_fw_loader_handle *handle,
800 unsigned char ae, unsigned int uaddr,
801 unsigned int words_num, unsigned int *data)
802{
803 unsigned int i, ustore_addr;
804
805 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
806 uaddr |= UA_ECS;
807 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
808 for (i = 0; i < words_num; i++) {
809 unsigned int uwrd_lo, uwrd_hi, tmp;
810
811 uwrd_lo = ((data[i] & 0xfff0000) << 4) | (0x3 << 18) |
812 ((data[i] & 0xff00) << 2) |
813 (0x3 << 8) | (data[i] & 0xff);
814 uwrd_hi = (0xf << 4) | ((data[i] & 0xf0000000) >> 28);
815 uwrd_hi |= (hweight32(data[i] & 0xffff) & 0x1) << 8;
816 tmp = ((data[i] >> 0x10) & 0xffff);
817 uwrd_hi |= (hweight32(tmp) & 0x1) << 9;
818 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
819 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
820 }
821 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
822}
823
824#define MAX_EXEC_INST 100
825static int qat_hal_exec_micro_inst(struct icp_qat_fw_loader_handle *handle,
826 unsigned char ae, unsigned char ctx,
827 uint64_t *micro_inst, unsigned int inst_num,
828 int code_off, unsigned int max_cycle,
829 unsigned int *endpc)
830{
831 uint64_t savuwords[MAX_EXEC_INST];
832 unsigned int ind_lm_addr0, ind_lm_addr1;
833 unsigned int ind_lm_addr_byte0, ind_lm_addr_byte1;
834 unsigned int ind_cnt_sig;
835 unsigned int ind_sig, act_sig;
836 unsigned int csr_val = 0, newcsr_val;
837 unsigned int savctx;
838 unsigned int savcc, wakeup_events, savpc;
839 unsigned int ctxarb_ctl, ctx_enables;
840
841 if ((inst_num > handle->hal_handle->max_ustore) || !micro_inst) {
842 pr_err("QAT: invalid instruction num %d\n", inst_num);
843 return -EINVAL;
844 }
845
846 qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_0_INDIRECT, &ind_lm_addr0);
847 qat_hal_rd_indr_csr(handle, ae, ctx, LM_ADDR_1_INDIRECT, &ind_lm_addr1);
848 qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_0_BYTE_INDEX,
849 &ind_lm_addr_byte0);
850 qat_hal_rd_indr_csr(handle, ae, ctx, INDIRECT_LM_ADDR_1_BYTE_INDEX,
851 &ind_lm_addr_byte1);
852 if (inst_num <= MAX_EXEC_INST)
853 qat_hal_get_uwords(handle, ae, 0, inst_num, savuwords);
854 qat_hal_get_wakeup_event(handle, ae, ctx, &wakeup_events);
855 qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT, &savpc);
856 savpc = (savpc & handle->hal_handle->upc_mask) >> 0;
857 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
858 ctx_enables &= IGNORE_W1C_MASK;
859 qat_hal_rd_ae_csr(handle, ae, CC_ENABLE, &savcc);
860 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
861 qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_ctl);
862 qat_hal_rd_indr_csr(handle, ae, ctx, FUTURE_COUNT_SIGNAL_INDIRECT,
863 &ind_cnt_sig);
864 qat_hal_rd_indr_csr(handle, ae, ctx, CTX_SIG_EVENTS_INDIRECT, &ind_sig);
865 qat_hal_rd_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, &act_sig);
866
867 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
868 qat_hal_wr_uwords(handle, ae, 0, inst_num, micro_inst);
869 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT, 0);
870 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, ctx & ACS_ACNO);
871 if (code_off)
872 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc & 0xffffdfff);
873 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), XCWE_VOLUNTARY);
874 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_SIG_EVENTS_INDIRECT, 0);
875 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, 0);
876 qat_hal_enable_ctx(handle, ae, (1 << ctx));
877
878 if (qat_hal_wait_cycles(handle, ae, max_cycle, 1) != 0)
879 return -EFAULT;
880 if (endpc) {
881 unsigned int ctx_status;
882
883 qat_hal_rd_indr_csr(handle, ae, ctx, CTX_STS_INDIRECT,
884 &ctx_status);
885 *endpc = ctx_status & handle->hal_handle->upc_mask;
886 }
887
888 qat_hal_disable_ctx(handle, ae, (1 << ctx));
889 if (inst_num <= MAX_EXEC_INST)
890 qat_hal_wr_uwords(handle, ae, 0, inst_num, savuwords);
891 qat_hal_put_wakeup_event(handle, ae, (1 << ctx), wakeup_events);
892 qat_hal_wr_indr_csr(handle, ae, (1 << ctx), CTX_STS_INDIRECT,
893 handle->hal_handle->upc_mask & savpc);
894 qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val);
895 newcsr_val = CLR_BIT(csr_val, MMC_SHARE_CS_BITPOS);
896 qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, newcsr_val);
897 qat_hal_wr_ae_csr(handle, ae, CC_ENABLE, savcc);
898 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS, savctx & ACS_ACNO);
899 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_ctl);
900 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
901 LM_ADDR_0_INDIRECT, ind_lm_addr0);
902 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
903 LM_ADDR_1_INDIRECT, ind_lm_addr1);
904 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
905 INDIRECT_LM_ADDR_0_BYTE_INDEX, ind_lm_addr_byte0);
906 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
907 INDIRECT_LM_ADDR_1_BYTE_INDEX, ind_lm_addr_byte1);
908 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
909 FUTURE_COUNT_SIGNAL_INDIRECT, ind_cnt_sig);
910 qat_hal_wr_indr_csr(handle, ae, (1 << ctx),
911 CTX_SIG_EVENTS_INDIRECT, ind_sig);
912 qat_hal_wr_ae_csr(handle, ae, CTX_SIG_EVENTS_ACTIVE, act_sig);
913 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
914
915 return 0;
916}
917
918static int qat_hal_rd_rel_reg(struct icp_qat_fw_loader_handle *handle,
919 unsigned char ae, unsigned char ctx,
920 enum icp_qat_uof_regtype reg_type,
921 unsigned short reg_num, unsigned int *data)
922{
923 unsigned int savctx, uaddr, uwrd_lo, uwrd_hi;
924 unsigned int ctxarb_cntl, ustore_addr, ctx_enables;
925 unsigned short reg_addr;
926 int status = 0;
927 uint64_t insts, savuword;
928
929 reg_addr = qat_hal_get_reg_addr(reg_type, reg_num);
930 if (reg_addr == BAD_REGADDR) {
931 pr_err("QAT: bad regaddr=0x%x\n", reg_addr);
932 return -EINVAL;
933 }
934 switch (reg_type) {
935 case ICP_GPA_REL:
936 insts = 0xA070000000ull | (reg_addr & 0x3ff);
937 break;
938 default:
939 insts = (uint64_t)0xA030000000ull | ((reg_addr & 0x3ff) << 10);
940 break;
941 }
942 qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &savctx);
943 qat_hal_rd_ae_csr(handle, ae, CTX_ARB_CNTL, &ctxarb_cntl);
944 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
945 ctx_enables &= IGNORE_W1C_MASK;
946 if (ctx != (savctx & ACS_ACNO))
947 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
948 ctx & ACS_ACNO);
949 qat_hal_get_uwords(handle, ae, 0, 1, &savuword);
950 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
951 qat_hal_rd_ae_csr(handle, ae, USTORE_ADDRESS, &ustore_addr);
952 uaddr = UA_ECS;
953 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
954 insts = qat_hal_set_uword_ecc(insts);
955 uwrd_lo = (unsigned int)(insts & 0xffffffff);
956 uwrd_hi = (unsigned int)(insts >> 0x20);
957 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_LOWER, uwrd_lo);
958 qat_hal_wr_ae_csr(handle, ae, USTORE_DATA_UPPER, uwrd_hi);
959 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, uaddr);
960
961 qat_hal_wait_cycles(handle, ae, 0x8, 0);
962
963
964
965
966
967 qat_hal_rd_ae_csr(handle, ae, ALU_OUT, data);
968 qat_hal_wr_ae_csr(handle, ae, USTORE_ADDRESS, ustore_addr);
969 qat_hal_wr_uwords(handle, ae, 0, 1, &savuword);
970 if (ctx != (savctx & ACS_ACNO))
971 qat_hal_wr_ae_csr(handle, ae, ACTIVE_CTX_STATUS,
972 savctx & ACS_ACNO);
973 qat_hal_wr_ae_csr(handle, ae, CTX_ARB_CNTL, ctxarb_cntl);
974 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
975
976 return status;
977}
978
979static int qat_hal_wr_rel_reg(struct icp_qat_fw_loader_handle *handle,
980 unsigned char ae, unsigned char ctx,
981 enum icp_qat_uof_regtype reg_type,
982 unsigned short reg_num, unsigned int data)
983{
984 unsigned short src_hiaddr, src_lowaddr, dest_addr, data16hi, data16lo;
985 uint64_t insts[] = {
986 0x0F440000000ull,
987 0x0F040000000ull,
988 0x0F0000C0300ull,
989 0x0E000010000ull
990 };
991 const int num_inst = ARRAY_SIZE(insts), code_off = 1;
992 const int imm_w1 = 0, imm_w0 = 1;
993
994 dest_addr = qat_hal_get_reg_addr(reg_type, reg_num);
995 if (dest_addr == BAD_REGADDR) {
996 pr_err("QAT: bad destAddr=0x%x\n", dest_addr);
997 return -EINVAL;
998 }
999
1000 data16lo = 0xffff & data;
1001 data16hi = 0xffff & (data >> 0x10);
1002 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1003 (0xff & data16hi));
1004 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST, (unsigned short)
1005 (0xff & data16lo));
1006 switch (reg_type) {
1007 case ICP_GPA_REL:
1008 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1009 ((src_hiaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1010 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1011 ((src_lowaddr & 0x3ff) << 10) | (dest_addr & 0x3ff);
1012 break;
1013 default:
1014 insts[imm_w1] = insts[imm_w1] | ((data16hi >> 8) << 20) |
1015 ((dest_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1016
1017 insts[imm_w0] = insts[imm_w0] | ((data16lo >> 8) << 20) |
1018 ((dest_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1019 break;
1020 }
1021
1022 return qat_hal_exec_micro_inst(handle, ae, ctx, insts, num_inst,
1023 code_off, num_inst * 0x5, NULL);
1024}
1025
1026int qat_hal_get_ins_num(void)
1027{
1028 return ARRAY_SIZE(inst_4b);
1029}
1030
1031static int qat_hal_concat_micro_code(uint64_t *micro_inst,
1032 unsigned int inst_num, unsigned int size,
1033 unsigned int addr, unsigned int *value)
1034{
1035 int i, val_indx;
1036 unsigned int cur_value;
1037 const uint64_t *inst_arr;
1038 int fixup_offset;
1039 int usize = 0;
1040 int orig_num;
1041
1042 orig_num = inst_num;
1043 val_indx = 0;
1044 cur_value = value[val_indx++];
1045 inst_arr = inst_4b;
1046 usize = ARRAY_SIZE(inst_4b);
1047 fixup_offset = inst_num;
1048 for (i = 0; i < usize; i++)
1049 micro_inst[inst_num++] = inst_arr[i];
1050 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], (addr));
1051 fixup_offset++;
1052 INSERT_IMMED_GPRA_CONST(micro_inst[fixup_offset], 0);
1053 fixup_offset++;
1054 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0));
1055 fixup_offset++;
1056 INSERT_IMMED_GPRB_CONST(micro_inst[fixup_offset], (cur_value >> 0x10));
1057
1058 return inst_num - orig_num;
1059}
1060
1061static int qat_hal_exec_micro_init_lm(struct icp_qat_fw_loader_handle *handle,
1062 unsigned char ae, unsigned char ctx,
1063 int *pfirst_exec, uint64_t *micro_inst,
1064 unsigned int inst_num)
1065{
1066 int stat = 0;
1067 unsigned int gpra0 = 0, gpra1 = 0, gpra2 = 0;
1068 unsigned int gprb0 = 0, gprb1 = 0;
1069
1070 if (*pfirst_exec) {
1071 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, &gpra0);
1072 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, &gpra1);
1073 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, &gpra2);
1074 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, &gprb0);
1075 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, &gprb1);
1076 *pfirst_exec = 0;
1077 }
1078 stat = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, inst_num, 1,
1079 inst_num * 0x5, NULL);
1080 if (stat != 0)
1081 return -EFAULT;
1082 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0, gpra0);
1083 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x1, gpra1);
1084 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPA_REL, 0x2, gpra2);
1085 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0, gprb0);
1086 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, 0x1, gprb1);
1087
1088 return 0;
1089}
1090
1091int qat_hal_batch_wr_lm(struct icp_qat_fw_loader_handle *handle,
1092 unsigned char ae,
1093 struct icp_qat_uof_batch_init *lm_init_header)
1094{
1095 struct icp_qat_uof_batch_init *plm_init;
1096 uint64_t *micro_inst_arry;
1097 int micro_inst_num;
1098 int alloc_inst_size;
1099 int first_exec = 1;
1100 int stat = 0;
1101
1102 plm_init = lm_init_header->next;
1103 alloc_inst_size = lm_init_header->size;
1104 if ((unsigned int)alloc_inst_size > handle->hal_handle->max_ustore)
1105 alloc_inst_size = handle->hal_handle->max_ustore;
1106 micro_inst_arry = kmalloc_array(alloc_inst_size, sizeof(uint64_t),
1107 GFP_KERNEL);
1108 if (!micro_inst_arry)
1109 return -ENOMEM;
1110 micro_inst_num = 0;
1111 while (plm_init) {
1112 unsigned int addr, *value, size;
1113
1114 ae = plm_init->ae;
1115 addr = plm_init->addr;
1116 value = plm_init->value;
1117 size = plm_init->size;
1118 micro_inst_num += qat_hal_concat_micro_code(micro_inst_arry,
1119 micro_inst_num,
1120 size, addr, value);
1121 plm_init = plm_init->next;
1122 }
1123
1124 if (micro_inst_arry && (micro_inst_num > 0)) {
1125 micro_inst_arry[micro_inst_num++] = 0x0E000010000ull;
1126 stat = qat_hal_exec_micro_init_lm(handle, ae, 0, &first_exec,
1127 micro_inst_arry,
1128 micro_inst_num);
1129 }
1130 kfree(micro_inst_arry);
1131 return stat;
1132}
1133
1134static int qat_hal_put_rel_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1135 unsigned char ae, unsigned char ctx,
1136 enum icp_qat_uof_regtype reg_type,
1137 unsigned short reg_num, unsigned int val)
1138{
1139 int status = 0;
1140 unsigned int reg_addr;
1141 unsigned int ctx_enables;
1142 unsigned short mask;
1143 unsigned short dr_offset = 0x10;
1144
1145 status = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1146 if (CE_INUSE_CONTEXTS & ctx_enables) {
1147 if (ctx & 0x1) {
1148 pr_err("QAT: bad 4-ctx mode,ctx=0x%x\n", ctx);
1149 return -EINVAL;
1150 }
1151 mask = 0x1f;
1152 dr_offset = 0x20;
1153 } else {
1154 mask = 0x0f;
1155 }
1156 if (reg_num & ~mask)
1157 return -EINVAL;
1158 reg_addr = reg_num + (ctx << 0x5);
1159 switch (reg_type) {
1160 case ICP_SR_RD_REL:
1161 case ICP_SR_REL:
1162 SET_AE_XFER(handle, ae, reg_addr, val);
1163 break;
1164 case ICP_DR_RD_REL:
1165 case ICP_DR_REL:
1166 SET_AE_XFER(handle, ae, (reg_addr + dr_offset), val);
1167 break;
1168 default:
1169 status = -EINVAL;
1170 break;
1171 }
1172 return status;
1173}
1174
1175static int qat_hal_put_rel_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1176 unsigned char ae, unsigned char ctx,
1177 enum icp_qat_uof_regtype reg_type,
1178 unsigned short reg_num, unsigned int data)
1179{
1180 unsigned int gprval, ctx_enables;
1181 unsigned short src_hiaddr, src_lowaddr, gpr_addr, xfr_addr, data16hi,
1182 data16low;
1183 unsigned short reg_mask;
1184 int status = 0;
1185 uint64_t micro_inst[] = {
1186 0x0F440000000ull,
1187 0x0F040000000ull,
1188 0x0A000000000ull,
1189 0x0F0000C0300ull,
1190 0x0E000010000ull
1191 };
1192 const int num_inst = ARRAY_SIZE(micro_inst), code_off = 1;
1193 const unsigned short gprnum = 0, dly = num_inst * 0x5;
1194
1195 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1196 if (CE_INUSE_CONTEXTS & ctx_enables) {
1197 if (ctx & 0x1) {
1198 pr_err("QAT: 4-ctx mode,ctx=0x%x\n", ctx);
1199 return -EINVAL;
1200 }
1201 reg_mask = (unsigned short)~0x1f;
1202 } else {
1203 reg_mask = (unsigned short)~0xf;
1204 }
1205 if (reg_num & reg_mask)
1206 return -EINVAL;
1207 xfr_addr = qat_hal_get_reg_addr(reg_type, reg_num);
1208 if (xfr_addr == BAD_REGADDR) {
1209 pr_err("QAT: bad xfrAddr=0x%x\n", xfr_addr);
1210 return -EINVAL;
1211 }
1212 qat_hal_rd_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, &gprval);
1213 gpr_addr = qat_hal_get_reg_addr(ICP_GPB_REL, gprnum);
1214 data16low = 0xffff & data;
1215 data16hi = 0xffff & (data >> 0x10);
1216 src_hiaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1217 (unsigned short)(0xff & data16hi));
1218 src_lowaddr = qat_hal_get_reg_addr(ICP_NO_DEST,
1219 (unsigned short)(0xff & data16low));
1220 micro_inst[0] = micro_inst[0x0] | ((data16hi >> 8) << 20) |
1221 ((gpr_addr & 0x3ff) << 10) | (src_hiaddr & 0x3ff);
1222 micro_inst[1] = micro_inst[0x1] | ((data16low >> 8) << 20) |
1223 ((gpr_addr & 0x3ff) << 10) | (src_lowaddr & 0x3ff);
1224 micro_inst[0x2] = micro_inst[0x2] |
1225 ((xfr_addr & 0x3ff) << 20) | ((gpr_addr & 0x3ff) << 10);
1226 status = qat_hal_exec_micro_inst(handle, ae, ctx, micro_inst, num_inst,
1227 code_off, dly, NULL);
1228 qat_hal_wr_rel_reg(handle, ae, ctx, ICP_GPB_REL, gprnum, gprval);
1229 return status;
1230}
1231
1232static int qat_hal_put_rel_nn(struct icp_qat_fw_loader_handle *handle,
1233 unsigned char ae, unsigned char ctx,
1234 unsigned short nn, unsigned int val)
1235{
1236 unsigned int ctx_enables;
1237 int stat = 0;
1238
1239 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1240 ctx_enables &= IGNORE_W1C_MASK;
1241 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables | CE_NN_MODE);
1242
1243 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, ICP_NEIGH_REL, nn, val);
1244 qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx_enables);
1245 return stat;
1246}
1247
1248static int qat_hal_convert_abs_to_rel(struct icp_qat_fw_loader_handle
1249 *handle, unsigned char ae,
1250 unsigned short absreg_num,
1251 unsigned short *relreg,
1252 unsigned char *ctx)
1253{
1254 unsigned int ctx_enables;
1255
1256 qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &ctx_enables);
1257 if (ctx_enables & CE_INUSE_CONTEXTS) {
1258
1259 *relreg = absreg_num & 0x1F;
1260 *ctx = (absreg_num >> 0x4) & 0x6;
1261 } else {
1262
1263 *relreg = absreg_num & 0x0F;
1264 *ctx = (absreg_num >> 0x4) & 0x7;
1265 }
1266 return 0;
1267}
1268
1269int qat_hal_init_gpr(struct icp_qat_fw_loader_handle *handle,
1270 unsigned char ae, unsigned char ctx_mask,
1271 enum icp_qat_uof_regtype reg_type,
1272 unsigned short reg_num, unsigned int regdata)
1273{
1274 int stat = 0;
1275 unsigned short reg;
1276 unsigned char ctx = 0;
1277 enum icp_qat_uof_regtype type;
1278
1279 if (reg_num >= ICP_QAT_UCLO_MAX_GPR_REG)
1280 return -EINVAL;
1281
1282 do {
1283 if (ctx_mask == 0) {
1284 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®,
1285 &ctx);
1286 type = reg_type - 1;
1287 } else {
1288 reg = reg_num;
1289 type = reg_type;
1290 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1291 continue;
1292 }
1293 stat = qat_hal_wr_rel_reg(handle, ae, ctx, type, reg, regdata);
1294 if (stat) {
1295 pr_err("QAT: write gpr fail\n");
1296 return -EINVAL;
1297 }
1298 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1299
1300 return 0;
1301}
1302
1303int qat_hal_init_wr_xfer(struct icp_qat_fw_loader_handle *handle,
1304 unsigned char ae, unsigned char ctx_mask,
1305 enum icp_qat_uof_regtype reg_type,
1306 unsigned short reg_num, unsigned int regdata)
1307{
1308 int stat = 0;
1309 unsigned short reg;
1310 unsigned char ctx = 0;
1311 enum icp_qat_uof_regtype type;
1312
1313 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1314 return -EINVAL;
1315
1316 do {
1317 if (ctx_mask == 0) {
1318 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®,
1319 &ctx);
1320 type = reg_type - 3;
1321 } else {
1322 reg = reg_num;
1323 type = reg_type;
1324 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1325 continue;
1326 }
1327 stat = qat_hal_put_rel_wr_xfer(handle, ae, ctx, type, reg,
1328 regdata);
1329 if (stat) {
1330 pr_err("QAT: write wr xfer fail\n");
1331 return -EINVAL;
1332 }
1333 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1334
1335 return 0;
1336}
1337
1338int qat_hal_init_rd_xfer(struct icp_qat_fw_loader_handle *handle,
1339 unsigned char ae, unsigned char ctx_mask,
1340 enum icp_qat_uof_regtype reg_type,
1341 unsigned short reg_num, unsigned int regdata)
1342{
1343 int stat = 0;
1344 unsigned short reg;
1345 unsigned char ctx = 0;
1346 enum icp_qat_uof_regtype type;
1347
1348 if (reg_num >= ICP_QAT_UCLO_MAX_XFER_REG)
1349 return -EINVAL;
1350
1351 do {
1352 if (ctx_mask == 0) {
1353 qat_hal_convert_abs_to_rel(handle, ae, reg_num, ®,
1354 &ctx);
1355 type = reg_type - 3;
1356 } else {
1357 reg = reg_num;
1358 type = reg_type;
1359 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1360 continue;
1361 }
1362 stat = qat_hal_put_rel_rd_xfer(handle, ae, ctx, type, reg,
1363 regdata);
1364 if (stat) {
1365 pr_err("QAT: write rd xfer fail\n");
1366 return -EINVAL;
1367 }
1368 } while (ctx_mask && (ctx++ < ICP_QAT_UCLO_MAX_CTX));
1369
1370 return 0;
1371}
1372
1373int qat_hal_init_nn(struct icp_qat_fw_loader_handle *handle,
1374 unsigned char ae, unsigned char ctx_mask,
1375 unsigned short reg_num, unsigned int regdata)
1376{
1377 int stat = 0;
1378 unsigned char ctx;
1379
1380 if (ctx_mask == 0)
1381 return -EINVAL;
1382
1383 for (ctx = 0; ctx < ICP_QAT_UCLO_MAX_CTX; ctx++) {
1384 if (!test_bit(ctx, (unsigned long *)&ctx_mask))
1385 continue;
1386 stat = qat_hal_put_rel_nn(handle, ae, ctx, reg_num, regdata);
1387 if (stat) {
1388 pr_err("QAT: write neigh error\n");
1389 return -EINVAL;
1390 }
1391 }
1392
1393 return 0;
1394}
1395
