blob: 1e30568d7c0412454a73a8e09971768f66b1baa0 [file] [log] [blame]
Tom Lendacky63b94502013-11-12 11:46:16 -06001/*
2 * AMD Cryptographic Coprocessor (CCP) driver
3 *
Gary R Hook553d2372016-03-01 13:49:04 -06004 * Copyright (C) 2013,2016 Advanced Micro Devices, Inc.
Tom Lendacky63b94502013-11-12 11:46:16 -06005 *
6 * Author: Tom Lendacky <thomas.lendacky@amd.com>
Gary R Hookfba88552016-07-26 19:09:20 -05007 * Author: Gary R Hook <gary.hook@amd.com>
Tom Lendacky63b94502013-11-12 11:46:16 -06008 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13
14#ifndef __CCP_DEV_H__
15#define __CCP_DEV_H__
16
17#include <linux/device.h>
18#include <linux/pci.h>
19#include <linux/spinlock.h>
20#include <linux/mutex.h>
21#include <linux/list.h>
22#include <linux/wait.h>
23#include <linux/dmapool.h>
24#include <linux/hw_random.h>
Tom Lendacky8db88462015-02-03 13:07:05 -060025#include <linux/bitops.h>
Gary R Hook58ea8ab2016-04-18 09:21:44 -050026#include <linux/interrupt.h>
27#include <linux/irqreturn.h>
28#include <linux/dmaengine.h>
Tom Lendacky63b94502013-11-12 11:46:16 -060029
Gary R Hook553d2372016-03-01 13:49:04 -060030#define MAX_CCP_NAME_LEN 16
Tom Lendacky63b94502013-11-12 11:46:16 -060031#define MAX_DMAPOOL_NAME_LEN 32
32
33#define MAX_HW_QUEUES 5
34#define MAX_CMD_QLEN 100
35
36#define TRNG_RETRIES 10
37
Tom Lendacky126ae9a2014-07-10 10:58:35 -050038#define CACHE_NONE 0x00
Tom Lendackyc4f4b322014-06-05 10:17:57 -050039#define CACHE_WB_NO_ALLOC 0xb7
40
Tom Lendacky63b94502013-11-12 11:46:16 -060041/****** Register Mappings ******/
42#define Q_MASK_REG 0x000
43#define TRNG_OUT_REG 0x00c
44#define IRQ_MASK_REG 0x040
45#define IRQ_STATUS_REG 0x200
46
47#define DEL_CMD_Q_JOB 0x124
48#define DEL_Q_ACTIVE 0x00000200
49#define DEL_Q_ID_SHIFT 6
50
51#define CMD_REQ0 0x180
52#define CMD_REQ_INCR 0x04
53
54#define CMD_Q_STATUS_BASE 0x210
55#define CMD_Q_INT_STATUS_BASE 0x214
56#define CMD_Q_STATUS_INCR 0x20
57
Tom Lendackyc4f4b322014-06-05 10:17:57 -050058#define CMD_Q_CACHE_BASE 0x228
Tom Lendacky63b94502013-11-12 11:46:16 -060059#define CMD_Q_CACHE_INC 0x20
60
Tom Lendacky8db88462015-02-03 13:07:05 -060061#define CMD_Q_ERROR(__qs) ((__qs) & 0x0000003f)
62#define CMD_Q_DEPTH(__qs) (((__qs) >> 12) & 0x0000000f)
Tom Lendacky63b94502013-11-12 11:46:16 -060063
64/****** REQ0 Related Values ******/
65#define REQ0_WAIT_FOR_WRITE 0x00000004
66#define REQ0_INT_ON_COMPLETE 0x00000002
67#define REQ0_STOP_ON_COMPLETE 0x00000001
68
69#define REQ0_CMD_Q_SHIFT 9
70#define REQ0_JOBID_SHIFT 3
71
72/****** REQ1 Related Values ******/
73#define REQ1_PROTECT_SHIFT 27
74#define REQ1_ENGINE_SHIFT 23
75#define REQ1_KEY_KSB_SHIFT 2
76
77#define REQ1_EOM 0x00000002
78#define REQ1_INIT 0x00000001
79
80/* AES Related Values */
81#define REQ1_AES_TYPE_SHIFT 21
82#define REQ1_AES_MODE_SHIFT 18
83#define REQ1_AES_ACTION_SHIFT 17
84#define REQ1_AES_CFB_SIZE_SHIFT 10
85
86/* XTS-AES Related Values */
87#define REQ1_XTS_AES_SIZE_SHIFT 10
88
89/* SHA Related Values */
90#define REQ1_SHA_TYPE_SHIFT 21
91
92/* RSA Related Values */
93#define REQ1_RSA_MOD_SIZE_SHIFT 10
94
95/* Pass-Through Related Values */
96#define REQ1_PT_BW_SHIFT 12
97#define REQ1_PT_BS_SHIFT 10
98
99/* ECC Related Values */
100#define REQ1_ECC_AFFINE_CONVERT 0x00200000
101#define REQ1_ECC_FUNCTION_SHIFT 18
102
103/****** REQ4 Related Values ******/
104#define REQ4_KSB_SHIFT 18
105#define REQ4_MEMTYPE_SHIFT 16
106
107/****** REQ6 Related Values ******/
108#define REQ6_MEMTYPE_SHIFT 16
109
Tom Lendacky63b94502013-11-12 11:46:16 -0600110/****** Key Storage Block ******/
111#define KSB_START 77
112#define KSB_END 127
113#define KSB_COUNT (KSB_END - KSB_START + 1)
Gary R Hook956ee212016-07-26 19:09:40 -0500114#define CCP_SB_BITS 256
Tom Lendacky63b94502013-11-12 11:46:16 -0600115
116#define CCP_JOBID_MASK 0x0000003f
117
118#define CCP_DMAPOOL_MAX_SIZE 64
Tom Lendacky8db88462015-02-03 13:07:05 -0600119#define CCP_DMAPOOL_ALIGN BIT(5)
Tom Lendacky63b94502013-11-12 11:46:16 -0600120
121#define CCP_REVERSE_BUF_SIZE 64
122
Gary R Hook956ee212016-07-26 19:09:40 -0500123#define CCP_AES_KEY_SB_COUNT 1
124#define CCP_AES_CTX_SB_COUNT 1
Tom Lendacky63b94502013-11-12 11:46:16 -0600125
Gary R Hook956ee212016-07-26 19:09:40 -0500126#define CCP_XTS_AES_KEY_SB_COUNT 1
127#define CCP_XTS_AES_CTX_SB_COUNT 1
Tom Lendacky63b94502013-11-12 11:46:16 -0600128
Gary R Hook956ee212016-07-26 19:09:40 -0500129#define CCP_SHA_SB_COUNT 1
Tom Lendacky63b94502013-11-12 11:46:16 -0600130
131#define CCP_RSA_MAX_WIDTH 4096
132
133#define CCP_PASSTHRU_BLOCKSIZE 256
134#define CCP_PASSTHRU_MASKSIZE 32
Gary R Hook956ee212016-07-26 19:09:40 -0500135#define CCP_PASSTHRU_SB_COUNT 1
Tom Lendacky63b94502013-11-12 11:46:16 -0600136
137#define CCP_ECC_MODULUS_BYTES 48 /* 384-bits */
138#define CCP_ECC_MAX_OPERANDS 6
139#define CCP_ECC_MAX_OUTPUTS 3
140#define CCP_ECC_SRC_BUF_SIZE 448
141#define CCP_ECC_DST_BUF_SIZE 192
142#define CCP_ECC_OPERAND_SIZE 64
143#define CCP_ECC_OUTPUT_SIZE 64
144#define CCP_ECC_RESULT_OFFSET 60
145#define CCP_ECC_RESULT_SUCCESS 0x0001
146
Gary R Hook956ee212016-07-26 19:09:40 -0500147#define CCP_SB_BYTES 32
148
Gary R Hookea0375a2016-03-01 13:49:25 -0600149struct ccp_op;
150
151/* Structure for computation functions that are device-specific */
152struct ccp_actions {
Gary R Hooka43eb982016-07-26 19:09:31 -0500153 int (*aes)(struct ccp_op *);
154 int (*xts_aes)(struct ccp_op *);
155 int (*sha)(struct ccp_op *);
156 int (*rsa)(struct ccp_op *);
157 int (*passthru)(struct ccp_op *);
158 int (*ecc)(struct ccp_op *);
Gary R Hookea0375a2016-03-01 13:49:25 -0600159 int (*init)(struct ccp_device *);
160 void (*destroy)(struct ccp_device *);
161 irqreturn_t (*irqhandler)(int, void *);
162};
163
Gary R Hookc7019c42016-03-01 13:49:15 -0600164/* Structure to hold CCP version-specific values */
165struct ccp_vdata {
166 unsigned int version;
Julia Lawallbc197b2a2016-05-01 13:52:55 +0200167 const struct ccp_actions *perform;
Gary R Hookfba88552016-07-26 19:09:20 -0500168 const unsigned int bar;
169 const unsigned int offset;
Gary R Hookc7019c42016-03-01 13:49:15 -0600170};
171
172extern struct ccp_vdata ccpv3;
173
Tom Lendacky63b94502013-11-12 11:46:16 -0600174struct ccp_device;
175struct ccp_cmd;
176
Gary R Hook58ea8ab2016-04-18 09:21:44 -0500177struct ccp_dma_cmd {
178 struct list_head entry;
179
180 struct ccp_cmd ccp_cmd;
181};
182
183struct ccp_dma_desc {
184 struct list_head entry;
185
186 struct ccp_device *ccp;
187
188 struct list_head pending;
189 struct list_head active;
190
191 enum dma_status status;
192 struct dma_async_tx_descriptor tx_desc;
193 size_t len;
194};
195
196struct ccp_dma_chan {
197 struct ccp_device *ccp;
198
199 spinlock_t lock;
200 struct list_head pending;
201 struct list_head active;
202 struct list_head complete;
203
204 struct tasklet_struct cleanup_tasklet;
205
206 enum dma_status status;
207 struct dma_chan dma_chan;
208};
209
Tom Lendacky63b94502013-11-12 11:46:16 -0600210struct ccp_cmd_queue {
211 struct ccp_device *ccp;
212
213 /* Queue identifier */
214 u32 id;
215
216 /* Queue dma pool */
217 struct dma_pool *dma_pool;
218
Gary R Hook956ee212016-07-26 19:09:40 -0500219 /* Per-queue reserved storage block(s) */
220 u32 sb_key;
221 u32 sb_ctx;
Tom Lendacky63b94502013-11-12 11:46:16 -0600222
223 /* Queue processing thread */
224 struct task_struct *kthread;
225 unsigned int active;
226 unsigned int suspended;
227
228 /* Number of free command slots available */
229 unsigned int free_slots;
230
231 /* Interrupt masks */
232 u32 int_ok;
233 u32 int_err;
234
235 /* Register addresses for queue */
236 void __iomem *reg_status;
237 void __iomem *reg_int_status;
238
239 /* Status values from job */
240 u32 int_status;
241 u32 q_status;
242 u32 q_int_status;
243 u32 cmd_error;
244
245 /* Interrupt wait queue */
246 wait_queue_head_t int_queue;
247 unsigned int int_rcvd;
248} ____cacheline_aligned;
249
250struct ccp_device {
Gary R Hook553d2372016-03-01 13:49:04 -0600251 struct list_head entry;
252
Gary R Hookc7019c42016-03-01 13:49:15 -0600253 struct ccp_vdata *vdata;
Gary R Hook553d2372016-03-01 13:49:04 -0600254 unsigned int ord;
255 char name[MAX_CCP_NAME_LEN];
256 char rngname[MAX_CCP_NAME_LEN];
257
Tom Lendacky63b94502013-11-12 11:46:16 -0600258 struct device *dev;
259
Gary R Hookfa242e82016-07-26 18:09:46 -0500260 /* Bus specific device information
Tom Lendacky63b94502013-11-12 11:46:16 -0600261 */
262 void *dev_specific;
263 int (*get_irq)(struct ccp_device *ccp);
264 void (*free_irq)(struct ccp_device *ccp);
Tom Lendacky3d775652014-06-05 10:17:45 -0500265 unsigned int irq;
Tom Lendacky63b94502013-11-12 11:46:16 -0600266
Gary R Hookfa242e82016-07-26 18:09:46 -0500267 /* I/O area used for device communication. The register mapping
Tom Lendacky63b94502013-11-12 11:46:16 -0600268 * starts at an offset into the mapped bar.
269 * The CMD_REQx registers and the Delete_Cmd_Queue_Job register
270 * need to be protected while a command queue thread is accessing
271 * them.
272 */
273 struct mutex req_mutex ____cacheline_aligned;
274 void __iomem *io_map;
275 void __iomem *io_regs;
276
Gary R Hookfa242e82016-07-26 18:09:46 -0500277 /* Master lists that all cmds are queued on. Because there can be
Tom Lendacky63b94502013-11-12 11:46:16 -0600278 * more than one CCP command queue that can process a cmd a separate
279 * backlog list is neeeded so that the backlog completion call
280 * completes before the cmd is available for execution.
281 */
282 spinlock_t cmd_lock ____cacheline_aligned;
283 unsigned int cmd_count;
284 struct list_head cmd;
285 struct list_head backlog;
286
Gary R Hookfa242e82016-07-26 18:09:46 -0500287 /* The command queues. These represent the queues available on the
Tom Lendacky63b94502013-11-12 11:46:16 -0600288 * CCP that are available for processing cmds
289 */
290 struct ccp_cmd_queue cmd_q[MAX_HW_QUEUES];
291 unsigned int cmd_q_count;
292
Gary R Hookfa242e82016-07-26 18:09:46 -0500293 /* Support for the CCP True RNG
Tom Lendacky63b94502013-11-12 11:46:16 -0600294 */
295 struct hwrng hwrng;
296 unsigned int hwrng_retries;
297
Gary R Hookfa242e82016-07-26 18:09:46 -0500298 /* Support for the CCP DMA capabilities
Gary R Hook58ea8ab2016-04-18 09:21:44 -0500299 */
300 struct dma_device dma_dev;
301 struct ccp_dma_chan *ccp_dma_chan;
302 struct kmem_cache *dma_cmd_cache;
303 struct kmem_cache *dma_desc_cache;
304
Gary R Hookfa242e82016-07-26 18:09:46 -0500305 /* A counter used to generate job-ids for cmds submitted to the CCP
Tom Lendacky63b94502013-11-12 11:46:16 -0600306 */
307 atomic_t current_id ____cacheline_aligned;
308
Gary R Hookfa242e82016-07-26 18:09:46 -0500309 /* The CCP uses key storage blocks (KSB) to maintain context for certain
Tom Lendacky63b94502013-11-12 11:46:16 -0600310 * operations. To prevent multiple cmds from using the same KSB range
311 * a command queue reserves a KSB range for the duration of the cmd.
312 * Each queue, will however, reserve 2 KSB blocks for operations that
313 * only require single KSB entries (eg. AES context/iv and key) in order
314 * to avoid allocation contention. This will reserve at most 10 KSB
315 * entries, leaving 40 KSB entries available for dynamic allocation.
316 */
Gary R Hook956ee212016-07-26 19:09:40 -0500317 struct mutex sb_mutex ____cacheline_aligned;
318 DECLARE_BITMAP(sb, KSB_COUNT);
319 wait_queue_head_t sb_queue;
320 unsigned int sb_avail;
321 unsigned int sb_count;
322 u32 sb_start;
Tom Lendacky63b94502013-11-12 11:46:16 -0600323
324 /* Suspend support */
325 unsigned int suspending;
326 wait_queue_head_t suspend_queue;
Tom Lendacky126ae9a2014-07-10 10:58:35 -0500327
328 /* DMA caching attribute support */
329 unsigned int axcache;
Tom Lendacky63b94502013-11-12 11:46:16 -0600330};
331
Gary R Hookea0375a2016-03-01 13:49:25 -0600332enum ccp_memtype {
333 CCP_MEMTYPE_SYSTEM = 0,
Gary R Hook956ee212016-07-26 19:09:40 -0500334 CCP_MEMTYPE_SB,
Gary R Hookea0375a2016-03-01 13:49:25 -0600335 CCP_MEMTYPE_LOCAL,
336 CCP_MEMTYPE__LAST,
337};
338
339struct ccp_dma_info {
340 dma_addr_t address;
341 unsigned int offset;
342 unsigned int length;
343 enum dma_data_direction dir;
344};
345
346struct ccp_dm_workarea {
347 struct device *dev;
348 struct dma_pool *dma_pool;
349 unsigned int length;
350
351 u8 *address;
352 struct ccp_dma_info dma;
353};
354
355struct ccp_sg_workarea {
356 struct scatterlist *sg;
357 int nents;
358
359 struct scatterlist *dma_sg;
360 struct device *dma_dev;
361 unsigned int dma_count;
362 enum dma_data_direction dma_dir;
363
364 unsigned int sg_used;
365
366 u64 bytes_left;
367};
368
369struct ccp_data {
370 struct ccp_sg_workarea sg_wa;
371 struct ccp_dm_workarea dm_wa;
372};
373
374struct ccp_mem {
375 enum ccp_memtype type;
376 union {
377 struct ccp_dma_info dma;
Gary R Hook956ee212016-07-26 19:09:40 -0500378 u32 sb;
Gary R Hookea0375a2016-03-01 13:49:25 -0600379 } u;
380};
381
382struct ccp_aes_op {
383 enum ccp_aes_type type;
384 enum ccp_aes_mode mode;
385 enum ccp_aes_action action;
386};
387
388struct ccp_xts_aes_op {
389 enum ccp_aes_action action;
390 enum ccp_xts_aes_unit_size unit_size;
391};
392
393struct ccp_sha_op {
394 enum ccp_sha_type type;
395 u64 msg_bits;
396};
397
398struct ccp_rsa_op {
399 u32 mod_size;
400 u32 input_len;
401};
402
403struct ccp_passthru_op {
404 enum ccp_passthru_bitwise bit_mod;
405 enum ccp_passthru_byteswap byte_swap;
406};
407
408struct ccp_ecc_op {
409 enum ccp_ecc_function function;
410};
411
412struct ccp_op {
413 struct ccp_cmd_queue *cmd_q;
414
415 u32 jobid;
416 u32 ioc;
417 u32 soc;
Gary R Hook956ee212016-07-26 19:09:40 -0500418 u32 sb_key;
419 u32 sb_ctx;
Gary R Hookea0375a2016-03-01 13:49:25 -0600420 u32 init;
421 u32 eom;
422
423 struct ccp_mem src;
424 struct ccp_mem dst;
425
426 union {
427 struct ccp_aes_op aes;
428 struct ccp_xts_aes_op xts;
429 struct ccp_sha_op sha;
430 struct ccp_rsa_op rsa;
431 struct ccp_passthru_op passthru;
432 struct ccp_ecc_op ecc;
433 } u;
434};
435
436static inline u32 ccp_addr_lo(struct ccp_dma_info *info)
437{
438 return lower_32_bits(info->address + info->offset);
439}
440
441static inline u32 ccp_addr_hi(struct ccp_dma_info *info)
442{
443 return upper_32_bits(info->address + info->offset) & 0x0000ffff;
444}
445
Tom Lendacky63b94502013-11-12 11:46:16 -0600446int ccp_pci_init(void);
447void ccp_pci_exit(void);
448
Tom Lendackyc4f4b322014-06-05 10:17:57 -0500449int ccp_platform_init(void);
450void ccp_platform_exit(void);
451
Gary R Hookea0375a2016-03-01 13:49:25 -0600452void ccp_add_device(struct ccp_device *ccp);
453void ccp_del_device(struct ccp_device *ccp);
Tom Lendacky63b94502013-11-12 11:46:16 -0600454
Gary R Hookea0375a2016-03-01 13:49:25 -0600455struct ccp_device *ccp_alloc_struct(struct device *dev);
456bool ccp_queues_suspended(struct ccp_device *ccp);
457int ccp_cmd_queue_thread(void *data);
Tom Lendacky63b94502013-11-12 11:46:16 -0600458
459int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd);
460
Gary R Hook58ea8ab2016-04-18 09:21:44 -0500461int ccp_dmaengine_register(struct ccp_device *ccp);
462void ccp_dmaengine_unregister(struct ccp_device *ccp);
463
Tom Lendacky63b94502013-11-12 11:46:16 -0600464#endif