Merge tag 's390-6.12-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux

Pull s390 updates from Vasily Gorbik:

 - Optimize ftrace and kprobes code patching and avoid stop machine for
   kprobes if sequential instruction fetching facility is available

 - Add hiperdispatch feature to dynamically adjust CPU capacity in
   vertical polarization to improve scheduling efficiency and overall
   performance. Also add infrastructure for handling warning track
   interrupts (WTI), allowing for graceful CPU preemption

 - Rework crypto code pkey module and split it into separate,
   independent modules for sysfs, PCKMO, CCA, and EP11, allowing modules
   to load only when the relevant hardware is available

 - Add hardware acceleration for HMAC modes and the full AES-XTS cipher,
   utilizing message-security assist extensions (MSA) 10 and 11. It
   introduces new shash implementations for HMAC-SHA224/256/384/512 and
   registers the hardware-accelerated AES-XTS cipher as the preferred
   option. Also add clear key token support

 - Add MSA 10 and 11 processor activity instrumentation counters to perf
   and update PAI Extension 1 NNPA counters

 - Cleanup cpu sampling facility code and rework debug/WARN_ON_ONCE
   statements

 - Add support for SHA3 performance enhancements introduced with MSA 12

 - Add support for the query authentication information feature of MSA
   13 and introduce the KDSA CPACF instruction. Provide query and query
   authentication information in sysfs, enabling tools like cpacfinfo to
   present this data in a human-readable form

 - Update kernel disassembler instructions

 - Always enable EXPOLINE_EXTERN if supported by the compiler to ensure
   kpatch compatibility

 - Add missing warning handling and relocated lowcore support to the
   early program check handler

 - Optimize ftrace_return_address() and avoid calling unwinder

 - Make modules use kernel ftrace trampolines

 - Strip relocs from the final vmlinux ELF file to make it roughly 2
   times smaller

 - Dump register contents and call trace for early crashes to the
   console

 - Generate ptdump address marker array dynamically

 - Fix rcu_sched stalls that might occur when adding or removing large
   amounts of pages at once to or from the CMM balloon

 - Fix deadlock caused by recursive lock of the AP bus scan mutex

 - Unify sync and async register save areas in entry code

 - Cleanup debug prints in crypto code

 - Various cleanup and sanitizing patches for the decompressor

 - Various small ftrace cleanups

* tag 's390-6.12-1' of git://git.kernel.org/pub/scm/linux/kernel/git/s390/linux: (84 commits)
  s390/crypto: Display Query and Query Authentication Information in sysfs
  s390/crypto: Add Support for Query Authentication Information
  s390/crypto: Rework RRE and RRF CPACF inline functions
  s390/crypto: Add KDSA CPACF Instruction
  s390/disassembler: Remove duplicate instruction format RSY_RDRU
  s390/boot: Move boot_printk() code to own file
  s390/boot: Use boot_printk() instead of sclp_early_printk()
  s390/boot: Rename decompressor_printk() to boot_printk()
  s390/boot: Compile all files with the same march flag
  s390: Use MARCH_HAS_*_FEATURES defines
  s390: Provide MARCH_HAS_*_FEATURES defines
  s390/facility: Disable compile time optimization for decompressor code
  s390/boot: Increase minimum architecture to z10
  s390/als: Remove obsolete comment
  s390/sha3: Fix SHA3 selftests failures
  s390/pkey: Add AES xts and HMAC clear key token support
  s390/cpacf: Add MSA 10 and 11 new PCKMO functions
  s390/mm: Add cond_resched() to cmm_alloc/free_pages()
  s390/pai_ext: Update PAI extension 1 counters
  s390/pai_crypto: Add support for MSA 10 and 11 pai counters
  ...

18 files changed, 3652 insertions, 2202 deletions

diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c
index 07df04af82f2..29156455970e 100644
--- a/drivers/s390/char/sclp_early.c
+++ b/drivers/s390/char/sclp_early.c
@@ -44,6 +44,7 @@ static void __init sclp_early_facilities_detect(void)
 	sclp.has_ibs = !!(sccb->fac117 & 0x20);
 	sclp.has_gisaf = !!(sccb->fac118 & 0x08);
 	sclp.has_hvs = !!(sccb->fac119 & 0x80);
+	sclp.has_wti = !!(sccb->fac119 & 0x40);
 	sclp.has_kss = !!(sccb->fac98 & 0x01);
 	sclp.has_aisii = !!(sccb->fac118 & 0x40);
 	sclp.has_aeni = !!(sccb->fac118 & 0x20);
diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile
index bd94811fd9f1..c88b6e071847 100644
--- a/drivers/s390/crypto/Makefile
+++ b/drivers/s390/crypto/Makefile
@@ -13,10 +13,22 @@ obj-$(CONFIG_ZCRYPT) += zcrypt.o
 # adapter drivers depend on ap.o and zcrypt.o
 obj-$(CONFIG_ZCRYPT) += zcrypt_cex4.o
 
-# pkey kernel module
-pkey-objs := pkey_api.o
+# pkey base and api module
+pkey-objs := pkey_base.o pkey_api.o pkey_sysfs.o
 obj-$(CONFIG_PKEY) += pkey.o
 
+# pkey cca handler module
+pkey-cca-objs := pkey_cca.o
+obj-$(CONFIG_PKEY_CCA) += pkey-cca.o
+
+# pkey ep11 handler module
+pkey-ep11-objs := pkey_ep11.o
+obj-$(CONFIG_PKEY_EP11) += pkey-ep11.o
+
+# pkey pckmo handler module
+pkey-pckmo-objs := pkey_pckmo.o
+obj-$(CONFIG_PKEY_PCKMO) += pkey-pckmo.o
+
 # adjunct processor matrix
 vfio_ap-objs := vfio_ap_drv.o vfio_ap_ops.o
 obj-$(CONFIG_VFIO_AP) += vfio_ap.o
diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c
index f9f682f19415..60cea6c24349 100644
--- a/drivers/s390/crypto/ap_bus.c
+++ b/drivers/s390/crypto/ap_bus.c
@@ -107,6 +107,7 @@ debug_info_t *ap_dbf_info;
 static bool ap_scan_bus(void);
 static bool ap_scan_bus_result; /* result of last ap_scan_bus() */
 static DEFINE_MUTEX(ap_scan_bus_mutex); /* mutex ap_scan_bus() invocations */
+static struct task_struct *ap_scan_bus_task; /* thread holding the scan mutex */
 static atomic64_t ap_scan_bus_count; /* counter ap_scan_bus() invocations */
 static int ap_scan_bus_time = AP_CONFIG_TIME;
 static struct timer_list ap_scan_bus_timer;
@@ -733,7 +734,7 @@ static void ap_check_bindings_complete(void)
 			if (!completion_done(&ap_apqn_bindings_complete)) {
 				complete_all(&ap_apqn_bindings_complete);
 				ap_send_bindings_complete_uevent();
-				pr_debug("%s all apqn bindings complete\n", __func__);
+				pr_debug("all apqn bindings complete\n");
 			}
 		}
 	}
@@ -768,7 +769,7 @@ int ap_wait_apqn_bindings_complete(unsigned long timeout)
 	else if (l == 0 && timeout)
 		rc = -ETIME;
 
-	pr_debug("%s rc=%d\n", __func__, rc);
+	pr_debug("rc=%d\n", rc);
 	return rc;
 }
 EXPORT_SYMBOL(ap_wait_apqn_bindings_complete);
@@ -795,8 +796,7 @@ static int __ap_revise_reserved(struct device *dev, void *dummy)
 		drvres = to_ap_drv(dev->driver)->flags
 			& AP_DRIVER_FLAG_DEFAULT;
 		if (!!devres != !!drvres) {
-			pr_debug("%s reprobing queue=%02x.%04x\n",
-				 __func__, card, queue);
+			pr_debug("reprobing queue=%02x.%04x\n", card, queue);
 			rc = device_reprobe(dev);
 			if (rc)
 				AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n",
@@ -1000,17 +1000,31 @@ bool ap_bus_force_rescan(void)
 	unsigned long scan_counter = atomic64_read(&ap_scan_bus_count);
 	bool rc = false;
 
-	pr_debug(">%s scan counter=%lu\n", __func__, scan_counter);
+	pr_debug("> scan counter=%lu\n", scan_counter);
 
 	/* Only trigger AP bus scans after the initial scan is done */
 	if (scan_counter <= 0)
 		goto out;
 
+	/*
+	 * There is one unlikely but nevertheless valid scenario where the
+	 * thread holding the mutex may try to send some crypto load but
+	 * all cards are offline so a rescan is triggered which causes
+	 * a recursive call of ap_bus_force_rescan(). A simple return if
+	 * the mutex is already locked by this thread solves this.
+	 */
+	if (mutex_is_locked(&ap_scan_bus_mutex)) {
+		if (ap_scan_bus_task == current)
+			goto out;
+	}
+
 	/* Try to acquire the AP scan bus mutex */
 	if (mutex_trylock(&ap_scan_bus_mutex)) {
 		/* mutex acquired, run the AP bus scan */
+		ap_scan_bus_task = current;
 		ap_scan_bus_result = ap_scan_bus();
 		rc = ap_scan_bus_result;
+		ap_scan_bus_task = NULL;
 		mutex_unlock(&ap_scan_bus_mutex);
 		goto out;
 	}
@@ -1029,7 +1043,7 @@ bool ap_bus_force_rescan(void)
 	mutex_unlock(&ap_scan_bus_mutex);
 
 out:
-	pr_debug("%s rc=%d\n", __func__, rc);
+	pr_debug("rc=%d\n", rc);
 	return rc;
 }
 EXPORT_SYMBOL(ap_bus_force_rescan);
@@ -1043,7 +1057,7 @@ static int ap_bus_cfg_chg(struct notifier_block *nb,
 	if (action != CHSC_NOTIFY_AP_CFG)
 		return NOTIFY_DONE;
 
-	pr_debug("%s config change, forcing bus rescan\n", __func__);
+	pr_debug("config change, forcing bus rescan\n");
 
 	ap_bus_force_rescan();
 
@@ -1900,8 +1914,8 @@ static inline void ap_scan_domains(struct ap_card *ac)
 				aq->last_err_rc = AP_RESPONSE_CHECKSTOPPED;
 			}
 			spin_unlock_bh(&aq->lock);
-			pr_debug("%s(%d,%d) queue dev checkstop on\n",
-				 __func__, ac->id, dom);
+			pr_debug("(%d,%d) queue dev checkstop on\n",
+				 ac->id, dom);
 			/* 'receive' pending messages with -EAGAIN */
 			ap_flush_queue(aq);
 			goto put_dev_and_continue;
@@ -1911,8 +1925,8 @@ static inline void ap_scan_domains(struct ap_card *ac)
 			if (aq->dev_state > AP_DEV_STATE_UNINITIATED)
 				_ap_queue_init_state(aq);
 			spin_unlock_bh(&aq->lock);
-			pr_debug("%s(%d,%d) queue dev checkstop off\n",
-				 __func__, ac->id, dom);
+			pr_debug("(%d,%d) queue dev checkstop off\n",
+				 ac->id, dom);
 			goto put_dev_and_continue;
 		}
 		/* config state change */
@@ -1924,8 +1938,8 @@ static inline void ap_scan_domains(struct ap_card *ac)
 				aq->last_err_rc = AP_RESPONSE_DECONFIGURED;
 			}
 			spin_unlock_bh(&aq->lock);
-			pr_debug("%s(%d,%d) queue dev config off\n",
-				 __func__, ac->id, dom);
+			pr_debug("(%d,%d) queue dev config off\n",
+				 ac->id, dom);
 			ap_send_config_uevent(&aq->ap_dev, aq->config);
 			/* 'receive' pending messages with -EAGAIN */
 			ap_flush_queue(aq);
@@ -1936,8 +1950,8 @@ static inline void ap_scan_domains(struct ap_card *ac)
 			if (aq->dev_state > AP_DEV_STATE_UNINITIATED)
 				_ap_queue_init_state(aq);
 			spin_unlock_bh(&aq->lock);
-			pr_debug("%s(%d,%d) queue dev config on\n",
-				 __func__, ac->id, dom);
+			pr_debug("(%d,%d) queue dev config on\n",
+				 ac->id, dom);
 			ap_send_config_uevent(&aq->ap_dev, aq->config);
 			goto put_dev_and_continue;
 		}
@@ -2009,8 +2023,8 @@ static inline void ap_scan_adapter(int ap)
 			ap_scan_rm_card_dev_and_queue_devs(ac);
 			put_device(dev);
 		} else {
-			pr_debug("%s(%d) no type info (no APQN found), ignored\n",
-				 __func__, ap);
+			pr_debug("(%d) no type info (no APQN found), ignored\n",
+				 ap);
 		}
 		return;
 	}
@@ -2022,8 +2036,7 @@ static inline void ap_scan_adapter(int ap)
 			ap_scan_rm_card_dev_and_queue_devs(ac);
 			put_device(dev);
 		} else {
-			pr_debug("%s(%d) no valid type (0) info, ignored\n",
-				 __func__, ap);
+			pr_debug("(%d) no valid type (0) info, ignored\n", ap);
 		}
 		return;
 	}
@@ -2202,7 +2215,7 @@ static bool ap_scan_bus(void)
 	bool config_changed;
 	int ap;
 
-	pr_debug(">%s\n", __func__);
+	pr_debug(">\n");
 
 	/* (re-)fetch configuration via QCI */
 	config_changed = ap_get_configuration();
@@ -2243,7 +2256,7 @@ static bool ap_scan_bus(void)
 	}
 
 	if (atomic64_inc_return(&ap_scan_bus_count) == 1) {
-		pr_debug("%s init scan complete\n", __func__);
+		pr_debug("init scan complete\n");
 		ap_send_init_scan_done_uevent();
 	}
 
@@ -2251,7 +2264,7 @@ static bool ap_scan_bus(void)
 
 	mod_timer(&ap_scan_bus_timer, jiffies + ap_scan_bus_time * HZ);
 
-	pr_debug("<%s config_changed=%d\n", __func__, config_changed);
+	pr_debug("< config_changed=%d\n", config_changed);
 
 	return config_changed;
 }
@@ -2284,7 +2297,9 @@ static void ap_scan_bus_wq_callback(struct work_struct *unused)
 	 * system_long_wq which invokes this function here again.
 	 */
 	if (mutex_trylock(&ap_scan_bus_mutex)) {
+		ap_scan_bus_task = current;
 		ap_scan_bus_result = ap_scan_bus();
+		ap_scan_bus_task = NULL;
 		mutex_unlock(&ap_scan_bus_mutex);
 	}
 }
diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c
index 1f647ffd6f4d..8c878c5aa31f 100644
--- a/drivers/s390/crypto/ap_queue.c
+++ b/drivers/s390/crypto/ap_queue.c
@@ -171,8 +171,8 @@ static struct ap_queue_status ap_sm_recv(struct ap_queue *aq)
 		aq->queue_count = 0;
 		list_splice_init(&aq->pendingq, &aq->requestq);
 		aq->requestq_count += aq->pendingq_count;
-		pr_debug("%s queue 0x%02x.%04x rescheduled %d reqs (new req %d)\n",
-			 __func__, AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid),
+		pr_debug("queue 0x%02x.%04x rescheduled %d reqs (new req %d)\n",
+			 AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid),
 			 aq->pendingq_count, aq->requestq_count);
 		aq->pendingq_count = 0;
 		break;
@@ -453,8 +453,8 @@ static enum ap_sm_wait ap_sm_assoc_wait(struct ap_queue *aq)
 	case AP_BS_Q_USABLE:
 		/* association is through */
 		aq->sm_state = AP_SM_STATE_IDLE;
-		pr_debug("%s queue 0x%02x.%04x associated with %u\n",
-			 __func__, AP_QID_CARD(aq->qid),
+		pr_debug("queue 0x%02x.%04x associated with %u\n",
+			 AP_QID_CARD(aq->qid),
 			 AP_QID_QUEUE(aq->qid), aq->assoc_idx);
 		return AP_SM_WAIT_NONE;
 	case AP_BS_Q_USABLE_NO_SECURE_KEY:
@@ -697,8 +697,8 @@ static ssize_t ap_functions_show(struct device *dev,
 
 	status = ap_test_queue(aq->qid, 1, &hwinfo);
 	if (status.response_code > AP_RESPONSE_BUSY) {
-		pr_debug("%s RC 0x%02x on tapq(0x%02x.%04x)\n",
-			 __func__, status.response_code,
+		pr_debug("RC 0x%02x on tapq(0x%02x.%04x)\n",
+			 status.response_code,
 			 AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
 		return -EIO;
 	}
@@ -853,8 +853,8 @@ static ssize_t se_bind_show(struct device *dev,
 
 	status = ap_test_queue(aq->qid, 1, &hwinfo);
 	if (status.response_code > AP_RESPONSE_BUSY) {
-		pr_debug("%s RC 0x%02x on tapq(0x%02x.%04x)\n",
-			 __func__, status.response_code,
+		pr_debug("RC 0x%02x on tapq(0x%02x.%04x)\n",
+			 status.response_code,
 			 AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
 		return -EIO;
 	}
@@ -981,8 +981,8 @@ static ssize_t se_associate_show(struct device *dev,
 
 	status = ap_test_queue(aq->qid, 1, &hwinfo);
 	if (status.response_code > AP_RESPONSE_BUSY) {
-		pr_debug("%s RC 0x%02x on tapq(0x%02x.%04x)\n",
-			 __func__, status.response_code,
+		pr_debug("RC 0x%02x on tapq(0x%02x.%04x)\n",
+			 status.response_code,
 			 AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid));
 		return -EIO;
 	}
diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c
index ffc0b5db55c2..c20251e00cf9 100644
--- a/drivers/s390/crypto/pkey_api.c
+++ b/drivers/s390/crypto/pkey_api.c
@@ -10,1338 +10,698 @@
 #define KMSG_COMPONENT "pkey"
 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
 
-#include <linux/fs.h>
 #include <linux/init.h>
 #include <linux/miscdevice.h>
-#include <linux/module.h>
 #include <linux/slab.h>
-#include <linux/kallsyms.h>
-#include <linux/debugfs.h>
-#include <linux/random.h>
-#include <linux/cpufeature.h>
-#include <asm/zcrypt.h>
-#include <asm/cpacf.h>
-#include <asm/pkey.h>
-#include <crypto/aes.h>
 
 #include "zcrypt_api.h"
 #include "zcrypt_ccamisc.h"
-#include "zcrypt_ep11misc.h"
 
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("IBM Corporation");
-MODULE_DESCRIPTION("s390 protected key interface");
-
-#define KEYBLOBBUFSIZE 8192	/* key buffer size used for internal processing */
-#define MINKEYBLOBBUFSIZE (sizeof(struct keytoken_header))
-#define PROTKEYBLOBBUFSIZE 256	/* protected key buffer size used internal */
-#define MAXAPQNSINLIST 64	/* max 64 apqns within a apqn list */
-#define AES_WK_VP_SIZE 32	/* Size of WK VP block appended to a prot key */
+#include "pkey_base.h"
 
 /*
- * debug feature data and functions
+ * Helper functions
  */
-
-static debug_info_t *pkey_dbf_info;
-
-#define PKEY_DBF_INFO(...) debug_sprintf_event(pkey_dbf_info, 5, ##__VA_ARGS__)
-#define PKEY_DBF_WARN(...) debug_sprintf_event(pkey_dbf_info, 4, ##__VA_ARGS__)
-#define PKEY_DBF_ERR(...)  debug_sprintf_event(pkey_dbf_info, 3, ##__VA_ARGS__)
-
-static void __init pkey_debug_init(void)
+static int key2protkey(const struct pkey_apqn *apqns, size_t nr_apqns,
+		       const u8 *key, size_t keylen,
+		       u8 *protkey, u32 *protkeylen, u32 *protkeytype)
 {
-	/* 5 arguments per dbf entry (including the format string ptr) */
-	pkey_dbf_info = debug_register("pkey", 1, 1, 5 * sizeof(long));
-	debug_register_view(pkey_dbf_info, &debug_sprintf_view);
-	debug_set_level(pkey_dbf_info, 3);
-}
-
-static void __exit pkey_debug_exit(void)
-{
-	debug_unregister(pkey_dbf_info);
-}
+	int rc;
 
-/* inside view of a protected key token (only type 0x00 version 0x01) */
-struct protaeskeytoken {
-	u8  type;     /* 0x00 for PAES specific key tokens */
-	u8  res0[3];
-	u8  version;  /* should be 0x01 for protected AES key token */
-	u8  res1[3];
-	u32 keytype;  /* key type, one of the PKEY_KEYTYPE values */
-	u32 len;      /* bytes actually stored in protkey[] */
-	u8  protkey[MAXPROTKEYSIZE]; /* the protected key blob */
-} __packed;
-
-/* inside view of a clear key token (type 0x00 version 0x02) */
-struct clearkeytoken {
-	u8  type;	/* 0x00 for PAES specific key tokens */
-	u8  res0[3];
-	u8  version;	/* 0x02 for clear key token */
-	u8  res1[3];
-	u32 keytype;	/* key type, one of the PKEY_KEYTYPE_* values */
-	u32 len;	/* bytes actually stored in clearkey[] */
-	u8  clearkey[]; /* clear key value */
-} __packed;
-
-/* helper function which translates the PKEY_KEYTYPE_AES_* to their keysize */
-static inline u32 pkey_keytype_aes_to_size(u32 keytype)
-{
-	switch (keytype) {
-	case PKEY_KEYTYPE_AES_128:
-		return 16;
-	case PKEY_KEYTYPE_AES_192:
-		return 24;
-	case PKEY_KEYTYPE_AES_256:
-		return 32;
-	default:
-		return 0;
+	/* try the direct way */
+	rc = pkey_handler_key_to_protkey(apqns, nr_apqns,
+					 key, keylen,
+					 protkey, protkeylen,
+					 protkeytype);
+
+	/* if this did not work, try the slowpath way */
+	if (rc == -ENODEV) {
+		rc = pkey_handler_slowpath_key_to_protkey(apqns, nr_apqns,
+							  key, keylen,
+							  protkey, protkeylen,
+							  protkeytype);
+		if (rc)
+			rc = -ENODEV;
 	}
+
+	pr_debug("rc=%d\n", rc);
+	return rc;
 }
 
 /*
- * Create a protected key from a clear key value via PCKMO instruction.
+ * In-Kernel function: Transform a key blob (of any type) into a protected key
  */
-static int pkey_clr2protkey(u32 keytype, const u8 *clrkey,
-			    u8 *protkey, u32 *protkeylen, u32 *protkeytype)
+int pkey_key2protkey(const u8 *key, u32 keylen,
+		     u8 *protkey, u32 *protkeylen, u32 *protkeytype)
 {
-	/* mask of available pckmo subfunctions */
-	static cpacf_mask_t pckmo_functions;
-
-	u8 paramblock[112];
-	u32 pkeytype;
-	int keysize;
-	long fc;
-
-	switch (keytype) {
-	case PKEY_KEYTYPE_AES_128:
-		/* 16 byte key, 32 byte aes wkvp, total 48 bytes */
-		keysize = 16;
-		pkeytype = keytype;
-		fc = CPACF_PCKMO_ENC_AES_128_KEY;
-		break;
-	case PKEY_KEYTYPE_AES_192:
-		/* 24 byte key, 32 byte aes wkvp, total 56 bytes */
-		keysize = 24;
-		pkeytype = keytype;
-		fc = CPACF_PCKMO_ENC_AES_192_KEY;
-		break;
-	case PKEY_KEYTYPE_AES_256:
-		/* 32 byte key, 32 byte aes wkvp, total 64 bytes */
-		keysize = 32;
-		pkeytype = keytype;
-		fc = CPACF_PCKMO_ENC_AES_256_KEY;
-		break;
-	case PKEY_KEYTYPE_ECC_P256:
-		/* 32 byte key, 32 byte aes wkvp, total 64 bytes */
-		keysize = 32;
-		pkeytype = PKEY_KEYTYPE_ECC;
-		fc = CPACF_PCKMO_ENC_ECC_P256_KEY;
-		break;
-	case PKEY_KEYTYPE_ECC_P384:
-		/* 48 byte key, 32 byte aes wkvp, total 80 bytes */
-		keysize = 48;
-		pkeytype = PKEY_KEYTYPE_ECC;
-		fc = CPACF_PCKMO_ENC_ECC_P384_KEY;
-		break;
-	case PKEY_KEYTYPE_ECC_P521:
-		/* 80 byte key, 32 byte aes wkvp, total 112 bytes */
-		keysize = 80;
-		pkeytype = PKEY_KEYTYPE_ECC;
-		fc = CPACF_PCKMO_ENC_ECC_P521_KEY;
-		break;
-	case PKEY_KEYTYPE_ECC_ED25519:
-		/* 32 byte key, 32 byte aes wkvp, total 64 bytes */
-		keysize = 32;
-		pkeytype = PKEY_KEYTYPE_ECC;
-		fc = CPACF_PCKMO_ENC_ECC_ED25519_KEY;
-		break;
-	case PKEY_KEYTYPE_ECC_ED448:
-		/* 64 byte key, 32 byte aes wkvp, total 96 bytes */
-		keysize = 64;
-		pkeytype = PKEY_KEYTYPE_ECC;
-		fc = CPACF_PCKMO_ENC_ECC_ED448_KEY;
-		break;
-	default:
-		PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n",
-			     __func__, keytype);
-		return -EINVAL;
-	}
-
-	if (*protkeylen < keysize + AES_WK_VP_SIZE) {
-		PKEY_DBF_ERR("%s prot key buffer size too small: %u < %d\n",
-			     __func__, *protkeylen, keysize + AES_WK_VP_SIZE);
-		return -EINVAL;
-	}
+	int rc;
 
-	/* Did we already check for PCKMO ? */
-	if (!pckmo_functions.bytes[0]) {
-		/* no, so check now */
-		if (!cpacf_query(CPACF_PCKMO, &pckmo_functions))
-			return -ENODEV;
-	}
-	/* check for the pckmo subfunction we need now */
-	if (!cpacf_test_func(&pckmo_functions, fc)) {
-		PKEY_DBF_ERR("%s pckmo functions not available\n", __func__);
-		return -ENODEV;
+	rc = key2protkey(NULL, 0, key, keylen,
+			 protkey, protkeylen, protkeytype);
+	if (rc == -ENODEV) {
+		pkey_handler_request_modules();
+		rc = key2protkey(NULL, 0, key, keylen,
+				 protkey, protkeylen, protkeytype);
 	}
 
-	/* prepare param block */
-	memset(paramblock, 0, sizeof(paramblock));
-	memcpy(paramblock, clrkey, keysize);
-
-	/* call the pckmo instruction */
-	cpacf_pckmo(fc, paramblock);
-
-	/* copy created protected key to key buffer including the wkvp block */
-	*protkeylen = keysize + AES_WK_VP_SIZE;
-	memcpy(protkey, paramblock, *protkeylen);
-	*protkeytype = pkeytype;
-
-	return 0;
+	return rc;
 }
+EXPORT_SYMBOL(pkey_key2protkey);
 
 /*
- * Find card and transform secure key into protected key.
+ * Ioctl functions
  */
-static int pkey_skey2pkey(const u8 *key, u8 *protkey,
-			  u32 *protkeylen, u32 *protkeytype)
-{
-	struct keytoken_header *hdr = (struct keytoken_header *)key;
-	u16 cardnr, domain;
-	int rc, verify;
-
-	zcrypt_wait_api_operational();
-
-	/*
-	 * The cca_xxx2protkey call may fail when a card has been
-	 * addressed where the master key was changed after last fetch
-	 * of the mkvp into the cache. Try 3 times: First without verify
-	 * then with verify and last round with verify and old master
-	 * key verification pattern match not ignored.
-	 */
-	for (verify = 0; verify < 3; verify++) {
-		rc = cca_findcard(key, &cardnr, &domain, verify);
-		if (rc < 0)
-			continue;
-		if (rc > 0 && verify < 2)
-			continue;
-		switch (hdr->version) {
-		case TOKVER_CCA_AES:
-			rc = cca_sec2protkey(cardnr, domain, key,
-					     protkey, protkeylen, protkeytype);
-			break;
-		case TOKVER_CCA_VLSC:
-			rc = cca_cipher2protkey(cardnr, domain, key,
-						protkey, protkeylen,
-						protkeytype);
-			break;
-		default:
-			return -EINVAL;
-		}
-		if (rc == 0)
-			break;
-	}
 
-	if (rc)
-		pr_debug("%s failed rc=%d\n", __func__, rc);
+static void *_copy_key_from_user(void __user *ukey, size_t keylen)
+{
+	if (!ukey || keylen < MINKEYBLOBBUFSIZE || keylen > KEYBLOBBUFSIZE)
+		return ERR_PTR(-EINVAL);
 
-	return rc;
+	return memdup_user(ukey, keylen);
 }
 
-/*
- * Construct EP11 key with given clear key value.
- */
-static int pkey_clr2ep11key(const u8 *clrkey, size_t clrkeylen,
-			    u8 *keybuf, size_t *keybuflen)
+static void *_copy_apqns_from_user(void __user *uapqns, size_t nr_apqns)
 {
-	u32 nr_apqns, *apqns = NULL;
-	u16 card, dom;
-	int i, rc;
-
-	zcrypt_wait_api_operational();
-
-	/* build a list of apqns suitable for ep11 keys with cpacf support */
-	rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF,
-			    ZCRYPT_CEX7,
-			    ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4,
-			    NULL);
-	if (rc)
-		goto out;
-
-	/* go through the list of apqns and try to bild an ep11 key */
-	for (rc = -ENODEV, i = 0; i < nr_apqns; i++) {
-		card = apqns[i] >> 16;
-		dom = apqns[i] & 0xFFFF;
-		rc = ep11_clr2keyblob(card, dom, clrkeylen * 8,
-				      0, clrkey, keybuf, keybuflen,
-				      PKEY_TYPE_EP11);
-		if (rc == 0)
-			break;
-	}
+	if (!uapqns || nr_apqns == 0)
+		return NULL;
 
-out:
-	kfree(apqns);
-	if (rc)
-		pr_debug("%s failed rc=%d\n", __func__, rc);
-	return rc;
+	return memdup_user(uapqns, nr_apqns * sizeof(struct pkey_apqn));
 }
 
-/*
- * Find card and transform EP11 secure key into protected key.
- */
-static int pkey_ep11key2pkey(const u8 *key, size_t keylen,
-			     u8 *protkey, u32 *protkeylen, u32 *protkeytype)
+static int pkey_ioctl_genseck(struct pkey_genseck __user *ugs)
 {
-	u32 nr_apqns, *apqns = NULL;
-	int i, j, rc = -ENODEV;
-	u16 card, dom;
+	struct pkey_genseck kgs;
+	struct pkey_apqn apqn;
+	u32 keybuflen;
+	int rc;
 
-	zcrypt_wait_api_operational();
+	if (copy_from_user(&kgs, ugs, sizeof(kgs)))
+		return -EFAULT;
 
-	/* try two times in case of failure */
-	for (i = 0; i < 2 && rc; i++) {
+	apqn.card = kgs.cardnr;
+	apqn.domain = kgs.domain;
+	keybuflen = sizeof(kgs.seckey.seckey);
+	rc = pkey_handler_gen_key(&apqn, 1,
+				  kgs.keytype, PKEY_TYPE_CCA_DATA, 0, 0,
+				  kgs.seckey.seckey, &keybuflen, NULL);
+	pr_debug("gen_key()=%d\n", rc);
+	if (!rc && copy_to_user(ugs, &kgs, sizeof(kgs)))
+		rc = -EFAULT;
+	memzero_explicit(&kgs, sizeof(kgs));
 
-		/* build a list of apqns suitable for this key */
-		rc = ep11_findcard2(&apqns, &nr_apqns, 0xFFFF, 0xFFFF,
-				    ZCRYPT_CEX7,
-				    ap_is_se_guest() ? EP11_API_V6 : EP11_API_V4,
-				    ep11_kb_wkvp(key, keylen));
-		if (rc)
-			continue; /* retry findcard on failure */
-
-		/* go through the list of apqns and try to derive an pkey */
-		for (rc = -ENODEV, j = 0; j < nr_apqns && rc; j++) {
-			card = apqns[j] >> 16;
-			dom = apqns[j] & 0xFFFF;
-			rc = ep11_kblob2protkey(card, dom, key, keylen,
-						protkey, protkeylen, protkeytype);
-		}
+	return rc;
+}
 
-		kfree(apqns);
-	}
+static int pkey_ioctl_clr2seck(struct pkey_clr2seck __user *ucs)
+{
+	struct pkey_clr2seck kcs;
+	struct pkey_apqn apqn;
+	u32 keybuflen;
+	int rc;
 
-	if (rc)
-		pr_debug("%s failed rc=%d\n", __func__, rc);
+	if (copy_from_user(&kcs, ucs, sizeof(kcs)))
+		return -EFAULT;
+
+	apqn.card = kcs.cardnr;
+	apqn.domain = kcs.domain;
+	keybuflen = sizeof(kcs.seckey.seckey);
+	rc = pkey_handler_clr_to_key(&apqn, 1,
+				     kcs.keytype, PKEY_TYPE_CCA_DATA, 0, 0,
+				     kcs.clrkey.clrkey,
+				     pkey_keytype_aes_to_size(kcs.keytype),
+				     kcs.seckey.seckey, &keybuflen, NULL);
+	pr_debug("clr_to_key()=%d\n", rc);
+	if (!rc && copy_to_user(ucs, &kcs, sizeof(kcs)))
+		rc = -EFAULT;
+	memzero_explicit(&kcs, sizeof(kcs));
 
 	return rc;
 }
 
-/*
- * Verify key and give back some info about the key.
- */
-static int pkey_verifykey(const struct pkey_seckey *seckey,
-			  u16 *pcardnr, u16 *pdomain,
-			  u16 *pkeysize, u32 *pattributes)
+static int pkey_ioctl_sec2protk(struct pkey_sec2protk __user *usp)
 {
-	struct secaeskeytoken *t = (struct secaeskeytoken *)seckey;
-	u16 cardnr, domain;
+	struct pkey_sec2protk ksp;
+	struct pkey_apqn apqn;
 	int rc;
 
-	/* check the secure key for valid AES secure key */
-	rc = cca_check_secaeskeytoken(pkey_dbf_info, 3, (u8 *)seckey, 0);
-	if (rc)
-		goto out;
-	if (pattributes)
-		*pattributes = PKEY_VERIFY_ATTR_AES;
-	if (pkeysize)
-		*pkeysize = t->bitsize;
-
-	/* try to find a card which can handle this key */
-	rc = cca_findcard(seckey->seckey, &cardnr, &domain, 1);
-	if (rc < 0)
-		goto out;
-
-	if (rc > 0) {
-		/* key mkvp matches to old master key mkvp */
-		pr_debug("%s secure key has old mkvp\n", __func__);
-		if (pattributes)
-			*pattributes |= PKEY_VERIFY_ATTR_OLD_MKVP;
-		rc = 0;
-	}
+	if (copy_from_user(&ksp, usp, sizeof(ksp)))
+		return -EFAULT;
+
+	apqn.card = ksp.cardnr;
+	apqn.domain = ksp.domain;
+	ksp.protkey.len = sizeof(ksp.protkey.protkey);
+	rc = pkey_handler_key_to_protkey(&apqn, 1,
+					 ksp.seckey.seckey,
+					 sizeof(ksp.seckey.seckey),
+					 ksp.protkey.protkey,
+					 &ksp.protkey.len, &ksp.protkey.type);
+	pr_debug("key_to_protkey()=%d\n", rc);
+	if (!rc && copy_to_user(usp, &ksp, sizeof(ksp)))
+		rc = -EFAULT;
+	memzero_explicit(&ksp, sizeof(ksp));
 
-	if (pcardnr)
-		*pcardnr = cardnr;
-	if (pdomain)
-		*pdomain = domain;
-
-out:
-	pr_debug("%s rc=%d\n", __func__, rc);
 	return rc;
 }
 
-/*
- * Generate a random protected key
- */
-static int pkey_genprotkey(u32 keytype, u8 *protkey,
-			   u32 *protkeylen, u32 *protkeytype)
+static int pkey_ioctl_clr2protk(struct pkey_clr2protk __user *ucp)
 {
-	u8 clrkey[32];
-	int keysize;
+	struct pkey_clr2protk kcp;
+	struct clearkeytoken *t;
+	u32 keylen;
+	u8 *tmpbuf;
 	int rc;
 
-	keysize = pkey_keytype_aes_to_size(keytype);
-	if (!keysize) {
-		PKEY_DBF_ERR("%s unknown/unsupported keytype %d\n", __func__,
-			     keytype);
+	if (copy_from_user(&kcp, ucp, sizeof(kcp)))
+		return -EFAULT;
+
+	/* build a 'clear key token' from the clear key value */
+	keylen = pkey_keytype_aes_to_size(kcp.keytype);
+	if (!keylen) {
+		PKEY_DBF_ERR("%s unknown/unsupported keytype %u\n",
+			     __func__, kcp.keytype);
+		memzero_explicit(&kcp, sizeof(kcp));
 		return -EINVAL;
 	}
+	tmpbuf = kzalloc(sizeof(*t) + keylen, GFP_KERNEL);
+	if (!tmpbuf) {
+		memzero_explicit(&kcp, sizeof(kcp));
+		return -ENOMEM;
+	}
+	t = (struct clearkeytoken *)tmpbuf;
+	t->type = TOKTYPE_NON_CCA;
+	t->version = TOKVER_CLEAR_KEY;
+	t->keytype = (keylen - 8) >> 3;
+	t->len = keylen;
+	memcpy(t->clearkey, kcp.clrkey.clrkey, keylen);
+	kcp.protkey.len = sizeof(kcp.protkey.protkey);
 
-	/* generate a dummy random clear key */
-	get_random_bytes(clrkey, keysize);
+	rc = key2protkey(NULL, 0,
+			 tmpbuf, sizeof(*t) + keylen,
+			 kcp.protkey.protkey,
+			 &kcp.protkey.len, &kcp.protkey.type);
+	pr_debug("key2protkey()=%d\n", rc);
 
-	/* convert it to a dummy protected key */
-	rc = pkey_clr2protkey(keytype, clrkey,
-			      protkey, protkeylen, protkeytype);
-	if (rc)
-		return rc;
+	kfree_sensitive(tmpbuf);
 
-	/* replace the key part of the protected key with random bytes */
-	get_random_bytes(protkey, keysize);
+	if (!rc && copy_to_user(ucp, &kcp, sizeof(kcp)))
+		rc = -EFAULT;
+	memzero_explicit(&kcp, sizeof(kcp));
 
-	return 0;
+	return rc;
 }
 
-/*
- * Verify if a protected key is still valid
- */
-static int pkey_verifyprotkey(const u8 *protkey, u32 protkeylen,
-			      u32 protkeytype)
+static int pkey_ioctl_findcard(struct pkey_findcard __user *ufc)
 {
-	struct {
-