path: root/arch/s390/crypto/chacha-s390.S

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Original implementation written by Andy Polyakov, @dot-asm.
 * This is an adaptation of the original code for kernel use.
 *
 * Copyright (C) 2006-2019 CRYPTOGAMS by <appro@openssl.org>. All Rights Reserved.
 */

#include <linux/linkage.h>
#include <asm/nospec-insn.h>
#include <asm/fpu-insn.h>

#define SP	%r15
#define FRAME	(16 * 8 + 4 * 8)

	.data
	.balign	32

SYM_DATA_START_LOCAL(sigma)
	.long	0x61707865,0x3320646e,0x79622d32,0x6b206574	# endian-neutral
	.long	1,0,0,0
	.long	2,0,0,0
	.long	3,0,0,0
	.long	0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c	# byte swap

	.long	0,1,2,3
	.long	0x61707865,0x61707865,0x61707865,0x61707865	# smashed sigma
	.long	0x3320646e,0x3320646e,0x3320646e,0x3320646e
	.long	0x79622d32,0x79622d32,0x79622d32,0x79622d32
	.long	0x6b206574,0x6b206574,0x6b206574,0x6b206574
SYM_DATA_END(sigma)

	.previous

	GEN_BR_THUNK %r14

	.text

#############################################################################
# void chacha20_vx_4x(u8 *out, counst u8 *inp, size_t len,
#		      counst u32 *key, const u32 *counter)

#define	OUT		%r2
#define	INP		%r3
#define	LEN		%r4
#define	KEY		%r5
#define	COUNTER		%r6

#define BEPERM		%v31
#define CTR		%v26

#define K0		%v16
#define K1		%v17
#define K2		%v18
#define K3		%v19

#define XA0		%v0
#define XA1		%v1
#define XA2		%v2
#define XA3		%v3

#define XB0		%v4
#define XB1		%v5
#define XB2		%v6
#define XB3		%v7

#define XC0		%v8
#define XC1		%v9
#define XC2		%v10
#define XC3		%v11

#define XD0		%v12
#define XD1		%v13
#define XD2		%v14
#define XD3		%v15

#define XT0		%v27
#define XT1		%v28
#define XT2		%v29
#define XT3		%v30

SYM_FUNC_START(chacha20_vx_4x)
	stmg	%r6,%r7,6*8(SP)

	larl	%r7,sigma
	lhi	%r0,10
	lhi	%r1,0

	VL	K0,0,,%r7		# load sigma
	VL	K1,0,,KEY		# load key
	VL	K2,16,,KEY
	VL	K3,0,,COUNTER		# load counter

	VL	BEPERM,0x40,,%r7
	VL	CTR,0x50,,%r7

	VLM	XA0,XA3,0x60,%r7,4	# load [smashed] sigma

	VREPF	XB0,K1,0		# smash the key
	VREPF	XB1,K1,1
	VREPF	XB2,K1,2
	VREPF	XB3,K1,3

	VREPF	XD0,K3,0
	VREPF	XD1,K3,1
	VREPF	XD2,K3,2
	VREPF	XD3,K3,3
	VAF	XD0,XD0,CTR

	VREPF	XC0,K2,0
	VREPF	XC1,K2,1
	VREPF	XC2,K2,2
	VREPF	XC3,K2,3

.Loop_4x:
	VAF	XA0,XA0,XB0
	VX	XD0,XD0,XA0
	VERLLF	XD0,XD0,16

	VAF	XA1,XA1,XB1
	VX	XD1,XD1,XA1
	VERLLF	XD1,XD1,16

	VAF	XA2,XA2,XB2
	VX	XD2,XD2,XA2
	VERLLF	XD2,XD2,16

	VAF	XA3,XA3,XB3
	VX	XD3,XD3,XA3
	VERLLF	XD3,XD3,16

	VAF	XC0,XC0,XD0
	VX	XB0,XB0,XC0
	VERLLF	XB0,XB0,12

	VAF	XC1,XC1,XD1
	VX	XB1,XB1,XC1
	VERLLF	XB1,XB1,12

	VAF	XC2,XC2,XD2
	VX	XB2,XB2,XC2
	VERLLF	XB2,XB2,12

	VAF	XC3,XC3,XD3
	VX	XB3,XB3,XC3
	VERLLF	XB3,XB3,12

	VAF	XA0,XA0,XB0
	VX	XD0,XD0,XA0
	VERLLF	XD0,XD0,8

	VAF	XA1,XA1,XB1
	VX	XD1,XD1,XA1
	VERLLF	XD1,XD1,8

	VAF	XA2,XA2,XB2
	VX	XD2,XD2,XA2
	VERLLF	XD2,XD2,8

	VAF	XA3,XA3,XB3
	VX	XD3,XD3,XA3
	VERLLF	XD3,XD3,8

	VAF	XC0,XC0,XD0
	VX	XB0,XB0,XC0
	VERLLF	XB0,XB0,7

	VAF	XC1,XC1,XD1
	VX	XB1,XB1,XC1
	VERLLF	XB1,XB1,7

	VAF	XC2,XC2,XD2
	VX	XB2,XB2,XC2
	VERLLF	XB2,XB2,7

	VAF	XC3,XC3,XD3
	VX	XB3,XB3,XC3
	VERLLF	XB3,XB3,7

	VAF	XA0,XA0,XB1
	VX	XD3,XD3,XA0
	VERLLF	XD3,XD3,16

	VAF	XA1,XA1,XB2
	VX	XD0,XD0,XA1
	VERLLF	XD0,XD0,16

	VAF	XA2,XA2,XB3
	VX	XD1,XD1,XA2
	VERLLF	XD1,XD1,16

	VAF	XA3,XA3,XB0
	VX	XD2,XD2,XA3
	VERLLF	XD2,XD2,16

	VAF	XC2,XC2,XD3
	VX	XB1,XB1,XC2
	VERLLF	XB1,XB1,12

	VAF	XC3,XC3,XD0
	VX	XB2,XB2,XC3
	VERLLF	XB2,XB2,12

	VAF	XC0,XC0,XD1
	VX	XB3,XB3,XC0
	VERLLF	XB3,XB3,12

	VAF	XC1,XC1,XD2
	VX	XB0,XB0,XC1
	VERLLF	XB0,XB0,12

	VAF	XA0,XA0,XB1
	VX	XD3,XD3,XA0
	VERLLF	XD3,XD3,8

	VAF	XA1,XA1,XB2
	VX	XD0,XD0,XA1
	VERLLF	XD0,XD0,8

	VAF	XA2,XA2,XB3
	VX	XD1,XD1,XA2
	VERLLF	XD1,XD1,8

	VAF	XA3,XA3,XB0
	VX	XD2,XD2,XA3
	VERLLF	XD2,XD2,8

	VAF	XC2,XC2,XD3
	VX	XB1,XB1,XC2
	VERLLF	XB1,XB1,7

	VAF	XC3,XC3,XD0
	VX	XB2,XB2,XC3
	VERLLF	XB2,XB2,7

	VAF	XC0,XC0,XD1
	VX	XB3,XB3,XC0
	VERLLF	XB3,XB3,7

	VAF	XC1,XC1,XD2
	VX	XB0,XB0,XC1
	VERLLF	XB0,XB0,7
	brct	%r0,.Loop_4x

	VAF	XD0,XD0,CTR

	VMRHF	XT0,XA0,XA1		# transpose data
	VMRHF	XT1,XA2,XA3
	VMRLF	XT2,XA0,XA1
	VMRLF	XT3,XA2,XA3
	VPDI	XA0,XT0,XT1,0b0000
	VPDI	XA1,XT0,XT1,