ISAL-Crypto
view release on metacpan or search on metacpan
isa-l_crypto/aes/gcm_avx_gen4.asm view on Meta::CPAN
; poly = x^128 + x^127 + x^126 + x^121 + 1
; throughout the code, one tab and two tab indentations are used. one tab is for GHASH part, two tabs is for AES part.
;
%include "reg_sizes.asm"
%include "gcm_defines.asm"
%ifndef GCM128_MODE
%ifndef GCM192_MODE
%ifndef GCM256_MODE
%error "No GCM mode selected for gcm_avx_gen4.asm!"
%endif
%endif
%endif
%ifndef FUNCT_EXTENSION
%define FUNCT_EXTENSION
%endif
;; Decide on AES-GCM key size to compile for
%ifdef GCM128_MODE
%define NROUNDS 9
%define FN_NAME(x,y) aes_gcm_ %+ x %+ _128 %+ y %+ avx_gen4 %+ FUNCT_EXTENSION
%endif
%ifdef GCM192_MODE
%define NROUNDS 11
%define FN_NAME(x,y) aes_gcm_ %+ x %+ _192 %+ y %+ avx_gen4 %+ FUNCT_EXTENSION
%endif
%ifdef GCM256_MODE
%define NROUNDS 13
%define FN_NAME(x,y) aes_gcm_ %+ x %+ _256 %+ y %+ avx_gen4 %+ FUNCT_EXTENSION
%endif
section .text
default rel
; need to push 5 registers into stack to maintain
%define STACK_OFFSET 8*5
%define TMP2 16*0 ; Temporary storage for AES State 2 (State 1 is stored in an XMM register)
%define TMP3 16*1 ; Temporary storage for AES State 3
%define TMP4 16*2 ; Temporary storage for AES State 4
%define TMP5 16*3 ; Temporary storage for AES State 5
%define TMP6 16*4 ; Temporary storage for AES State 6
%define TMP7 16*5 ; Temporary storage for AES State 7
%define TMP8 16*6 ; Temporary storage for AES State 8
%define LOCAL_STORAGE 16*7
%ifidn __OUTPUT_FORMAT__, win64
%define XMM_STORAGE 16*10
%else
%define XMM_STORAGE 0
%endif
%define VARIABLE_OFFSET LOCAL_STORAGE + XMM_STORAGE
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; Utility Macros
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; GHASH_MUL MACRO to implement: Data*HashKey mod (128,127,126,121,0)
; Input: A and B (128-bits each, bit-reflected)
; Output: C = A*B*x mod poly, (i.e. >>1 )
; To compute GH = GH*HashKey mod poly, give HK = HashKey<<1 mod poly as input
; GH = GH * HK * x mod poly which is equivalent to GH*HashKey mod poly.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
%macro GHASH_MUL 7
%define %%GH %1 ; 16 Bytes
%define %%HK %2 ; 16 Bytes
%define %%T1 %3
%define %%T2 %4
%define %%T3 %5
%define %%T4 %6
%define %%T5 %7
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
vpclmulqdq %%T1, %%GH, %%HK, 0x11 ; %%T1 = a1*b1
vpclmulqdq %%T2, %%GH, %%HK, 0x00 ; %%T2 = a0*b0
vpclmulqdq %%T3, %%GH, %%HK, 0x01 ; %%T3 = a1*b0
vpclmulqdq %%GH, %%GH, %%HK, 0x10 ; %%GH = a0*b1
vpxor %%GH, %%GH, %%T3
vpsrldq %%T3, %%GH, 8 ; shift-R %%GH 2 DWs
vpslldq %%GH, %%GH, 8 ; shift-L %%GH 2 DWs
vpxor %%T1, %%T1, %%T3
vpxor %%GH, %%GH, %%T2
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;first phase of the reduction
vmovdqu %%T3, [POLY2]
vpclmulqdq %%T2, %%T3, %%GH, 0x01
vpslldq %%T2, %%T2, 8 ; shift-L %%T2 2 DWs
vpxor %%GH, %%GH, %%T2 ; first phase of the reduction complete
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;second phase of the reduction
vpclmulqdq %%T2, %%T3, %%GH, 0x00
vpsrldq %%T2, %%T2, 4 ; shift-R %%T2 1 DW (Shift-R only 1-DW to obtain 2-DWs shift-R)
vpclmulqdq %%GH, %%T3, %%GH, 0x10
vpslldq %%GH, %%GH, 4 ; shift-L %%GH 1 DW (Shift-L 1-DW to obtain result with no shifts)
vpxor %%GH, %%GH, %%T2 ; second phase of the reduction complete
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
vpxor %%GH, %%GH, %%T1 ; the result is in %%GH
%endmacro
; In PRECOMPUTE, the commands filling Hashkey_i_k are not required for avx_gen4
; functions, but are kept to allow users to switch cpu architectures between calls
; of pre, init, update, and finalize.
%macro PRECOMPUTE 8
%define %%GDATA %1
isa-l_crypto/aes/gcm_avx_gen4.asm view on Meta::CPAN
%define %%T4 %13
%define %%T5 %14 ; temp reg 5
mov %%T1, %%A_IN ; T1 = AAD
mov %%T2, %%A_LEN ; T2 = aadLen
vpxor %%AAD_HASH, %%AAD_HASH
cmp %%T2, 16
jl %%_get_small_AAD_block
%%_get_AAD_loop16:
vmovdqu %%XTMP1, [%%T1]
;byte-reflect the AAD data
vpshufb %%XTMP1, [SHUF_MASK]
vpxor %%AAD_HASH, %%XTMP1
GHASH_MUL %%AAD_HASH, %%HASH_KEY, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5
sub %%T2, 16
je %%_CALC_AAD_done
add %%T1, 16
cmp %%T2, 16
jge %%_get_AAD_loop16
%%_get_small_AAD_block:
READ_SMALL_DATA_INPUT %%XTMP1, %%T1, %%T2, %%T3, %%T4, %%T5
;byte-reflect the AAD data
vpshufb %%XTMP1, [SHUF_MASK]
vpxor %%AAD_HASH, %%XTMP1
GHASH_MUL %%AAD_HASH, %%HASH_KEY, %%XTMP1, %%XTMP2, %%XTMP3, %%XTMP4, %%XTMP5
%%_CALC_AAD_done:
%endmacro ; CALC_AAD_HASH
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; PARTIAL_BLOCK: Handles encryption/decryption and the tag partial blocks between update calls.
; Requires the input data be at least 1 byte long.
; Input: gcm_key_data * (GDATA_KEY), gcm_context_data *(GDATA_CTX), input text (PLAIN_CYPH_IN),
; input text length (PLAIN_CYPH_LEN), the current data offset (DATA_OFFSET),
; and whether encoding or decoding (ENC_DEC)
; Output: A cypher of the first partial block (CYPH_PLAIN_OUT), and updated GDATA_CTX
; Clobbers rax, r10, r12, r13, r15, xmm0, xmm1, xmm2, xmm3, xmm5, xmm6, xmm9, xmm10, xmm11, xmm13
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
%macro PARTIAL_BLOCK 8
%define %%GDATA_KEY %1
%define %%GDATA_CTX %2
%define %%CYPH_PLAIN_OUT %3
%define %%PLAIN_CYPH_IN %4
%define %%PLAIN_CYPH_LEN %5
%define %%DATA_OFFSET %6
%define %%AAD_HASH %7
%define %%ENC_DEC %8
mov r13, [%%GDATA_CTX + PBlockLen]
cmp r13, 0
je %%_partial_block_done ;Leave Macro if no partial blocks
cmp %%PLAIN_CYPH_LEN, 16 ;Read in input data without over reading
jl %%_fewer_than_16_bytes
VXLDR xmm1, [%%PLAIN_CYPH_IN] ;If more than 16 bytes of data, just fill the xmm register
jmp %%_data_read
%%_fewer_than_16_bytes:
lea r10, [%%PLAIN_CYPH_IN + %%DATA_OFFSET]
READ_SMALL_DATA_INPUT xmm1, r10, %%PLAIN_CYPH_LEN, rax, r12, r15
%%_data_read: ;Finished reading in data
vmovdqu xmm9, [%%GDATA_CTX + PBlockEncKey] ;xmm9 = my_ctx_data.partial_block_enc_key
vmovdqu xmm13, [%%GDATA_KEY + HashKey]
lea r12, [SHIFT_MASK]
add r12, r13 ; adjust the shuffle mask pointer to be able to shift r13 bytes (16-r13 is the number of bytes in plaintext mod 16)
vmovdqu xmm2, [r12] ; get the appropriate shuffle mask
vpshufb xmm9, xmm2 ;shift right r13 bytes
%ifidn %%ENC_DEC, DEC
vmovdqa xmm3, xmm1
vpxor xmm9, xmm1 ; Cyphertext XOR E(K, Yn)
mov r15, %%PLAIN_CYPH_LEN
add r15, r13
sub r15, 16 ;Set r15 to be the amount of data left in CYPH_PLAIN_IN after filling the block
jge %%_no_extra_mask_1 ;Determine if if partial block is not being filled and shift mask accordingly
sub r12, r15
%%_no_extra_mask_1:
vmovdqu xmm1, [r12 + ALL_F - SHIFT_MASK]; get the appropriate mask to mask out bottom r13 bytes of xmm9
vpand xmm9, xmm1 ; mask out bottom r13 bytes of xmm9
vpand xmm3, xmm1
vpshufb xmm3, [SHUF_MASK]
vpshufb xmm3, xmm2
vpxor %%AAD_HASH, xmm3
cmp r15,0
jl %%_partial_incomplete_1
GHASH_MUL %%AAD_HASH, xmm13, xmm0, xmm10, xmm11, xmm5, xmm6 ;GHASH computation for the last <16 Byte block
xor rax,rax
mov [%%GDATA_CTX + PBlockLen], rax
jmp %%_dec_done
%%_partial_incomplete_1:
add [%%GDATA_CTX + PBlockLen], %%PLAIN_CYPH_LEN
%%_dec_done:
vmovdqu [%%GDATA_CTX + AadHash], %%AAD_HASH
%else
vpxor xmm9, xmm1 ; Plaintext XOR E(K, Yn)
mov r15, %%PLAIN_CYPH_LEN
add r15, r13
sub r15, 16 ;Set r15 to be the amount of data left in CYPH_PLAIN_IN after filling the block
isa-l_crypto/aes/gcm_avx_gen4.asm view on Meta::CPAN
;; xmm2 - XMM2
;; xmm3 - XMM3
;; xmm4 - XMM4
;; xmm5 - XMM5
;; xmm6 - XMM6
;; xmm7 - XMM7
;; xmm8 - XMM8 - AAD HASH IN
;; xmm10 - T6
;; xmm0 - T_key
INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 7, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE
jmp %%_small_initial_blocks_encrypted
%%_small_initial_num_blocks_is_6:
INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 6, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE
jmp %%_small_initial_blocks_encrypted
%%_small_initial_num_blocks_is_5:
INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 5, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE
jmp %%_small_initial_blocks_encrypted
%%_small_initial_num_blocks_is_4:
INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 4, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE
jmp %%_small_initial_blocks_encrypted
%%_small_initial_num_blocks_is_3:
INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 3, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE
jmp %%_small_initial_blocks_encrypted
%%_small_initial_num_blocks_is_2:
INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 2, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE
jmp %%_small_initial_blocks_encrypted
%%_small_initial_num_blocks_is_1:
INITIAL_BLOCKS_PARTIAL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, r13, %%DATA_OFFSET, 1, xmm12, xmm13, xmm14, xmm15, xmm11, xmm9, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, xmm8, xmm10, xmm0, %%ENC_DEC, %%INSTANCE_TYPE
;; Note: zero initial blocks not allowed.
%%_small_initial_blocks_encrypted:
%endmacro ; GCM_ENC_DEC_SMALL
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; GCM_ENC_DEC Encodes/Decodes given data. Assumes that the passed gcm_context_data struct
; has been initialized by GCM_INIT
; Requires the input data be at least 1 byte long because of READ_SMALL_INPUT_DATA.
; Input: gcm_key_data struct* (GDATA_KEY), gcm_context_data *(GDATA_CTX), input text (PLAIN_CYPH_IN),
; input text length (PLAIN_CYPH_LEN) and whether encoding or decoding (ENC_DEC).
; Output: A cypher of the given plain text (CYPH_PLAIN_OUT), and updated GDATA_CTX
; Clobbers rax, r10-r15, and xmm0-xmm15
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
%macro GCM_ENC_DEC 7
%define %%GDATA_KEY %1
%define %%GDATA_CTX %2
%define %%CYPH_PLAIN_OUT %3
%define %%PLAIN_CYPH_IN %4
%define %%PLAIN_CYPH_LEN %5
%define %%ENC_DEC %6
%define %%INSTANCE_TYPE %7
%define %%DATA_OFFSET r11
; Macro flow:
; calculate the number of 16byte blocks in the message
; process (number of 16byte blocks) mod 8 '%%_initial_num_blocks_is_# .. %%_initial_blocks_encrypted'
; process 8 16 byte blocks at a time until all are done '%%_encrypt_by_8_new .. %%_eight_cipher_left'
; if there is a block of less tahn 16 bytes process it '%%_zero_cipher_left .. %%_multiple_of_16_bytes'
cmp %%PLAIN_CYPH_LEN, 0
je %%_enc_dec_done
xor %%DATA_OFFSET, %%DATA_OFFSET
;; Update length of data processed
add [%%GDATA_CTX+InLen], %%PLAIN_CYPH_LEN
vmovdqu xmm13, [%%GDATA_KEY + HashKey]
vmovdqu xmm8, [%%GDATA_CTX + AadHash]
%ifidn %%INSTANCE_TYPE, multi_call
;; NOTE: partial block processing makes only sense for multi_call here.
;; Used for the update flow - if there was a previous partial
;; block fill the remaining bytes here.
PARTIAL_BLOCK %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%DATA_OFFSET, xmm8, %%ENC_DEC
%endif
;; lift CTR set from initial_blocks to here
%ifidn %%INSTANCE_TYPE, single_call
vmovdqu xmm9, xmm2
%else
vmovdqu xmm9, [%%GDATA_CTX + CurCount]
%endif
;; Save the amount of data left to process in r10
mov r13, %%PLAIN_CYPH_LEN
%ifidn %%INSTANCE_TYPE, multi_call
;; NOTE: %%DATA_OFFSET is zero in single_call case.
;; Consequently PLAIN_CYPH_LEN will never be zero after
;; %%DATA_OFFSET subtraction below.
sub r13, %%DATA_OFFSET
;; There may be no more data if it was consumed in the partial block.
cmp r13, 0
je %%_enc_dec_done
%endif ; %%INSTANCE_TYPE, multi_call
mov r10, r13
;; Determine how many blocks to process in INITIAL
mov r12, r13
shr r12, 4
and r12, 7
;; Process one additional block in INITIAL if there is a partial block
and r10, 0xf
blsmsk r10, r10 ; Set CF if zero
cmc ; Flip CF
adc r12, 0x0 ; Process an additional INITIAL block if CF set
;; Less than 127B will be handled by the small message code, which
;; can process up to 7 16B blocks.
cmp r13, 128
jge %%_large_message_path
GCM_ENC_DEC_SMALL %%GDATA_KEY, %%GDATA_CTX, %%CYPH_PLAIN_OUT, %%PLAIN_CYPH_IN, %%PLAIN_CYPH_LEN, %%ENC_DEC, %%DATA_OFFSET, r13, r12, xmm9, xmm14, %%INSTANCE_TYPE
jmp %%_ghash_done
( run in 2.260 seconds using v1.01-cache-2.11-cpan-9e1a9122474 )