Update
This commit is contained in:
bluepython508 committed
1 parent
3530afb854
commit
6903fcff27
2146 files changed
+112256
-190703
No files matched your search
+107
@@ -0,0 +1,107 @@
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
//
|
||||
// Generated from CryptoGAMS poly1305-armv4.pl by plan9-xlate.pl. The
|
||||
// generated poly1305_arm.s carries the upstream BSD-3-Clause notice
|
||||
// in its file header.
|
||||
|
||||
//go:build arm
|
||||
|
||||
//go:generate sh ../regen/regen.sh
|
||||
|
||||
// Package poly1305 implements Poly1305 with the CryptoGAMS ARMv4
|
||||
// assembly inner loop, regenerated by tsasm/arm/regen/plan9-xlate.pl.
|
||||
// It only builds on GOARCH=arm; other architectures should use
|
||||
// golang.org/x/crypto/poly1305.
|
||||
package poly1305
|
||||
|
||||
import "unsafe"
|
||||
|
||||
const (
|
||||
KeySize = 32
|
||||
TagSize = 16
|
||||
)
|
||||
|
||||
// ctxSize is the byte size of the upstream poly1305 state struct.
|
||||
// Scalar paths use the first 40 bytes (5 base-2^32 h limbs at 0..19,
|
||||
// 4 r limbs at 20..35, an is_base2_26 flag at 36). The NEON paths
|
||||
// use more; we size generously.
|
||||
const ctxSize = 192
|
||||
|
||||
//go:noescape
|
||||
func poly1305_init(ctx unsafe.Pointer, key unsafe.Pointer)
|
||||
|
||||
//go:noescape
|
||||
func poly1305_blocks(ctx unsafe.Pointer, inp unsafe.Pointer, length uintptr, padbit uintptr)
|
||||
|
||||
//go:noescape
|
||||
func poly1305_emit(ctx unsafe.Pointer, mac unsafe.Pointer, nonce unsafe.Pointer)
|
||||
|
||||
// Sum computes a Poly1305 MAC of m under key, writing the 16-byte tag
|
||||
// to out.
|
||||
func Sum(out *[TagSize]byte, m []byte, key *[KeySize]byte) {
|
||||
var mac MAC
|
||||
mac.Init(key)
|
||||
mac.Write(m)
|
||||
mac.Sum(out)
|
||||
}
|
||||
|
||||
// MAC is an incremental Poly1305 message-authentication code computer.
|
||||
// Single-use; not safe for concurrent use. Zero-value is not usable --
|
||||
// call Init first.
|
||||
type MAC struct {
|
||||
ctx [ctxSize]byte
|
||||
nonce [16]byte
|
||||
buf [16]byte
|
||||
n int
|
||||
}
|
||||
|
||||
// New returns a fresh MAC keyed with key. Prefer (*MAC).Init for hot
|
||||
// paths; this constructor heap-allocates.
|
||||
func New(key *[KeySize]byte) *MAC {
|
||||
m := new(MAC)
|
||||
m.Init(key)
|
||||
return m
|
||||
}
|
||||
|
||||
// Init keys an existing (typically stack-allocated) MAC with key.
|
||||
func (m *MAC) Init(key *[KeySize]byte) {
|
||||
*m = MAC{}
|
||||
poly1305_init(unsafe.Pointer(&m.ctx[0]), unsafe.Pointer(&key[0]))
|
||||
copy(m.nonce[:], key[16:])
|
||||
}
|
||||
|
||||
// Write feeds bytes into the MAC. Returns len(p), nil.
|
||||
func (m *MAC) Write(p []byte) (int, error) {
|
||||
n := len(p)
|
||||
if m.n > 0 {
|
||||
k := copy(m.buf[m.n:], p)
|
||||
m.n += k
|
||||
p = p[k:]
|
||||
if m.n < 16 {
|
||||
return n, nil
|
||||
}
|
||||
poly1305_blocks(unsafe.Pointer(&m.ctx[0]), unsafe.Pointer(&m.buf[0]), 16, 1)
|
||||
m.n = 0
|
||||
}
|
||||
if blocks := len(p) >> 4; blocks > 0 {
|
||||
poly1305_blocks(unsafe.Pointer(&m.ctx[0]), unsafe.Pointer(&p[0]), uintptr(blocks*16), 1)
|
||||
p = p[blocks*16:]
|
||||
}
|
||||
if len(p) > 0 {
|
||||
copy(m.buf[:], p)
|
||||
m.n = len(p)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
// Sum finalizes the MAC and writes the 16-byte tag to out.
|
||||
func (m *MAC) Sum(out *[TagSize]byte) {
|
||||
if m.n > 0 {
|
||||
var pad [16]byte
|
||||
copy(pad[:], m.buf[:m.n])
|
||||
pad[m.n] = 1
|
||||
poly1305_blocks(unsafe.Pointer(&m.ctx[0]), unsafe.Pointer(&pad[0]), 16, 0)
|
||||
m.n = 0
|
||||
}
|
||||
poly1305_emit(unsafe.Pointer(&m.ctx[0]), unsafe.Pointer(&out[0]), unsafe.Pointer(&m.nonce[0]))
|
||||
}
|
||||
+908
@@ -0,0 +1,908 @@
|
||||
// Code generated by plan9-xlate.pl from CryptoGAMS Perl. DO NOT EDIT.
|
||||
//
|
||||
// Run `go generate ./tsasm/arm/...` (or tsasm/arm/regen/regen.sh) to
|
||||
// regenerate. The upstream chacha-armv4.pl / poly1305-armv4.pl are
|
||||
// fetched at a pinned commit by regen.sh; see that script for the
|
||||
// upstream URL and SHA. Translation is by tsasm/arm/regen/plan9-xlate.pl.
|
||||
//
|
||||
// The original CryptoGAMS code this was derived from is dual-licensed
|
||||
// under the BSD-3-Clause and OpenSSL licenses, at the holder's option.
|
||||
// The CryptoGAMS BSD-3-Clause text is reproduced below; that license
|
||||
// requires the copyright notice to travel with the source. Files in
|
||||
// tsasm/arm/regen/ that are not derived from CryptoGAMS (plan9-xlate.pl
|
||||
// and friends) are BSD-3-Clause without the dual-license option.
|
||||
//
|
||||
// ----------------------------------------------------------------
|
||||
// Copyright (c) 2006, CRYPTOGAMS by <appro@openssl.org>
|
||||
// All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions
|
||||
// are met:
|
||||
//
|
||||
// * Redistributions of source code must retain copyright
|
||||
// notices, this list of conditions and the following
|
||||
// disclaimer.
|
||||
//
|
||||
// * Redistributions in binary form must reproduce the above
|
||||
// copyright notice, this list of conditions and the following
|
||||
// disclaimer in the documentation and/or other materials
|
||||
// provided with the distribution.
|
||||
//
|
||||
// * Neither the name of the CRYPTOGAMS nor the names of its
|
||||
// copyright holder and contributors may be used to endorse or
|
||||
// promote products derived from this software without specific
|
||||
// prior written permission.
|
||||
//
|
||||
// ALTERNATIVELY, provided that this notice is retained in full, this
|
||||
// product may be distributed under the terms of the GNU General Public
|
||||
// License (GPL), in which case the provisions of the GPL apply INSTEAD
|
||||
// OF those given above.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
|
||||
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
|
||||
// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
|
||||
// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
|
||||
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
||||
// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
||||
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
||||
// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
||||
// POSSIBILITY OF SUCH DAMAGE.
|
||||
// ----------------------------------------------------------------
|
||||
|
||||
#include "textflag.h"
|
||||
#include "funcdata.h"
|
||||
|
||||
|
||||
// func poly1305_init(ctx uintptr, key uintptr)
|
||||
TEXT ·poly1305_init(SB), NOSPLIT, $36-8
|
||||
PCDATA $PCDATA_UnsafePoint, $-2
|
||||
NO_LOCAL_POINTERS
|
||||
MOVW ctx+0(FP), R0
|
||||
MOVW key+4(FP), R1
|
||||
MOVW R4, 4(R13)
|
||||
MOVW R5, 8(R13)
|
||||
MOVW R6, 12(R13)
|
||||
MOVW R7, 16(R13)
|
||||
MOVW R8, 20(R13)
|
||||
MOVW R9, 24(R13)
|
||||
MOVW R11, 32(R13)
|
||||
lb_Lpoly1305_init:
|
||||
EOR R3, R3, R3
|
||||
CMP $0, R1
|
||||
MOVW R3, 0(R0) // zero hash value
|
||||
MOVW R3, 4(R0)
|
||||
MOVW R3, 8(R0)
|
||||
MOVW R3, 12(R0)
|
||||
MOVW R3, 16(R0)
|
||||
MOVW R3, 36(R0) // clear is_base2_26
|
||||
ADD $20, R0, R0
|
||||
MOVW.EQ $0, R0
|
||||
B.EQ lb_Lno_key
|
||||
MOVBU 0(R1), R4
|
||||
MOVW R14, 36(R13)
|
||||
MOVW $0x0fffffff, R14
|
||||
MOVW R14, 28(R13)
|
||||
MOVBU 1(R1), R5
|
||||
BIC $3, R14, R3 // 0x0ffffffc
|
||||
MOVBU 2(R1), R6
|
||||
MOVBU 3(R1), R7
|
||||
ORR R5<<8, R4, R4
|
||||
MOVBU 4(R1), R5
|
||||
ORR R6<<16, R4, R4
|
||||
MOVBU 5(R1), R6
|
||||
ORR R7<<24, R4, R4
|
||||
MOVBU 6(R1), R7
|
||||
AND R14, R4, R4
|
||||
MOVBU 7(R1), R8
|
||||
ORR R6<<8, R5, R5
|
||||
MOVBU 8(R1), R6
|
||||
ORR R7<<16, R5, R5
|
||||
MOVBU 9(R1), R7
|
||||
ORR R8<<24, R5, R5
|
||||
MOVBU 10(R1), R8
|
||||
AND R3, R5, R5
|
||||
MOVBU 11(R1), R9
|
||||
ORR R7<<8, R6, R6
|
||||
MOVBU 12(R1), R7
|
||||
ORR R8<<16, R6, R6
|
||||
MOVBU 13(R1), R8
|
||||
ORR R9<<24, R6, R6
|
||||
MOVBU 14(R1), R9
|
||||
AND R3, R6, R6
|
||||
MOVBU 15(R1), R14
|
||||
MOVW R14, 28(R13)
|
||||
ORR R8<<8, R7, R7
|
||||
MOVW R4, 0(R0)
|
||||
ORR R9<<16, R7, R7
|
||||
MOVW R5, 4(R0)
|
||||
ORR R14<<24, R7, R7
|
||||
MOVW R6, 8(R0)
|
||||
AND R3, R7, R7
|
||||
MOVW R7, 12(R0)
|
||||
MOVW $0, R0
|
||||
MOVW 36(R13), R14
|
||||
lb_Lno_key:
|
||||
RET
|
||||
|
||||
// func poly1305_blocks(ctx uintptr, inp uintptr, len uintptr, padbit uintptr)
|
||||
TEXT ·poly1305_blocks(SB), NOSPLIT, $72-16
|
||||
PCDATA $PCDATA_UnsafePoint, $-2
|
||||
NO_LOCAL_POINTERS
|
||||
MOVW ctx+0(FP), R0
|
||||
MOVW inp+4(FP), R1
|
||||
MOVW len+8(FP), R2
|
||||
MOVW padbit+12(FP), R3
|
||||
MOVW R3, 36(R13)
|
||||
MOVW R4, 40(R13)
|
||||
MOVW R5, 44(R13)
|
||||
MOVW R6, 48(R13)
|
||||
MOVW R7, 52(R13)
|
||||
MOVW R8, 56(R13)
|
||||
MOVW R9, 60(R13)
|
||||
MOVW R11, 68(R13)
|
||||
lb_Lpoly1305_blocks:
|
||||
BIC.S $15, R2, R2
|
||||
B.EQ lb_Lno_data
|
||||
ADD R1, R2, R2 // end pointer
|
||||
MOVW R14, 72(R13)
|
||||
MOVW 0(R0), R4
|
||||
MOVW 4(R0), R5
|
||||
MOVW 8(R0), R6
|
||||
MOVW 12(R0), R7
|
||||
MOVW 16(R0), R8
|
||||
MOVW 20(R0), R9
|
||||
MOVW 24(R0), R14
|
||||
MOVW R14, 64(R13)
|
||||
MOVW 28(R0), R11
|
||||
MOVW 32(R0), R12
|
||||
MOVW 72(R13), R14
|
||||
ADD $20, R0, R0
|
||||
MOVW R2, 20(R13) // offload stuff
|
||||
MOVW R0, 16(R13)
|
||||
MOVW R1, R14
|
||||
CMP $0, R3
|
||||
MOVW R14, 72(R13)
|
||||
MOVW 64(R13), R14
|
||||
MOVW R14, 24(R13)
|
||||
MOVW R11, 28(R13)
|
||||
MOVW R12, 32(R13)
|
||||
MOVW 72(R13), R14
|
||||
B lb_Loop
|
||||
lb_Loop:
|
||||
MOVBU.P 16(R14), R0 // load input
|
||||
ADD.HI $1, R8, R8 // 1<<128
|
||||
MOVBU -15(R14), R1
|
||||
MOVBU -14(R14), R2
|
||||
MOVBU -13(R14), R3
|
||||
ORR R1<<8, R0, R1
|
||||
MOVBU -12(R14), R0
|
||||
ORR R2<<16, R1, R2
|
||||
MOVBU -11(R14), R1
|
||||
ORR R3<<24, R2, R3
|
||||
MOVBU -10(R14), R2
|
||||
ADD.S R3, R4, R4 // accumulate input
|
||||
MOVBU -9(R14), R3
|
||||
ORR R1<<8, R0, R1
|
||||
MOVBU -8(R14), R0
|
||||
ORR R2<<16, R1, R2
|
||||
MOVBU -7(R14), R1
|
||||
ORR R3<<24, R2, R3
|
||||
MOVBU -6(R14), R2
|
||||
ADC.S R3, R5, R5
|
||||
MOVBU -5(R14), R3
|
||||
ORR R1<<8, R0, R1
|
||||
MOVBU -4(R14), R0
|
||||
ORR R2<<16, R1, R2
|
||||
MOVBU -3(R14), R1
|
||||
ORR R3<<24, R2, R3
|
||||
MOVBU -2(R14), R2
|
||||
ADC.S R3, R6, R6
|
||||
MOVBU -1(R14), R3
|
||||
ORR R1<<8, R0, R1
|
||||
MOVW R14, 12(R13) // offload input pointer
|
||||
ORR R2<<16, R1, R2
|
||||
MOVW R14, 72(R13)
|
||||
MOVW 64(R13), R14
|
||||
ADD R14>>2, R14, R14
|
||||
MOVW R14, 64(R13)
|
||||
ORR R3<<24, R2, R3
|
||||
ADD R11>>2, R11, R11
|
||||
ADC.S R3, R7, R7
|
||||
ADD R12>>2, R12, R12
|
||||
MULLU R5, R9, (R3, R2)
|
||||
ADC $0, R8, R8
|
||||
MULLU R4, R9, (R1, R0)
|
||||
MULALU R8, R14, (R3, R2)
|
||||
MULALU R7, R14, (R1, R0)
|
||||
MOVW 24(R13), R14 // reload r10
|
||||
MOVW R14, 64(R13)
|
||||
MULALU R6, R12, (R3, R2)
|
||||
MULALU R5, R12, (R1, R0)
|
||||
MULALU R7, R11, (R3, R2)
|
||||
MULALU R6, R11, (R1, R0)
|
||||
MULALU R4, R14, (R3, R2)
|
||||
MOVW R0, 4(R13) // future r4
|
||||
MUL R8, R11, R0
|
||||
MOVW 28(R13), R11 // reload r11
|
||||
ADD.S R1, R2, R2 // d1+=d0>>32
|
||||
EOR R1, R1, R1
|
||||
MOVW 72(R13), R14
|
||||
ADC $0, R3, R14 // future r6
|
||||
MOVW R2, 8(R13) // future r5
|
||||
MUL R8, R12, R2
|
||||
EOR R3, R3, R3
|
||||
MULALU R7, R12, (R1, R0)
|
||||
MOVW 32(R13), R12 // reload r12
|
||||
MULALU R7, R9, (R3, R2)
|
||||
MULALU R6, R9, (R1, R0)
|
||||
MOVW R14, 72(R13)
|
||||
MOVW 64(R13), R14
|
||||
MULALU R6, R14, (R3, R2)
|
||||
MULALU R5, R14, (R1, R0)
|
||||
MULALU R5, R11, (R3, R2)
|
||||
MULALU R4, R11, (R1, R0)
|
||||
MULALU R4, R12, (R3, R2)
|
||||
MOVW 4(R13), R4
|
||||
MUL R8, R9, R8
|
||||
MOVW 8(R13), R5
|
||||
MOVW 72(R13), R14
|
||||
ADD.S R0, R14, R6 // d2+=d1>>32
|
||||
MOVW 12(R13), R14 // reload input pointer
|
||||
ADC $0, R1, R1
|
||||
ADD.S R1, R2, R7 // d3+=d2>>32
|
||||
MOVW 20(R13), R0 // reload end pointer
|
||||
ADC $0, R3, R3
|
||||
ADD R3, R8, R8 // h4+=d3>>32
|
||||
BIC $3, R8, R1
|
||||
AND $3, R8, R8
|
||||
ADD R1>>2, R1, R1 // *=5
|
||||
ADD.S R1, R4, R4
|
||||
ADC.S $0, R5, R5
|
||||
ADC.S $0, R6, R6
|
||||
ADC.S $0, R7, R7
|
||||
ADC $0, R8, R8
|
||||
CMP R14, R0 // done yet?
|
||||
B.HI lb_Loop
|
||||
MOVW 16(R13), R0
|
||||
MOVM.DB [R4, R5, R6, R7, R8], (R0) // store the result
|
||||
lb_Lno_data:
|
||||
RET
|
||||
|
||||
// func poly1305_emit(ctx uintptr, mac uintptr, nonce uintptr)
|
||||
TEXT ·poly1305_emit(SB), NOSPLIT, $36-12
|
||||
PCDATA $PCDATA_UnsafePoint, $-2
|
||||
NO_LOCAL_POINTERS
|
||||
MOVW ctx+0(FP), R0
|
||||
MOVW mac+4(FP), R1
|
||||
MOVW nonce+8(FP), R2
|
||||
MOVW R4, 4(R13)
|
||||
MOVW R5, 8(R13)
|
||||
MOVW R6, 12(R13)
|
||||
MOVW R7, 16(R13)
|
||||
MOVW R8, 20(R13)
|
||||
MOVW R9, 24(R13)
|
||||
MOVW R11, 32(R13)
|
||||
lb_Lpoly1305_emit:
|
||||
MOVM.IA (R0), [R3, R4, R5, R6, R7]
|
||||
ADD.S $5, R3, R8 // compare to modulus
|
||||
ADC.S $0, R4, R9
|
||||
MOVW R14, 36(R13)
|
||||
ADC.S $0, R5, R14
|
||||
MOVW R14, 28(R13)
|
||||
ADC.S $0, R6, R11
|
||||
ADC $0, R7, R0
|
||||
TST $4, R0 // did it carry/borrow?
|
||||
MOVW.NE R8, R3
|
||||
MOVW 0(R2), R8
|
||||
MOVW.NE R9, R4
|
||||
MOVW 4(R2), R9
|
||||
MOVW.NE R14, R5
|
||||
MOVW 8(R2), R14
|
||||
MOVW R14, 28(R13)
|
||||
MOVW.NE R11, R6
|
||||
MOVW 12(R2), R11
|
||||
ADD.S R8, R3, R3
|
||||
ADC.S R9, R4, R4
|
||||
ADC.S R14, R5, R5
|
||||
ADC R11, R6, R6
|
||||
MOVB R3, 0(R1)
|
||||
MOVW R3>>8, R3
|
||||
MOVB R4, 4(R1)
|
||||
MOVW R4>>8, R4
|
||||
MOVB R5, 8(R1)
|
||||
MOVW R5>>8, R5
|
||||
MOVB R6, 12(R1)
|
||||
MOVW R6>>8, R6
|
||||
MOVB R3, 1(R1)
|
||||
MOVW R3>>8, R3
|
||||
MOVB R4, 5(R1)
|
||||
MOVW R4>>8, R4
|
||||
MOVB R5, 9(R1)
|
||||
MOVW R5>>8, R5
|
||||
MOVB R6, 13(R1)
|
||||
MOVW R6>>8, R6
|
||||
MOVB R3, 2(R1)
|
||||
MOVW R3>>8, R3
|
||||
MOVB R4, 6(R1)
|
||||
MOVW R4>>8, R4
|
||||
MOVB R5, 10(R1)
|
||||
MOVW R5>>8, R5
|
||||
MOVB R6, 14(R1)
|
||||
MOVW R6>>8, R6
|
||||
MOVB R3, 3(R1)
|
||||
MOVB R4, 7(R1)
|
||||
MOVB R5, 11(R1)
|
||||
MOVB R6, 15(R1)
|
||||
RET
|
||||
TEXT ·poly1305_init_neon(SB), NOSPLIT, $0-8
|
||||
PCDATA $PCDATA_UnsafePoint, $-2
|
||||
NO_LOCAL_POINTERS
|
||||
MOVW ctx+0(FP), R0
|
||||
MOVW key+4(FP), R1
|
||||
lb_Lpoly1305_init_neon:
|
||||
MOVW 48(R0), R3 // first table element
|
||||
CMP $-1, R3 // is value impossible?
|
||||
B.NE lb_Lno_init_neon
|
||||
MOVW 20(R0), R4 // load key base 2^32
|
||||
MOVW 24(R0), R5
|
||||
MOVW 28(R0), R6
|
||||
MOVW 32(R0), R7
|
||||
AND $0x03ffffff, R4, R2 // base 2^32 -> base 2^26
|
||||
MOVW R4>>26, R3
|
||||
MOVW R5>>20, R4
|
||||
ORR R5<<6, R3, R3
|
||||
MOVW R6>>14, R5
|
||||
ORR R6<<12, R4, R4
|
||||
MOVW R7>>8, R6
|
||||
ORR R7<<18, R5, R5
|
||||
AND $0x03ffffff, R3, R3
|
||||
AND $0x03ffffff, R4, R4
|
||||
AND $0x03ffffff, R5, R5
|
||||
WORD $0xee802b10 // vdup.32 d0,r2
|
||||
ADD R3<<2, R3, R2 // *5
|
||||
WORD $0xee813b10 // vdup.32 d1,r3
|
||||
ADD R4<<2, R4, R3
|
||||
WORD $0xee822b10 // vdup.32 d2,r2
|
||||
WORD $0xee834b10 // vdup.32 d3,r4
|
||||
ADD R5<<2, R5, R4
|
||||
WORD $0xee843b10 // vdup.32 d4,r3
|
||||
WORD $0xee855b10 // vdup.32 d5,r5
|
||||
ADD R6<<2, R6, R5
|
||||
WORD $0xee864b10 // vdup.32 d6,r4
|
||||
WORD $0xee876b10 // vdup.32 d7,r6
|
||||
WORD $0xee885b10 // vdup.32 d8,r5
|
||||
MOVW $2, R5 // counter
|
||||
lb_Lsquare_neon:
|
||||
// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
|
||||
// d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
|
||||
// d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
|
||||
// d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
|
||||
// d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
|
||||
// d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
|
||||
WORD $0xf3a0aa60 // vmull.u32 q5,d0,d0[1]
|
||||
WORD $0xf3a1ca60 // vmull.u32 q6,d1,d0[1]
|
||||
WORD $0xf3a3ea60 // vmull.u32 q7,d3,d0[1]
|
||||
WORD $0xf3e50a60 // vmull.u32 q8,d5,d0[1]
|
||||
WORD $0xf3e72a60 // vmull.u32 q9,d7,d0[1]
|
||||
WORD $0xf3a7a262 // vmlal.u32 q5,d7,d2[1]
|
||||
WORD $0xf3a0c261 // vmlal.u32 q6,d0,d1[1]
|
||||
WORD $0xf3a1e261 // vmlal.u32 q7,d1,d1[1]
|
||||
WORD $0xf3e30261 // vmlal.u32 q8,d3,d1[1]
|
||||
WORD $0xf3e52261 // vmlal.u32 q9,d5,d1[1]
|
||||
WORD $0xf3a5a264 // vmlal.u32 q5,d5,d4[1]
|
||||
WORD $0xf3a7c264 // vmlal.u32 q6,d7,d4[1]
|
||||
WORD $0xf3e10263 // vmlal.u32 q8,d1,d3[1]
|
||||
WORD $0xf3a0e263 // vmlal.u32 q7,d0,d3[1]
|
||||
WORD $0xf3e32263 // vmlal.u32 q9,d3,d3[1]
|
||||
WORD $0xf3a3a266 // vmlal.u32 q5,d3,d6[1]
|
||||
WORD $0xf3e00265 // vmlal.u32 q8,d0,d5[1]
|
||||
WORD $0xf3a5c266 // vmlal.u32 q6,d5,d6[1]
|
||||
WORD $0xf3a7e266 // vmlal.u32 q7,d7,d6[1]
|
||||
WORD $0xf3e12265 // vmlal.u32 q9,d1,d5[1]
|
||||
WORD $0xf3e70268 // vmlal.u32 q8,d7,d8[1]
|
||||
WORD $0xf3a1a268 // vmlal.u32 q5,d1,d8[1]
|
||||
WORD $0xf3a3c268 // vmlal.u32 q6,d3,d8[1]
|
||||
WORD $0xf3a5e268 // vmlal.u32 q7,d5,d8[1]
|
||||
WORD $0xf3e02267 // vmlal.u32 q9,d0,d7[1]
|
||||
// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
|
||||
// lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
|
||||
// and P. Schwabe
|
||||
// H0>>+H1>>+H2>>+H3>>+H4
|
||||
// H3>>+H4>>*5+H0>>+H1
|
||||
// Trivia.
|
||||
// Result of multiplication of n-bit number by m-bit number is
|
||||
// n+m bits wide. However! Even though 2^n is a n+1-bit number,
|
||||
// m-bit number multiplied by 2^n is still n+m bits wide.
|
||||
// Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2,
|
||||
// and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit
|
||||
// one is n+1 bits wide.
|
||||
// >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that
|
||||
// H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4
|
||||
// can be 27. However! In cases when their width exceeds 26 bits
|
||||
// they are limited by 2^26+2^6. This in turn means that *sum*
|
||||
// of the products with these values can still be viewed as sum
|
||||
// of 52-bit numbers as long as the amount of addends is not a
|
||||
// power of 2. For example,
|
||||
// H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4,
|
||||
// which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or
|
||||
// 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than
|
||||
// 8 * (2^52) or 2^55. However, the value is then multiplied by
|
||||
// by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12),
|
||||
// which is less than 32 * (2^52) or 2^57. And when processing
|
||||
// data we are looking at triple as many addends...
|
||||
// In key setup procedure pre-reduced H0 is limited by 5*4+1 and
|
||||
// 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the
|
||||
// input H0 is limited by (5*4+1)*3 addends, or 58 bits, while
|
||||
// 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32
|
||||
// instruction accepts 2x32-bit input and writes 2x64-bit result.
|
||||
// This means that result of reduction have to be compressed upon
|
||||
// loop wrap-around. This can be done in the process of reduction
|
||||
// to minimize amount of instructions [as well as amount of
|
||||
// 128-bit instructions, which benefits low-end processors], but
|
||||
// one has to watch for H2 (which is narrower than H0) and 5*H4
|
||||
// not being wider than 58 bits, so that result of right shift
|
||||
// by 26 bits fits in 32 bits. This is also useful on x86,
|
||||
// because it allows to use paddd in place for paddq, which
|
||||
// benefits Atom, where paddq is ridiculously slow.
|
||||
WORD $0xf3e6e0f0 // vshr.u64 q15,q8,#26
|
||||
WORD $0xf3fa0220 // vmovn.i64 d16,q8
|
||||
WORD $0xf3a680da // vshr.u64 q4,q5,#26
|
||||
WORD $0xf3baa20a // vmovn.i64 d10,q5
|
||||
WORD $0xf27228ee // vadd.i64 q9,q9,q15
|
||||
WORD $0xf3c7073c // vbic.i32 d16,#0xfc000000
|
||||
WORD $0xf23cc848 // vadd.i64 q6,q6,q4
|
||||
WORD $0xf387a73c // vbic.i32 d10,#0xfc000000
|
||||
WORD $0xf2e6e832 // vshrn.u64 d30,q9,#26
|
||||
WORD $0xf3fa2222 // vmovn.i64 d18,q9
|
||||
WORD $0xf3a680dc // vshr.u64 q4,q6,#26
|
||||
WORD $0xf3bac20c // vmovn.i64 d12,q6
|
||||
WORD $0xf23ee848 // vadd.i64 q7,q7,q4
|
||||
WORD $0xf3c7273c // vbic.i32 d18,#0xfc000000
|
||||
WORD $0xf387c73c // vbic.i32 d12,#0xfc000000
|
||||
WORD $0xf22aa82e // vadd.i32 d10,d10,d30
|
||||
WORD $0xf2e2e53e // vshl.u32 d30,d30,#2
|
||||
WORD $0xf2a6881e // vshrn.u64 d8,q7,#26
|
||||
WORD $0xf3bae20e // vmovn.i64 d14,q7
|
||||
WORD $0xf22aa82e // vadd.i32 d10,d10,d30
|
||||
WORD $0xf2600888 // vadd.i32 d16,d16,d8
|
||||
WORD $0xf387e73c // vbic.i32 d14,#0xfc000000
|
||||
WORD $0xf3e6e01a // vshr.u32 d30,d10,#26
|
||||
WORD $0xf387a73c // vbic.i32 d10,#0xfc000000
|
||||
WORD $0xf3a68030 // vshr.u32 d8,d16,#26
|
||||
WORD $0xf3c7073c // vbic.i32 d16,#0xfc000000
|
||||
WORD $0xf22cc82e // vadd.i32 d12,d12,d30
|
||||
WORD $0xf2622888 // vadd.i32 d18,d18,d8
|
||||
SUB.S $1, R5, R5
|
||||
B.EQ lb_Lsquare_break_neon
|
||||
ADD $(48+0*9*4), R0, R6
|
||||
ADD $(48+1*9*4), R0, R7
|
||||
WORD $0xf3ba008a // vtrn.32 d0,d10
|
||||
WORD $0xf3ba308e // vtrn.32 d3,d14
|
||||
WORD $0xf3ba50a0 // vtrn.32 d5,d16
|
||||
WORD $0xf3ba108c // vtrn.32 d1,d12
|
||||
WORD $0xf3ba70a2 // vtrn.32 d7,d18
|
||||
WORD $0xf2a24513 // vshl.u32 d4,d3,#2
|
||||
WORD $0xf2a26515 // vshl.u32 d6,d5,#2
|
||||
WORD $0xf2a22511 // vshl.u32 d2,d1,#2
|
||||
WORD $0xf2a28517 // vshl.u32 d8,d7,#2
|
||||
WORD $0xf2244803 // vadd.i32 d4,d4,d3
|
||||
WORD $0xf2222801 // vadd.i32 d2,d2,d1
|
||||
WORD $0xf2266805 // vadd.i32 d6,d6,d5
|
||||
WORD $0xf2288807 // vadd.i32 d8,d8,d7
|
||||
WORD $0xf4860b0d // vst4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
|
||||
WORD $0xf4870b8d // vst4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
|
||||
WORD $0xf4864b0d // vst4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
|
||||
WORD $0xf4874b8d // vst4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
|
||||
WORD $0xf486883f // vst1.32 {d8[0]},[r6,:32]
|
||||
WORD $0xf48788bf // vst1.32 {d8[1]},[r7,:32]
|
||||
B lb_Lsquare_neon
|
||||
lb_Lsquare_break_neon:
|
||||
ADD $(48+2*4*9), R0, R6
|
||||
ADD $(48+3*4*9), R0, R7
|
||||
WORD $0xf22a011a // vmov d0,d10
|
||||
WORD $0xf2a2251c // vshl.u32 d2,d12,#2
|
||||
WORD $0xf22c111c // vmov d1,d12
|
||||
WORD $0xf2a2451e // vshl.u32 d4,d14,#2
|
||||
WORD $0xf22e311e // vmov d3,d14
|
||||
WORD $0xf2a26530 // vshl.u32 d6,d16,#2
|
||||
WORD $0xf22051b0 // vmov d5,d16
|
||||
WORD $0xf2a28532 // vshl.u32 d8,d18,#2
|
||||
WORD $0xf22271b2 // vmov d7,d18
|
||||
WORD $0xf222280c // vadd.i32 d2,d2,d12
|
||||
WORD $0xf224480e // vadd.i32 d4,d4,d14
|
||||
WORD $0xf2266820 // vadd.i32 d6,d6,d16
|
||||
WORD $0xf2288822 // vadd.i32 d8,d8,d18
|
||||
WORD $0xf4860b0d // vst4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
|
||||
WORD $0xf4870b8d // vst4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
|
||||
WORD $0xf4864b0d // vst4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
|
||||
WORD $0xf4874b8d // vst4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
|
||||
WORD $0xf486880f // vst1.32 {d8[0]},[r6]
|
||||
WORD $0xf487888f // vst1.32 {d8[1]},[r7]
|
||||
lb_Lno_init_neon:
|
||||
RET
|
||||
|
||||
// func poly1305_blocks_neon(ctx uintptr, inp uintptr, len uintptr, padbit uintptr)
|
||||
TEXT ·poly1305_blocks_neon(SB), NOSPLIT, $96-16
|
||||
PCDATA $PCDATA_UnsafePoint, $-2
|
||||
NO_LOCAL_POINTERS
|
||||
MOVW ctx+0(FP), R0
|
||||
MOVW inp+4(FP), R1
|
||||
MOVW len+8(FP), R2
|
||||
MOVW padbit+12(FP), R3
|
||||
MOVW R4, 84(R13)
|
||||
MOVW R5, 88(R13)
|
||||
MOVW R6, 92(R13)
|
||||
MOVW R7, 96(R13)
|
||||
lb_Lpoly1305_blocks_neon:
|
||||
MOVW 36(R0), R12 // is_base2_26
|
||||
CMP $64, R2
|
||||
TST R12, R12 // is_base2_26?
|
||||
B.NE lb_Lbase2_26_neon
|
||||
MOVW R1, 4(R13)
|
||||
MOVW R2, 8(R13)
|
||||
MOVW R3, 12(R13)
|
||||
BL ·poly1305_init_neon(SB)
|
||||
MOVW 0(R0), R4 // load hash value base 2^32
|
||||
MOVW 4(R0), R5
|
||||
MOVW 8(R0), R6
|
||||
MOVW 12(R0), R7
|
||||
MOVW 16(R0), R12
|
||||
AND $0x03ffffff, R4, R2 // base 2^32 -> base 2^26
|
||||
MOVW R4>>26, R3
|
||||
WORD $0xf30aa11a // veor d10,d10,d10
|
||||
MOVW R5>>20, R4
|
||||
ORR R5<<6, R3, R3
|
||||
WORD $0xf30cc11c // veor d12,d12,d12
|
||||
MOVW R6>>14, R5
|
||||
ORR R6<<12, R4, R4
|
||||
WORD $0xf30ee11e // veor d14,d14,d14
|
||||
MOVW R7>>8, R6
|
||||
ORR R7<<18, R5, R5
|
||||
WORD $0xf34001b0 // veor d16,d16,d16
|
||||
AND $0x03ffffff, R3, R3
|
||||
ORR R12<<24, R6, R6
|
||||
WORD $0xf34221b2 // veor d18,d18,d18
|
||||
AND $0x03ffffff, R4, R4
|
||||
MOVW $1, R1
|
||||
AND $0x03ffffff, R5, R5
|
||||
MOVW R1, 36(R0) // set is_base2_26
|
||||
WORD $0xee0a2b10 // vmov.32 d10[0],r2
|
||||
WORD $0xee0c3b10 // vmov.32 d12[0],r3
|
||||
WORD $0xee0e4b10 // vmov.32 d14[0],r4
|
||||
WORD $0xee005b90 // vmov.32 d16[0],r5
|
||||
WORD $0xee026b90 // vmov.32 d18[0],r6
|
||||
MOVW $zeros<>(SB), R5
|
||||
MOVM.IA.W (R13), [R1, R2, R3, R14]
|
||||
B lb_Lhash_loaded
|
||||
lb_Lbase2_26_neon:
|
||||
// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
|
||||
// load hash value
|
||||
WORD $0xf30aa11a // veor d10,d10,d10
|
||||
WORD $0xf30cc11c // veor d12,d12,d12
|
||||
WORD $0xf30ee11e // veor d14,d14,d14
|
||||
WORD $0xf34001b0 // veor d16,d16,d16
|
||||
WORD $0xf34221b2 // veor d18,d18,d18
|
||||
WORD $0xf4a0ab4d // vld4.32 {d10[0],d12[0],d14[0],d16[0]},[r0]!
|
||||
MOVW $zeros<>(SB), R5
|
||||
WORD $0xf4e0280f // vld1.32 {d18[0]},[r0]
|
||||
SUB $16, R0, R0 // rewind
|
||||
lb_Lhash_loaded:
|
||||
ADD $32, R1, R4
|
||||
MOVW R3<<24, R3
|
||||
TST $31, R2
|
||||
B.EQ lb_Leven
|
||||
WORD $0xf4e14b4d // vld4.32 {d20[0],d22[0],d24[0],d26[0]},[r1]!
|
||||
WORD $0xee0c3b90 // vmov.32 d28[0],r3
|
||||
SUB $16, R2, R2
|
||||
ADD $32, R1, R4
|
||||
WORD $0xf3f8c43a // vsri.u32 d28,d26,#8
|
||||
WORD $0xf2f2a53a // vshl.u32 d26,d26,#18
|
||||
WORD $0xf3f2a438 // vsri.u32 d26,d24,#14
|
||||
WORD $0xf2ec8538 // vshl.u32 d24,d24,#12
|
||||
WORD $0xf26cd8a2 // vadd.i32 d29,d28,d18
|
||||
WORD $0xf3c7a73c // vbic.i32 d26,#0xfc000000
|
||||
WORD $0xf3ec8436 // vsri.u32 d24,d22,#20
|
||||
WORD $0xf2e66536 // vshl.u32 d22,d22,#6
|
||||
WORD $0xf3c7873c // vbic.i32 d24,#0xfc000000
|
||||
WORD $0xf3e66434 // vsri.u32 d22,d20,#26
|
||||
WORD $0xf26ab8a0 // vadd.i32 d27,d26,d16
|
||||
WORD $0xf3c7473c // vbic.i32 d20,#0xfc000000
|
||||
WORD $0xf3c7673c // vbic.i32 d22,#0xfc000000
|
||||
WORD $0xf268988e // vadd.i32 d25,d24,d14
|
||||
WORD $0xf264588a // vadd.i32 d21,d20,d10
|
||||
WORD $0xf266788c // vadd.i32 d23,d22,d12
|
||||
MOVW R5, R7
|
||||
ADD $48, R0, R6
|
||||
CMP R2, R2
|
||||
B lb_Long_tail
|
||||
lb_Leven:
|
||||
SUB.S $64, R2, R2
|
||||
MOVW.LO R5, R4
|
||||
WORD $0xf2c0c651 // vmov.i32 q14,#1<<24
|
||||
WORD $0xf461418f // vld4.32 {d20,d22,d24,d26},[r1]
|
||||
ADD $64, R1, R1
|
||||
WORD $0xf464518f // vld4.32 {d21,d23,d25,d27},[r4]
|
||||
ADD $64, R4, R4
|
||||
ADD.HI $(48+1*9*4), R0, R7
|
||||
ADD.HI $(48+3*9*4), R0, R6
|
||||
WORD $0xf3f8c47a // vsri.u32 q14,q13,#8
|
||||
WORD $0xf2f2a57a // vshl.u32 q13,q13,#18
|
||||
WORD $0xf3f2a478 // vsri.u32 q13,q12,#14
|
||||
WORD $0xf2ec8578 // vshl.u32 q12,q12,#12
|
||||
WORD $0xf3c7a77c // vbic.i32 q13,#0xfc000000
|
||||
WORD $0xf3ec8476 // vsri.u32 q12,q11,#20
|
||||
WORD $0xf2e66576 // vshl.u32 q11,q11,#6
|
||||
WORD $0xf3c7877c // vbic.i32 q12,#0xfc000000
|
||||
WORD $0xf3e66474 // vsri.u32 q11,q10,#26
|
||||
WORD $0xf3c7477c // vbic.i32 q10,#0xfc000000
|
||||
WORD $0xf3c7677c // vbic.i32 q11,#0xfc000000
|
||||
B.LS lb_Lskip_loop
|
||||
WORD $0xf4a70b8d // vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
|
||||
WORD $0xf4a60b0d // vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
|
||||
WORD $0xf4a74b8d // vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
|
||||
WORD $0xf4a64b0d // vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
|
||||
B lb_Loop_neon
|
||||
lb_Loop_neon:
|
||||
// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
|
||||
// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
|
||||
// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
|
||||
// ___________________/
|
||||
// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
|
||||
// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
|
||||
// ___________________/ ____________________/
|
||||
// Note that we start with inp[2:3]*r^2. This is because it
|
||||
// doesn't depend on reduction in previous iteration.
|
||||
// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
|
||||
// d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
|
||||
// d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
|
||||
// d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
|
||||
// d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
|
||||
// d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
|
||||
// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
|
||||
// inp[2:3]*r^2
|
||||
WORD $0xf268888e // vadd.i32 d24,d24,d14
|
||||
WORD $0xf3a9eae0 // vmull.u32 q7,d25,d0[1]
|
||||
WORD $0xf264488a // vadd.i32 d20,d20,d10
|
||||
WORD $0xf3a5aae0 // vmull.u32 q5,d21,d0[1]
|
||||
WORD $0xf26aa8a0 // vadd.i32 d26,d26,d16
|
||||
WORD $0xf3eb0ae0 // vmull.u32 q8,d27,d0[1]
|
||||
WORD $0xf3a7e2e1 // vmlal.u32 q7,d23,d1[1]
|
||||
WORD $0xf266688c // vadd.i32 d22,d22,d12
|
||||
WORD $0xf3a7cae0 // vmull.u32 q6,d23,d0[1]
|
||||
WORD $0xf26cc8a2 // vadd.i32 d28,d28,d18
|
||||
WORD $0xf3ed2ae0 // vmull.u32 q9,d29,d0[1]
|
||||
SUB.S $64, R2, R2
|
||||
WORD $0xf3ada2e2 // vmlal.u32 q5,d29,d2[1]
|
||||
MOVW.LO R5, R4
|
||||
WORD $0xf3e902e1 // vmlal.u32 q8,d25,d1[1]
|
||||
WORD $0xf4a788bf // vld1.32 d8[1],[r7,:32]
|
||||
WORD $0xf3a5c2e1 // vmlal.u32 q6,d21,d1[1]
|
||||
WORD $0xf3eb22e1 // vmlal.u32 q9,d27,d1[1]
|
||||
WORD $0xf3aba2e4 // vmlal.u32 q5,d27,d4[1]
|
||||
WORD $0xf3e702e3 // vmlal.u32 q8,d23,d3[1]
|
||||
WORD $0xf3e922e3 // vmlal.u32 q9,d25,d3[1]
|
||||
WORD $0xf3adc2e4 // vmlal.u32 q6,d29,d4[1]
|
||||
WORD $0xf3a5e2e3 // vmlal.u32 q7,d21,d3[1]
|
||||
WORD $0xf3e502e5 // vmlal.u32 q8,d21,d5[1]
|
||||
WORD $0xf3a9a2e6 // vmlal.u32 q5,d25,d6[1]
|
||||
WORD $0xf3e722e5 // vmlal.u32 q9,d23,d5[1]
|
||||
WORD $0xf3abc2e6 // vmlal.u32 q6,d27,d6[1]
|
||||
WORD $0xf3ade2e6 // vmlal.u32 q7,d29,d6[1]
|
||||
WORD $0xf3ed02e8 // vmlal.u32 q8,d29,d8[1]
|
||||
WORD $0xf3a7a2e8 // vmlal.u32 q5,d23,d8[1]
|
||||
WORD $0xf3e522e7 // vmlal.u32 q9,d21,d7[1]
|
||||
WORD $0xf3a9c2e8 // vmlal.u32 q6,d25,d8[1]
|
||||
WORD $0xf3abe2e8 // vmlal.u32 q7,d27,d8[1]
|
||||
WORD $0xf464518f // vld4.32 {d21,d23,d25,d27},[r4]
|
||||
ADD $64, R4, R4
|
||||
// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
|
||||
// (hash+inp[0:1])*r^4 and accumulate
|
||||
WORD $0xf3ea02c0 // vmlal.u32 q8,d26,d0[0]
|
||||
WORD $0xf3a4a2c0 // vmlal.u32 q5,d20,d0[0]
|
||||
WORD $0xf3ec22c0 // vmlal.u32 q9,d28,d0[0]
|
||||
WORD $0xf3a6c2c0 // vmlal.u32 q6,d22,d0[0]
|
||||
WORD $0xf3a8e2c0 // vmlal.u32 q7,d24,d0[0]
|
||||
WORD $0xf4a6883f // vld1.32 d8[0],[r6,:32]
|
||||
WORD $0xf3e802c1 // vmlal.u32 q8,d24,d1[0]
|
||||
WORD $0xf3aca2c2 // vmlal.u32 q5,d28,d2[0]
|
||||
WORD $0xf3ea22c1 // vmlal.u32 q9,d26,d1[0]
|
||||
WORD $0xf3a4c2c1 // vmlal.u32 q6,d20,d1[0]
|
||||
WORD $0xf3a6e2c1 // vmlal.u32 q7,d22,d1[0]
|
||||
WORD $0xf3e602c3 // vmlal.u32 q8,d22,d3[0]
|
||||
WORD $0xf3aaa2c4 // vmlal.u32 q5,d26,d4[0]
|
||||
WORD $0xf3e822c3 // vmlal.u32 q9,d24,d3[0]
|
||||
WORD $0xf3acc2c4 // vmlal.u32 q6,d28,d4[0]
|
||||
WORD $0xf3a4e2c3 // vmlal.u32 q7,d20,d3[0]
|
||||
WORD $0xf3e402c5 // vmlal.u32 q8,d20,d5[0]
|
||||
WORD $0xf3a8a2c6 // vmlal.u32 q5,d24,d6[0]
|
||||
WORD $0xf3e622c5 // vmlal.u32 q9,d22,d5[0]
|
||||
WORD $0xf3aac2c6 // vmlal.u32 q6,d26,d6[0]
|
||||
WORD $0xf3ec02c8 // vmlal.u32 q8,d28,d8[0]
|
||||
WORD $0xf3ace2c6 // vmlal.u32 q7,d28,d6[0]
|
||||
WORD $0xf3a6a2c8 // vmlal.u32 q5,d22,d8[0]
|
||||
WORD $0xf3e422c7 // vmlal.u32 q9,d20,d7[0]
|
||||
WORD $0xf2c0c651 // vmov.i32 q14,#1<<24
|
||||
WORD $0xf3a8c2c8 // vmlal.u32 q6,d24,d8[0]
|
||||
WORD $0xf3aae2c8 // vmlal.u32 q7,d26,d8[0]
|
||||
WORD $0xf461418f // vld4.32 {d20,d22,d24,d26},[r1]
|
||||
ADD $64, R1, R1
|
||||
// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
|
||||
// lazy reduction interleaved with base 2^32 -> base 2^26 of
|
||||
// inp[0:3] previously loaded to q10-q13 and smashed to q10-q14.
|
||||
WORD $0xf3e6e0f0 // vshr.u64 q15,q8,#26
|
||||
WORD $0xf3fa0220 // vmovn.i64 d16,q8
|
||||
WORD $0xf3a680da // vshr.u64 q4,q5,#26
|
||||
WORD $0xf3baa20a // vmovn.i64 d10,q5
|
||||
WORD $0xf27228ee // vadd.i64 q9,q9,q15
|
||||
WORD $0xf3c7073c // vbic.i32 d16,#0xfc000000
|
||||
WORD $0xf3f8c47a // vsri.u32 q14,q13,#8
|
||||
WORD $0xf23cc848 // vadd.i64 q6,q6,q4
|
||||
WORD $0xf2f2a57a // vshl.u32 q13,q13,#18
|
||||
WORD $0xf387a73c // vbic.i32 d10,#0xfc000000
|
||||
WORD $0xf2e6e832 // vshrn.u64 d30,q9,#26
|
||||
WORD $0xf3fa2222 // vmovn.i64 d18,q9
|
||||
WORD $0xf3a680dc // vshr.u64 q4,q6,#26
|
||||
WORD $0xf3bac20c // vmovn.i64 d12,q6
|
||||
WORD $0xf23ee848 // vadd.i64 q7,q7,q4
|
||||
WORD $0xf3f2a478 // vsri.u32 q13,q12,#14
|
||||
WORD $0xf3c7273c // vbic.i32 d18,#0xfc000000
|
||||
WORD $0xf2ec8578 // vshl.u32 q12,q12,#12
|
||||
WORD $0xf387c73c // vbic.i32 d12,#0xfc000000
|
||||
WORD $0xf22aa82e // vadd.i32 d10,d10,d30
|
||||
WORD $0xf2e2e53e // vshl.u32 d30,d30,#2
|
||||
WORD $0xf3c7a77c // vbic.i32 q13,#0xfc000000
|
||||
WORD $0xf2a6881e // vshrn.u64 d8,q7,#26
|
||||
WORD $0xf3bae20e // vmovn.i64 d14,q7
|
||||
WORD $0xf3aaa02e // vaddl.u32 q5,d10,d30
|
||||
WORD $0xf3ec8476 // vsri.u32 q12,q11,#20
|
||||
WORD $0xf2600888 // vadd.i32 d16,d16,d8
|
||||
WORD $0xf2e66576 // vshl.u32 q11,q11,#6
|
||||
WORD $0xf387e73c // vbic.i32 d14,#0xfc000000
|
||||
WORD $0xf3c7877c // vbic.i32 q12,#0xfc000000
|
||||
WORD $0xf2e6e81a // vshrn.u64 d30,q5,#26
|
||||
WORD $0xf3baa20a // vmovn.i64 d10,q5
|
||||
WORD $0xf3e66474 // vsri.u32 q11,q10,#26
|
||||
WORD $0xf3c7477c // vbic.i32 q10,#0xfc000000
|
||||
WORD $0xf3a68030 // vshr.u32 d8,d16,#26
|
||||
WORD $0xf3c7073c // vbic.i32 d16,#0xfc000000
|
||||
WORD $0xf387a73c // vbic.i32 d10,#0xfc000000
|
||||
WORD $0xf22cc82e // vadd.i32 d12,d12,d30
|
||||
WORD $0xf2622888 // vadd.i32 d18,d18,d8
|
||||
WORD $0xf3c7677c // vbic.i32 q11,#0xfc000000
|
||||
B.HI lb_Loop_neon
|
||||
lb_Lskip_loop:
|
||||
// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
|
||||
// multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
|
||||
ADD $(48+0*9*4), R0, R7
|
||||
ADD $(48+1*9*4), R0, R6
|
||||
ADD.S $32, R2, R2
|
||||
MOVW.NE $0, R2
|
||||
B.NE lb_Long_tail
|
||||
WORD $0xf268988e // vadd.i32 d25,d24,d14
|
||||
WORD $0xf264588a // vadd.i32 d21,d20,d10
|
||||
WORD $0xf26ab8a0 // vadd.i32 d27,d26,d16
|
||||
WORD $0xf266788c // vadd.i32 d23,d22,d12
|
||||
WORD $0xf26cd8a2 // vadd.i32 d29,d28,d18
|
||||
lb_Long_tail:
|
||||
WORD $0xf4a70b8d // vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
|
||||
WORD $0xf4a60b0d // vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
|
||||
WORD $0xf268888e // vadd.i32 d24,d24,d14
|
||||
WORD $0xf3a9ec80 // vmull.u32 q7,d25,d0
|
||||
WORD $0xf264488a // vadd.i32 d20,d20,d10
|
||||
WORD $0xf3a5ac80 // vmull.u32 q5,d21,d0
|
||||
WORD $0xf26aa8a0 // vadd.i32 d26,d26,d16
|
||||
WORD $0xf3eb0c80 // vmull.u32 q8,d27,d0
|
||||
WORD $0xf266688c // vadd.i32 d22,d22,d12
|
||||
WORD $0xf3a7cc80 // vmull.u32 q6,d23,d0
|
||||
WORD $0xf26cc8a2 // vadd.i32 d28,d28,d18
|
||||
WORD $0xf3ed2c80 // vmull.u32 q9,d29,d0
|
||||
WORD $0xf3ada882 // vmlal.u32 q5,d29,d2
|
||||
WORD $0xf4a74b8d // vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
|
||||
WORD $0xf3e90881 // vmlal.u32 q8,d25,d1
|
||||
WORD $0xf4a64b0d // vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
|
||||
WORD $0xf3a5c881 // vmlal.u32 q6,d21,d1
|
||||
WORD $0xf3eb2881 // vmlal.u32 q9,d27,d1
|
||||
WORD $0xf3a7e881 // vmlal.u32 q7,d23,d1
|
||||
WORD $0xf3e70883 // vmlal.u32 q8,d23,d3
|
||||
WORD $0xf4a788bf // vld1.32 d8[1],[r7,:32]
|
||||
WORD $0xf3aba884 // vmlal.u32 q5,d27,d4
|
||||
WORD $0xf4a6883f // vld1.32 d8[0],[r6,:32]
|
||||
WORD $0xf3e92883 // vmlal.u32 q9,d25,d3
|
||||
WORD $0xf3adc884 // vmlal.u32 q6,d29,d4
|
||||
WORD $0xf3a5e883 // vmlal.u32 q7,d21,d3
|
||||
WORD $0xf3e50885 // vmlal.u32 q8,d21,d5
|
||||
ADD.NE $(48+2*9*4), R0, R7
|
||||
WORD $0xf3a9a886 // vmlal.u32 q5,d25,d6
|
||||
ADD.NE $(48+3*9*4), R0, R6
|
||||
WORD $0xf3e72885 // vmlal.u32 q9,d23,d5
|
||||
WORD $0xf3abc886 // vmlal.u32 q6,d27,d6
|
||||
WORD $0xf3ade886 // vmlal.u32 q7,d29,d6
|
||||
WORD $0xf3ed0888 // vmlal.u32 q8,d29,d8
|
||||
WORD $0xf2300150 // vorn q0,q0,q0
|
||||
WORD $0xf3a7a888 // vmlal.u32 q5,d23,d8
|
||||
WORD $0xf39a00d0 // vshr.u64 q0,q0,#38
|
||||
WORD $0xf3e52887 // vmlal.u32 q9,d21,d7
|
||||
WORD $0xf3a9c888 // vmlal.u32 q6,d25,d8
|
||||
WORD $0xf3abe888 // vmlal.u32 q7,d27,d8
|
||||
B.EQ lb_Lshort_tail
|
||||
// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
|
||||
// (hash+inp[0:1])*r^4:r^3 and accumulate
|
||||
WORD $0xf4a70b8d // vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
|
||||
WORD $0xf4a60b0d // vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
|
||||
WORD $0xf3a8e880 // vmlal.u32 q7,d24,d0
|
||||
WORD $0xf3a4a880 // vmlal.u32 q5,d20,d0
|
||||
WORD $0xf3ea0880 // vmlal.u32 q8,d26,d0
|
||||
WORD $0xf3a6c880 // vmlal.u32 q6,d22,d0
|
||||
WORD $0xf3ec2880 // vmlal.u32 q9,d28,d0
|
||||
WORD $0xf3aca882 // vmlal.u32 q5,d28,d2
|
||||
WORD $0xf4a74b8d // vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
|
||||
WORD $0xf3e80881 // vmlal.u32 q8,d24,d1
|
||||
WORD $0xf4a64b0d // vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
|
||||
WORD $0xf3a4c881 // vmlal.u32 q6,d20,d1
|
||||
WORD $0xf3ea2881 // vmlal.u32 q9,d26,d1
|
||||
WORD $0xf3a6e881 // vmlal.u32 q7,d22,d1
|
||||
WORD $0xf3e60883 // vmlal.u32 q8,d22,d3
|
||||
WORD $0xf4a788bf // vld1.32 d8[1],[r7,:32]
|
||||
WORD $0xf3aaa884 // vmlal.u32 q5,d26,d4
|
||||
WORD $0xf4a6883f // vld1.32 d8[0],[r6,:32]
|
||||
WORD $0xf3e82883 // vmlal.u32 q9,d24,d3
|
||||
WORD $0xf3acc884 // vmlal.u32 q6,d28,d4
|
||||
WORD $0xf3a4e883 // vmlal.u32 q7,d20,d3
|
||||
WORD $0xf3e40885 // vmlal.u32 q8,d20,d5
|
||||
WORD $0xf3a8a886 // vmlal.u32 q5,d24,d6
|
||||
WORD $0xf3e62885 // vmlal.u32 q9,d22,d5
|
||||
WORD $0xf3aac886 // vmlal.u32 q6,d26,d6
|
||||
WORD $0xf3ace886 // vmlal.u32 q7,d28,d6
|
||||
WORD $0xf3ec0888 // vmlal.u32 q8,d28,d8
|
||||
WORD $0xf2300150 // vorn q0,q0,q0
|
||||
WORD $0xf3a6a888 // vmlal.u32 q5,d22,d8
|
||||
WORD $0xf39a00d0 // vshr.u64 q0,q0,#38
|
||||
WORD $0xf3e42887 // vmlal.u32 q9,d20,d7
|
||||
WORD $0xf3a8c888 // vmlal.u32 q6,d24,d8
|
||||
WORD $0xf3aae888 // vmlal.u32 q7,d26,d8
|
||||
lb_Lshort_tail:
|
||||
// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
|
||||
// horizontal addition
|
||||
WORD $0xf27008a1 // vadd.i64 d16,d16,d17
|
||||
WORD $0xf23aa80b // vadd.i64 d10,d10,d11
|
||||
WORD $0xf27228a3 // vadd.i64 d18,d18,d19
|
||||
WORD $0xf23cc80d // vadd.i64 d12,d12,d13
|
||||
WORD $0xf23ee80f // vadd.i64 d14,d14,d15
|
||||
// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
|
||||
// lazy reduction, but without narrowing
|
||||
WORD $0xf3e6e0f0 // vshr.u64 q15,q8,#26
|
||||
WORD $0xf24001d0 // vand.i64 q8,q8,q0
|
||||
WORD $0xf3a680da // vshr.u64 q4,q5,#26
|
||||
WORD $0xf20aa150 // vand.i64 q5,q5,q0
|
||||
WORD $0xf27228ee // vadd.i64 q9,q9,q15
|
||||
WORD $0xf23cc848 // vadd.i64 q6,q6,q4
|
||||
WORD $0xf3e6e0f2 // vshr.u64 q15,q9,#26
|
||||
WORD $0xf24221d0 // vand.i64 q9,q9,q0
|
||||
WORD $0xf3a680dc // vshr.u64 q4,q6,#26
|
||||
WORD $0xf20cc150 // vand.i64 q6,q6,q0
|
||||
WORD $0xf23ee848 // vadd.i64 q7,q7,q4
|
||||
WORD $0xf23aa86e // vadd.i64 q5,q5,q15
|
||||
WORD $0xf2c2e5fe // vshl.u64 q15,q15,#2
|
||||
WORD $0xf3a680de // vshr.u64 q4,q7,#26
|
||||
WORD $0xf20ee150 // vand.i64 q7,q7,q0
|
||||
WORD $0xf23aa86e // vadd.i64 q5,q5,q15
|
||||
WORD $0xf27008c8 // vadd.i64 q8,q8,q4
|
||||
WORD $0xf3e6e0da // vshr.u64 q15,q5,#26
|
||||
WORD $0xf20aa150 // vand.i64 q5,q5,q0
|
||||
WORD $0xf3a680f0 // vshr.u64 q4,q8,#26
|
||||
WORD $0xf24001d0 // vand.i64 q8,q8,q0
|
||||
WORD $0xf23cc86e // vadd.i64 q6,q6,q15
|
||||
WORD $0xf27228c8 // vadd.i64 q9,q9,q4
|
||||
CMP $0, R2
|
||||
B.NE lb_Leven
|
||||
// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
|
||||
// store hash value
|
||||
WORD $0xf480ab4d // vst4.32 {d10[0],d12[0],d14[0],d16[0]},[r0]!
|
||||
WORD $0xf4c0280f // vst1.32 {d18[0]},[r0]
|
||||
RET
|
||||
lb_LOPENSSL_armcap:
|
||||
|
||||
Reference in new issue
Block a user