diff --git a/CMakeLists.txt b/CMakeLists.txt index 8797a4b..68bfece 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -97,6 +97,9 @@ if (DEV_DEBUG) add_definitions(-DDEV_DEBUG) endif() +add_subdirectory(external/src/mx25519) +set(LIBS ${LIBS} mx25519) + include(cmake/flags.cmake) set(HEADERS @@ -105,6 +108,7 @@ set(HEADERS external/src/hardforks/hardforks.h src/block_cache.h src/block_template.h + src/carrot_crypto.h src/common.h src/console_commands.h src/crypto.h @@ -123,6 +127,7 @@ set(HEADERS src/pool_block.h src/pool_block_parser.inl src/pow_hash.h + src/protocol_tx_hash.h src/side_chain.h src/stratum_server.h src/tcp_server.h @@ -140,6 +145,7 @@ set(SOURCES external/src/hardforks/hardforks.cpp src/block_cache.cpp src/block_template.cpp + src/carrot_crypto.cpp src/console_commands.cpp src/crypto.cpp src/json_rpc_request.cpp @@ -157,6 +163,7 @@ set(SOURCES src/params.cpp src/pool_block.cpp src/pow_hash.cpp + src/protocol_tx_hash.cpp src/side_chain.cpp src/stratum_server.cpp src/tcp_server.cpp @@ -234,6 +241,7 @@ include_directories(src) include_directories(external/src) include_directories(external/src/crypto) include_directories(external/src/cryptonote) +include_directories(external/src/mx25519) include_directories(${UV_INCLUDE_DIR}) include_directories(external/src/cppzmq) include_directories(${ZMQ_INCLUDE_DIR}) diff --git a/external/src/mx25519/CMakeLists.txt b/external/src/mx25519/CMakeLists.txt new file mode 100644 index 0000000..cc91726 --- /dev/null +++ b/external/src/mx25519/CMakeLists.txt @@ -0,0 +1,47 @@ +cmake_minimum_required(VERSION 3.5) + +set(mx25519_sources + portable/scalarmult.c + cpu.c + impl.c + mx25519.c + scalar.c + platform.c) + +if(NOT ARCH_ID) + if(CMAKE_SYSTEM_PROCESSOR STREQUAL "") + set(CMAKE_SYSTEM_PROCESSOR ${CMAKE_HOST_SYSTEM_PROCESSOR}) + endif() + string(TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" ARCH_ID) +endif() + +if(NOT ARM_ID) + set(ARM_ID "${ARCH_ID}") +endif() + +message(STATUS "mx25519 build architecture: ${ARCH_ID}") + +# AMD64 +if(ARCH_ID STREQUAL "x86_64" OR ARCH_ID STREQUAL "x86-64" OR ARCH_ID STREQUAL "amd64") + if(MSVC) + enable_language(ASM_MASM) + list(APPEND mx25519_sources amd64/scalarmult_masm.asm) + set_property(SOURCE amd64/scalarmult_masm.asm PROPERTY LANGUAGE ASM_MASM) + else() + list(APPEND mx25519_sources amd64/scalarmult_gnu.S) + set_property(SOURCE amd64/scalarmult_gnu.S PROPERTY LANGUAGE C) + set_property(SOURCE amd64/scalarmult_gnu.S PROPERTY XCODE_EXPLICIT_FILE_TYPE sourcecode.asm) + endif() +endif() + +# ARM64 +if(ARM_ID STREQUAL "aarch64" OR ARM_ID STREQUAL "arm64" OR ARM_ID STREQUAL "armv8-a") + list(APPEND mx25519_sources arm64/scalarmult.S) + set_property(SOURCE arm64/scalarmult.S PROPERTY LANGUAGE C) + set_property(SOURCE arm64/scalarmult.S PROPERTY XCODE_EXPLICIT_FILE_TYPE sourcecode.asm) +endif() + +add_library(mx25519 STATIC ${mx25519_sources}) +set_property(TARGET mx25519 PROPERTY POSITION_INDEPENDENT_CODE ON) +target_include_directories(mx25519 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_compile_definitions(mx25519 PRIVATE MX25519_STATIC) diff --git a/external/src/mx25519/amd64/constants.inc b/external/src/mx25519/amd64/constants.inc new file mode 100644 index 0000000..a57b5e5 --- /dev/null +++ b/external/src/mx25519/amd64/constants.inc @@ -0,0 +1,30 @@ +;# Copyright (c) 2022 tevador +;# +;# This file is part of mx25519, which is released under LGPLv3. +;# See LICENSE for full license details. + +invtable: +;# square times, +;# skip mul, +;# mulsource, +;# destination +db 1, 1, 0, 1 +db 2, 0, 2, 2 +db 0, 0, 1, 1 +db 1, 0, 2, 2 +db 5, 0, 2, 2 +db 10, 0, 2, 3 +db 20, 0, 3, 0 +db 10, 0, 2, 2 +db 50, 0, 2, 3 +db 100, 0, 3, 0 +db 50, 0, 2, 0 +db 5, 0, 1, 0 +db 0, 0, 0, 0 + +p0: +db 237, 255, 255, 255, 255, 255, 255, 255 +p12: +db 255, 255, 255, 255, 255, 255, 255, 255 +p3: +db 255, 255, 255, 255, 255, 255, 255, 127 diff --git a/external/src/mx25519/amd64/scalarmult.h b/external/src/mx25519/amd64/scalarmult.h new file mode 100644 index 0000000..e677532 --- /dev/null +++ b/external/src/mx25519/amd64/scalarmult.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#ifndef AMD64_SCALARMULT_H +#define AMD64_SCALARMULT_H + +#include + +void mx25519_scalarmult_amd64(uint8_t* q, + const uint8_t* n, + const uint8_t* p); + +void mx25519_scalarmult_amd64x(uint8_t* q, + const uint8_t* n, + const uint8_t* p); + +#endif diff --git a/external/src/mx25519/amd64/scalarmult_compat.inc b/external/src/mx25519/amd64/scalarmult_compat.inc new file mode 100644 index 0000000..48ddc8d --- /dev/null +++ b/external/src/mx25519/amd64/scalarmult_compat.inc @@ -0,0 +1,1993 @@ +;# Copyright (c) 2022 tevador +;# +;# This file is part of mx25519, which is released under LGPLv3. +;# See LICENSE for full license details. +;# +;# +;# Parts of this file are derived from a work with the following license: +;# +;# Copyright (c) 2020, Kaushik Nath and Palash Sarkar. +;# +;# Permission to use this code is granted. +;# +;# Redistribution and use in source and binary forms, with or without +;# modification, are permitted provided that the following conditions are +;# met: +;# +;# * Redistributions of source code must retain the above copyright notice, +;# this list of conditions and the following disclaimer. +;# +;# * Redistributions in binary form must reproduce the above copyright +;# notice, this list of conditions and the following disclaimer in the +;# documentation and/or other materials provided with the distribution. +;# +;# * The names of the contributors may not be used to endorse or promote +;# products derived from this software without specific prior written +;# permission. +;# +;# THIS SOFTWARE IS PROVIDED BY THE AUTHORS ""AS IS"" AND ANY EXPRESS OR +;# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +;# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +;# IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +;# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +;# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +;# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +;# THEORY LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +;# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +;# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +;# arguments: +;# rdi -> public key address (out) +;# rsi -> private key address (in) +;# rdx -> base point address (in) + +sub rsp, 392 + +;# stack layout (offsets from rsp): +;# 0 T1 +;# 32 T2 +;# 64 T3 +;# 96 T4 +;# 128 X1 +;# 160 X2 +;# 192 X3 +;# 224 Z2 +;# 256 Z3 +;# 288 saved registers: rbx, rbp, rdi, rsi, r12, r13, r14, r15 +;# 352 key bit index (1 byte) +;# 353 swap flag (1 byte) +;# 354 current key byte value (1 byte) +;# 356 key byte index (4 bytes) + +mov qword ptr [rsp+288], rbx +mov qword ptr [rsp+296], rbp +mov qword ptr [rsp+304], rdi +mov qword ptr [rsp+312], rsi +mov qword ptr [rsp+320], r12 +mov qword ptr [rsp+328], r13 +mov qword ptr [rsp+336], r14 +mov qword ptr [rsp+344], r15 + +;# starting from bit 6 of the 31st byte +mov eax, 31 +mov byte ptr [rsp+352], 6 +mov byte ptr [rsp+353], 0 +mov dword ptr [rsp+356], eax + +;# load XP +mov r8, qword ptr [rdx] +mov r9, qword ptr [rdx+8] +mov r10, qword ptr [rdx+16] +mov r11, qword ptr [rdx+24] + +;# reduce XP mod 2^255-19 +btr r11, 63 +mov rbx, r8 +mov rcx, r9 +mov rdx, r10 +mov rdi, r11 +sub r8, qword ptr [p0+REG_REL] +sbb r9, qword ptr [p12+REG_REL] +sbb r10, qword ptr [p12+REG_REL] +sbb r11, qword ptr [p3+REG_REL] +bt r11, 63 +cmovb r8, rbx +cmovb r9, rcx +cmovb r10, rdx +cmovb r11, rdi + +;# X1 ← XP, X3 ← XP +mov qword ptr [rsp+128], r8 +mov qword ptr [rsp+192], r8 +mov qword ptr [rsp+136], r9 +mov qword ptr [rsp+200], r9 +mov qword ptr [rsp+144], r10 +mov qword ptr [rsp+208], r10 +mov qword ptr [rsp+152], r11 +mov qword ptr [rsp+216], r11 + +;# X2 ← 1 +mov qword ptr [rsp+160], 1 +mov qword ptr [rsp+168], 0 +mov qword ptr [rsp+176], 0 +mov qword ptr [rsp+184], 0 + +;# Z2 ← 0 +mov qword ptr [rsp+224], 0 +mov qword ptr [rsp+232], 0 +mov qword ptr [rsp+240], 0 +mov qword ptr [rsp+248], 0 + +;# Z3 ← 1 +mov qword ptr [rsp+256], 1 +mov qword ptr [rsp+264], 0 +mov qword ptr [rsp+272], 0 +mov qword ptr [rsp+280], 0 + +;# Montgomery ladder loop + +ALIGN 16 +loop_byte: + +;# rsi = private key address +;# eax = key byte index +movzx ecx, byte ptr [rsi+rax] +mov byte ptr [rsp+354], cl + +;# +;# Montgomery ladder step +;# +;# Reduction ideas for addition and subtraction are taken from the 64-bit implementation +;# "amd64-64" of the work "https://link.springer.com/article/10.1007/s13389-012-0027-1" +;# +;# T1 ← X2 + Z2 +;# T2 ← X2 - Z2 +;# T3 ← X3 + Z3 +;# T4 ← X3 - Z3 +;# Z3 ← T2 · T3 +;# X3 ← T1 · T4 +;# +;# bit ← n[i] +;# select ← bit ⊕ prevbit +;# prevbit ← bit +;# CSelect(T1,T3,select): if (select == 1) {T1 = T3} +;# CSelect(T2,T4,select): if (select == 1) {T2 = T4} +;# +;# T2 ← T2^2 +;# T1 ← T1^2 +;# X3 ← X3 + Z3 +;# Z3 ← X3 - Z3 +;# Z3 ← Z3^2 +;# X3 ← X3^2 +;# T3 ← T1 - T2 +;# T4 ← ((A + 2)/4) · T3 +;# T4 ← T4 + T2 +;# X2 ← T1 · T2 +;# Z2 ← T3 · T4 +;# Z3 ← Z3 · X1 +;# + +ALIGN 16 +loop_bit: + +;# X2 +mov r8, qword ptr [rsp+160] +mov r9, qword ptr [rsp+168] +mov r10, qword ptr [rsp+176] +mov r11, qword ptr [rsp+184] + +;# copy X2 +mov rax, r8 +mov rbx, r9 +mov rbp, r10 +mov rsi, r11 + +;# T1 ← X2 + Z2 +add r8, qword ptr [rsp+224] +adc r9, qword ptr [rsp+232] +adc r10, qword ptr [rsp+240] +adc r11, qword ptr [rsp+248] + +mov rdi, 0 +mov rcx, 38 +cmovae rcx, rdi + +add r8, rcx +adc r9, rdi +adc r10, rdi +adc r11, rdi + +cmovb rdi, rcx +add r8, rdi + +mov qword ptr [rsp+0], r8 +mov qword ptr [rsp+8], r9 +mov qword ptr [rsp+16], r10 +mov qword ptr [rsp+24], r11 + +;# T2 ← X2 - Z2 +sub rax, qword ptr [rsp+224] +sbb rbx, qword ptr [rsp+232] +sbb rbp, qword ptr [rsp+240] +sbb rsi, qword ptr [rsp+248] +mov edi, 0 +mov rcx, 38 +cmovae rcx, rdi +sub rax, rcx +sbb rbx, rdi +sbb rbp, rdi +sbb rsi, rdi +cmovb rdi, rcx +sub rax, rdi +mov qword ptr [rsp+32], rax +mov qword ptr [rsp+40], rbx +mov qword ptr [rsp+48], rbp +mov qword ptr [rsp+56], rsi + +;# X3 +mov r8, qword ptr [rsp+192] +mov r9, qword ptr [rsp+200] +mov r10, qword ptr [rsp+208] +mov r11, qword ptr [rsp+216] + +;# copy X3 +mov rax, r8 +mov rbx, r9 +mov rbp, r10 +mov rsi, r11 + +;# T3 ← X3 + Z3 +add r8, qword ptr [rsp+256] +adc r9, qword ptr [rsp+264] +adc r10, qword ptr [rsp+272] +adc r11, qword ptr [rsp+280] +mov edi, 0 +mov rcx, 38 +cmovae rcx, rdi +add r8, rcx +adc r9, rdi +adc r10, rdi +adc r11, rdi +cmovb rdi, rcx +add r8, rdi +mov qword ptr [rsp+64], r8 +mov qword ptr [rsp+72], r9 +mov qword ptr [rsp+80], r10 +mov qword ptr [rsp+88], r11 + +;# T4 ← X3 - Z3 +sub rax, qword ptr [rsp+256] +sbb rbx, qword ptr [rsp+264] +sbb rbp, qword ptr [rsp+272] +sbb rsi, qword ptr [rsp+280] +mov edi, 0 +mov rcx, 38 +cmovae rcx, rdi +sub rax, rcx +sbb rbx, rdi +sbb rbp, rdi +sbb rsi, rdi +cmovb rdi, rcx +sub rax, rdi +mov qword ptr [rsp+96], rax +mov qword ptr [rsp+104], rbx +mov qword ptr [rsp+112], rbp +mov qword ptr [rsp+120], rsi + +;# Z3 ← T2 · T3 +mov rax, qword ptr [rsp+40] +mul qword ptr [rsp+88] +mov r8, rax +xor r9d, r9d +mov r10, rdx +xor r11d, r11d +mov rax, qword ptr [rsp+48] +mul qword ptr [rsp+80] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+56] +mul qword ptr [rsp+72] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+48] +mul qword ptr [rsp+88] +add r10, rax +adc r11, 0 +mov r12, rdx +xor r13d, r13d +mov rax, qword ptr [rsp+56] +mul qword ptr [rsp+80] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, 38 +mul r10 +imul r11, r11, 38 +mov r10, rax +add r11, rdx +mov rax, qword ptr [rsp+56] +mul qword ptr [rsp+88] +add r12, rax +adc r13, 0 +mov rax, 38 +mul rdx +mov r14, rax +mov r15, rdx +mov rax, 38 +mul r12 +imul r13, r13, 38 +mov r12, rax +add r13, rdx +mov rax, qword ptr [rsp+32] +mul qword ptr [rsp+88] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+40] +mul qword ptr [rsp+80] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+48] +mul qword ptr [rsp+72] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+56] +mul qword ptr [rsp+64] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, 38 +mul r8 +imul r9, r9, 38 +mov r8, rax +add r9, rdx +mov rax, qword ptr [rsp+32] +mul qword ptr [rsp+64] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+32] +mul qword ptr [rsp+72] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, qword ptr [rsp+40] +mul qword ptr [rsp+64] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, qword ptr [rsp+32] +mul qword ptr [rsp+80] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, qword ptr [rsp+40] +mul qword ptr [rsp+72] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, qword ptr [rsp+48] +mul qword ptr [rsp+64] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +add r10, r9 +adc r11, 0 +add r12, r11 +adc r13, 0 +add r14, r13 +adc r15, 0 +shld r15, r14, 1 +btr r14, 63 +imul r15, r15, 19 +add r8, r15 +adc r10, 0 +adc r12, 0 +adc r14, 0 +mov qword ptr [rsp+256], r8 +mov qword ptr [rsp+264], r10 +mov qword ptr [rsp+272], r12 +mov qword ptr [rsp+280], r14 + +;# X3 ← T1 · T4 +mov rax, qword ptr [rsp+8] +mul qword ptr [rsp+120] +mov r8, rax +xor r9d, r9d +mov r10, rdx +xor r11d, r11d +mov rax, qword ptr [rsp+16] +mul qword ptr [rsp+112] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+24] +mul qword ptr [rsp+104] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+16] +mul qword ptr [rsp+120] +add r10, rax +adc r11, 0 +mov r12, rdx +xor r13d, r13d +mov rax, qword ptr [rsp+24] +mul qword ptr [rsp+112] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, 38 +mul r10 +imul r11, r11, 38 +mov r10, rax +add r11, rdx +mov rax, qword ptr [rsp+24] +mul qword ptr [rsp+120] +add r12, rax +adc r13, 0 +mov rax, 38 +mul rdx +mov r14, rax +mov r15, rdx +mov rax, 38 +mul r12 +imul r13, r13, 38 +mov r12, rax +add r13, rdx +mov rax, qword ptr [rsp+0] +mul qword ptr [rsp+120] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+8] +mul qword ptr [rsp+112] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+16] +mul qword ptr [rsp+104] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+24] +mul qword ptr [rsp+96] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, 38 +mul r8 +imul r9, r9, 38 +mov r8, rax +add r9, rdx +mov rax, qword ptr [rsp+0] +mul qword ptr [rsp+96] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+0] +mul qword ptr [rsp+104] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, qword ptr [rsp+8] +mul qword ptr [rsp+96] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, qword ptr [rsp+0] +mul qword ptr [rsp+112] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, qword ptr [rsp+8] +mul qword ptr [rsp+104] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, qword ptr [rsp+16] +mul qword ptr [rsp+96] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +add r10, r9 +adc r11, 0 +add r12, r11 +adc r13, 0 +add r14, r13 +adc r15, 0 +shld r15, r14, 1 +btr r14, 63 +imul r15, r15, 19 +add r8, r15 +adc r10, 0 +adc r12, 0 +adc r14, 0 +mov qword ptr [rsp+192], r8 +mov qword ptr [rsp+200], r10 +mov qword ptr [rsp+208], r12 +mov qword ptr [rsp+216], r14 + +movzx ecx, byte ptr [rsp+352] +movzx ebx, byte ptr [rsp+354] +shr bl, cl +and bl, 1 +mov cl, bl +xor bl, byte ptr [rsp+353] +mov byte ptr [rsp+353], cl +cmp bl, 1 + +;# CSelect(T1,T3,select) +mov r8, qword ptr [rsp+0] +mov r9, qword ptr [rsp+8] +mov r10, qword ptr [rsp+16] +mov r11, qword ptr [rsp+24] +mov r12, qword ptr [rsp+64] +mov r13, qword ptr [rsp+72] +mov r14, qword ptr [rsp+80] +mov r15, qword ptr [rsp+88] +cmove r8, r12 +cmove r9, r13 +cmove r10, r14 +cmove r11, r15 +mov qword ptr [rsp+0], r8 +mov qword ptr [rsp+8], r9 +mov qword ptr [rsp+16], r10 +mov qword ptr [rsp+24], r11 + +;# CSelect(T2,T4,select) +mov rbx, qword ptr [rsp+32] +mov rbp, qword ptr [rsp+40] +mov rcx, qword ptr [rsp+48] +mov rsi, qword ptr [rsp+56] +mov r12, qword ptr [rsp+96] +mov r13, qword ptr [rsp+104] +mov r14, qword ptr [rsp+112] +mov r15, qword ptr [rsp+120] +cmove rbx, r12 +cmove rbp, r13 +cmove rcx, r14 +cmove rsi, r15 + +;# T2 ← T2^2 +mov rax, rsi +mul rsi +mov r12, rax +xor r13d, r13d +mov rax, 38 +mul rdx +mov r14, rax +mov r15, rdx +mov rax, rbp +mul rsi +mov r8, rax +xor r9d, r9d +mov r10, rdx +xor r11d, r11d +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, rcx +mul rcx +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, rcx +mul rsi +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, 38 +mul r10 +imul r11, r11, 38 +mov r10, rax +add r11, rdx +mov rax, 38 +mul r12 +imul r13, r13, 38 +mov r12, rax +add r13, rdx +mov rax, rbx +mul rsi +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, rbp +mul rcx +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, 38 +mul r8 +imul r9, r9, 38 +mov r8, rax +add r9, rdx +mov rax, rbx +mul rbx +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, rbx +mul rbp +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, rbx +mul rcx +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, rbp +mul rbp +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +add r10, r9 +adc r11, 0 +add r12, r11 +adc r13, 0 +add r14, r13 +adc r15, 0 +shld r15, r14, 1 +imul r15, r15, 19 +btr r14, 63 +add r8, r15 +adc r10, 0 +adc r12, 0 +adc r14, 0 +mov qword ptr [rsp+32], r8 +mov qword ptr [rsp+40], r10 +mov qword ptr [rsp+48], r12 +mov qword ptr [rsp+56], r14 + +;# T1 ← T1^2 +mov rbx, qword ptr [rsp+0] +mov rbp, qword ptr [rsp+8] +mov rcx, qword ptr [rsp+16] +mov rsi, qword ptr [rsp+24] +mov rax, rsi +mul rsi +mov r12, rax +xor r13d, r13d +mov rax, 38 +mul rdx +mov r14, rax +mov r15, rdx +mov rax, rbp +mul rsi +mov r8, rax +xor r9d, r9d +mov r10, rdx +xor r11d, r11d +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, rcx +mul rcx +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, rcx +mul rsi +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, 38 +mul r10 +imul r11, r11, 38 +mov r10, rax +add r11, rdx +mov rax, 38 +mul r12 +imul r13, r13, 38 +mov r12, rax +add r13, rdx +mov rax, rbx +mul rsi +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, rbp +mul rcx +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, 38 +mul r8 +imul r9, r9, 38 +mov r8, rax +add r9, rdx +mov rax, rbx +mul rbx +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, rbx +mul rbp +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, rbx +mul rcx +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, rbp +mul rbp +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +add r10, r9 +adc r11, 0 +add r12, r11 +adc r13, 0 +add r14, r13 +adc r15, 0 +shld r15, r14, 1 +imul r15, r15, 19 +btr r14, 63 +add r8, r15 +adc r10, 0 +adc r12, 0 +adc r14, 0 +mov qword ptr [rsp+0], r8 +mov qword ptr [rsp+8], r10 +mov qword ptr [rsp+16], r12 +mov qword ptr [rsp+24], r14 + +;# X3 +mov r8, qword ptr [rsp+192] +mov r9, qword ptr [rsp+200] +mov r10, qword ptr [rsp+208] +mov r11, qword ptr [rsp+216] + +;# copy X3 +mov rbx, r8 +mov rbp, r9 +mov rcx, r10 +mov rsi, r11 + +;# X3 ← X3 + Z3 +add r8, qword ptr [rsp+256] +adc r9, qword ptr [rsp+264] +adc r10, qword ptr [rsp+272] +adc r11, qword ptr [rsp+280] +mov eax, 0 +mov rdx, 38 +cmovae rdx, rax +add r8, rdx +adc r9, rax +adc r10, rax +adc r11, rax +cmovb rax, rdx +add r8, rax +mov qword ptr [rsp+192], r8 +mov qword ptr [rsp+200], r9 +mov qword ptr [rsp+208], r10 +mov qword ptr [rsp+216], r11 + +;# Z3 ← X3 - Z3 +sub rbx, qword ptr [rsp+256] +sbb rbp, qword ptr [rsp+264] +sbb rcx, qword ptr [rsp+272] +sbb rsi, qword ptr [rsp+280] +mov eax, 0 +mov rdx, 38 +cmovae rdx, rax +sub rbx, rdx +sbb rbp, rax +sbb rcx, rax +sbb rsi, rax +cmovb rax, rdx +sub rbx, rax + +;# Z3 ← Z3^2 +mov rax, rsi +mul rsi +mov r12, rax +xor r13d, r13d +mov rax, 38 +mul rdx +mov r14, rax +mov r15, rdx +mov rax, rbp +mul rsi +mov r8, rax +xor r9d, r9d +mov r10, rdx +xor r11d, r11d +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, rcx +mul rcx +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, rcx +mul rsi +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, 38 +mul r10 +imul r11, r11, 38 +mov r10, rax +add r11, rdx +mov rax, 38 +mul r12 +imul r13, r13, 38 +mov r12, rax +add r13, rdx +mov rax, rbx +mul rsi +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, rbp +mul rcx +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, 38 +mul r8 +imul r9, r9, 38 +mov r8, rax +add r9, rdx +mov rax, rbx +mul rbx +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, rbx +mul rbp +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, rbx +mul rcx +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, rbp +mul rbp +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +add r10, r9 +adc r11, 0 +add r12, r11 +adc r13, 0 +add r14, r13 +adc r15, 0 +shld r15, r14, 1 +imul r15, r15, 19 +btr r14, 63 +add r8, r15 +adc r10, 0 +adc r12, 0 +adc r14, 0 +mov qword ptr [rsp+256], r8 +mov qword ptr [rsp+264], r10 +mov qword ptr [rsp+272], r12 +mov qword ptr [rsp+280], r14 + +;# X3 ← X3^2 +mov rbx, qword ptr [rsp+192] +mov rbp, qword ptr [rsp+200] +mov rcx, qword ptr [rsp+208] +mov rsi, qword ptr [rsp+216] +mov rax, rsi +mul rsi +mov r12, rax +xor r13d, r13d +mov rax, 38 +mul rdx +mov r14, rax +mov r15, rdx +mov rax, rbp +mul rsi +mov r8, rax +xor r9d, r9d +mov r10, rdx +xor r11d, r11d +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, rcx +mul rcx +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, rcx +mul rsi +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, 38 +mul r10 +imul r11, r11, 38 +mov r10, rax +add r11, rdx +mov rax, 38 +mul r12 +imul r13, r13, 38 +mov r12, rax +add r13, rdx +mov rax, rbx +mul rsi +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, rbp +mul rcx +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, 38 +mul r8 +imul r9, r9, 38 +mov r8, rax +add r9, rdx +mov rax, rbx +mul rbx +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, rbx +mul rbp +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, rbx +mul rcx +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, rbp +mul rbp +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +add r10, r9 +adc r11, 0 +add r12, r11 +adc r13, 0 +add r14, r13 +adc r15, 0 +shld r15, r14, 1 +imul r15, r15, 19 +btr r14, 63 +add r8, r15 +adc r10, 0 +adc r12, 0 +adc r14, 0 + +;# update X3 +mov qword ptr [rsp+192], r8 +mov qword ptr [rsp+200], r10 +mov qword ptr [rsp+208], r12 +mov qword ptr [rsp+216], r14 + +;# T3 ← T1 - T2 +mov rbx, qword ptr [rsp+0] +mov rbp, qword ptr [rsp+8] +mov rcx, qword ptr [rsp+16] +mov rsi, qword ptr [rsp+24] +sub rbx, qword ptr [rsp+32] +sbb rbp, qword ptr [rsp+40] +sbb rcx, qword ptr [rsp+48] +sbb rsi, qword ptr [rsp+56] +mov eax, 0 +mov rdx, 38 +cmovae rdx, rax +sub rbx, rdx +sbb rbp, rax +sbb rcx, rax +sbb rsi, rax +cmovb rax, rdx +sub rbx, rax +mov qword ptr [rsp+64], rbx +mov qword ptr [rsp+72], rbp +mov qword ptr [rsp+80], rcx +mov qword ptr [rsp+88], rsi + +;# T4 ← ((A + 2)/4) · T3 +mov rax, 121666 +mov rdi, rax +mul rbx +mov rbx, rax +mov r8, rdx +mov rax, rdi +mul rbp +mov rbp, rax +mov r9, rdx +mov rax, rdi +mul rcx +mov rcx, rax +mov r10, rdx +xor r11d, r11d +mov rax, rdi +mul rsi +mov rsi, rax +add rbp, r8 +adc rcx, r9 +adc rsi, r10 +adc r11, rdx +shld r11, rsi, 1 +btr rsi, 63 +imul r11, r11, 19 +add rbx, r11 +adc rbp, 0 +adc rcx, 0 +adc rsi, 0 + +;# T4 ← T4 + T2 +add rbx, qword ptr [rsp+32] +adc rbp, qword ptr [rsp+40] +adc rcx, qword ptr [rsp+48] +adc rsi, qword ptr [rsp+56] +mov eax, 0 +mov rdx, 38 +cmovae rdx, rax +add rbx, rdx +adc rbp, rax +adc rcx, rax +adc rsi, rax +cmovb rax, rdx +add rbx, rax +mov qword ptr [rsp+96], rbx +mov qword ptr [rsp+104], rbp +mov qword ptr [rsp+112], rcx +mov qword ptr [rsp+120], rsi + +;# X2 ← T1 · T2 +mov rax, qword ptr [rsp+8] +mul qword ptr [rsp+56] +mov r8, rax +xor r9d, r9d +mov r10, rdx +xor r11d, r11d +mov rax, qword ptr [rsp+16] +mul qword ptr [rsp+48] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+24] +mul qword ptr [rsp+40] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+16] +mul qword ptr [rsp+56] +add r10, rax +adc r11, 0 +mov r12, rdx +xor r13d, r13d +mov rax, qword ptr [rsp+24] +mul qword ptr [rsp+48] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, 38 +mul r10 +imul r11, r11, 38 +mov r10, rax +add r11, rdx +mov rax, qword ptr [rsp+24] +mul qword ptr [rsp+56] +add r12, rax +adc r13, 0 +mov rax, 38 +mul rdx +mov r14, rax +mov r15, rdx +mov rax, 38 +mul r12 +imul r13, r13, 38 +mov r12, rax +add r13, rdx +mov rax, qword ptr [rsp+0] +mul qword ptr [rsp+56] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+8] +mul qword ptr [rsp+48] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+16] +mul qword ptr [rsp+40] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+24] +mul qword ptr [rsp+32] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, 38 +mul r8 +imul r9, r9, 38 +mov r8, rax +add r9, rdx +mov rax, qword ptr [rsp+0] +mul qword ptr [rsp+32] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+0] +mul qword ptr [rsp+40] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, qword ptr [rsp+8] +mul qword ptr [rsp+32] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, qword ptr [rsp+0] +mul qword ptr [rsp+48] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, qword ptr [rsp+8] +mul qword ptr [rsp+40] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, qword ptr [rsp+16] +mul qword ptr [rsp+32] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +add r10, r9 +adc r11, 0 +add r12, r11 +adc r13, 0 +add r14, r13 +adc r15, 0 +shld r15, r14, 1 +btr r14, 63 +imul r15, r15, 19 +add r8, r15 +adc r10, 0 +adc r12, 0 +adc r14, 0 + +;# update X2 +mov qword ptr [rsp+160], r8 +mov qword ptr [rsp+168], r10 +mov qword ptr [rsp+176], r12 +mov qword ptr [rsp+184], r14 + +;# Z2 ← T3 · T4 +mov rax, qword ptr [rsp+72] +mul qword ptr [rsp+120] +mov r8, rax +xor r9d, r9d +mov r10, rdx +xor r11d, r11d +mov rax, qword ptr [rsp+80] +mul qword ptr [rsp+112] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+88] +mul qword ptr [rsp+104] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+80] +mul qword ptr [rsp+120] +add r10, rax +adc r11, 0 +mov r12, rdx +xor r13d, r13d +mov rax, qword ptr [rsp+88] +mul qword ptr [rsp+112] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, 38 +mul r10 +imul r11, r11, 38 +mov r10, rax +add r11, rdx +mov rax, qword ptr [rsp+88] +mul qword ptr [rsp+120] +add r12, rax +adc r13, 0 +mov rax, 38 +mul rdx +mov r14, rax +mov r15, rdx +mov rax, 38 +mul r12 +imul r13, r13, 38 +mov r12, rax +add r13, rdx +mov rax, qword ptr [rsp+64] +mul qword ptr [rsp+120] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+72] +mul qword ptr [rsp+112] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+80] +mul qword ptr [rsp+104] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+88] +mul qword ptr [rsp+96] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, 38 +mul r8 +imul r9, r9, 38 +mov r8, rax +add r9, rdx +mov rax, qword ptr [rsp+64] +mul qword ptr [rsp+96] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+64] +mul qword ptr [rsp+104] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, qword ptr [rsp+72] +mul qword ptr [rsp+96] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, qword ptr [rsp+64] +mul qword ptr [rsp+112] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, qword ptr [rsp+72] +mul qword ptr [rsp+104] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, qword ptr [rsp+80] +mul qword ptr [rsp+96] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +add r10, r9 +adc r11, 0 +add r12, r11 +adc r13, 0 +add r14, r13 +adc r15, 0 +shld r15, r14, 1 +btr r14, 63 +imul r15, r15, 19 +add r8, r15 +adc r10, 0 +adc r12, 0 +adc r14, 0 + +;# update Z2 +mov qword ptr [rsp+224], r8 +mov qword ptr [rsp+232], r10 +mov qword ptr [rsp+240], r12 +mov qword ptr [rsp+248], r14 + +;# Z3 ← Z3 · X1 +mov rax, qword ptr [rsp+136] +mul qword ptr [rsp+280] +mov r8, rax +xor r9d, r9d +mov r10, rdx +xor r11d, r11d +mov rax, qword ptr [rsp+144] +mul qword ptr [rsp+272] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+152] +mul qword ptr [rsp+264] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+144] +mul qword ptr [rsp+280] +add r10, rax +adc r11, 0 +mov r12, rdx +xor r13d, r13d +mov rax, qword ptr [rsp+152] +mul qword ptr [rsp+272] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, 38 +mul r10 +imul r11, r11, 38 +mov r10, rax +add r11, rdx +mov rax, qword ptr [rsp+152] +mul qword ptr [rsp+280] +add r12, rax +adc r13, 0 +mov rax, 38 +mul rdx +mov r14, rax +mov r15, rdx +mov rax, 38 +mul r12 +imul r13, r13, 38 +mov r12, rax +add r13, rdx +mov rax, qword ptr [rsp+128] +mul qword ptr [rsp+280] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+136] +mul qword ptr [rsp+272] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+144] +mul qword ptr [rsp+264] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, qword ptr [rsp+152] +mul qword ptr [rsp+256] +add r14, rax +adc r15, 0 +add r8, rdx +adc r9, 0 +mov rax, 38 +mul r8 +imul r9, r9, 38 +mov r8, rax +add r9, rdx +mov rax, qword ptr [rsp+128] +mul qword ptr [rsp+256] +add r8, rax +adc r9, 0 +add r10, rdx +adc r11, 0 +mov rax, qword ptr [rsp+128] +mul qword ptr [rsp+264] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, qword ptr [rsp+136] +mul qword ptr [rsp+256] +add r10, rax +adc r11, 0 +add r12, rdx +adc r13, 0 +mov rax, qword ptr [rsp+128] +mul qword ptr [rsp+272] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, qword ptr [rsp+136] +mul qword ptr [rsp+264] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, qword ptr [rsp+144] +mul qword ptr [rsp+256] +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +add r10, r9 +adc r11, 0 +add r12, r11 +adc r13, 0 +add r14, r13 +adc r15, 0 +shld r15, r14, 1 +btr r14, 63 +imul r15, r15, 19 +add r8, r15 +adc r10, 0 +adc r12, 0 +adc r14, 0 + +;# update Z3 +mov qword ptr [rsp+256], r8 +mov qword ptr [rsp+264], r10 +mov qword ptr [rsp+272], r12 +mov qword ptr [rsp+280], r14 + +movzx ecx, byte ptr [rsp+352] +sub ecx, 1 +mov byte ptr [rsp+352], cl +;cmp cl, 0 +jge loop_bit + +mov byte ptr [rsp+352], 7 +mov rsi, [rsp+312] +mov eax, dword ptr [rsp+356] +sub eax, 1 +mov dword ptr [rsp+356], eax +;cmp eax, 0 +jge loop_byte + +;# load select, AKA the LSB +mov bl, byte ptr [rsp+353] +cmp bl, 1 + +;# CSelect(X2,X3,select) +mov rsi, qword ptr [rsp+160] +mov rbp, qword ptr [rsp+168] +mov rcx, qword ptr [rsp+176] +mov rax, qword ptr [rsp+184] +mov r12, qword ptr [rsp+192] +mov r13, qword ptr [rsp+200] +mov r14, qword ptr [rsp+208] +mov r15, qword ptr [rsp+216] +cmove rsi, r12 +cmove rbp, r13 +cmove rcx, r14 +cmove rax, r15 + +;# CSelect(Z2,Z3,select) +mov r8, qword ptr [rsp+224] +mov r9, qword ptr [rsp+232] +mov r10, qword ptr [rsp+240] +mov r11, qword ptr [rsp+248] +mov r12, qword ptr [rsp+256] +mov r13, qword ptr [rsp+264] +mov r14, qword ptr [rsp+272] +mov r15, qword ptr [rsp+280] +cmove r8, r12 +cmove r9, r13 +cmove r10, r14 +cmove r11, r15 + +;# result ← X2 · Z2^(-1) + +;# result in: r8, r9, r10, r11 +;# Stack variables: +;# 0 T0 +;# 32 T1 +;# 64 T2 +;# 96 T3 +;# 128 T4 +;# 352 invtable position (4 bytes) + +;# T0 ← X2 +mov qword ptr [rsp+0], rsi +mov qword ptr [rsp+8], rbp +mov qword ptr [rsp+16], rcx +mov qword ptr [rsp+24], rax + +;# T2 ← Z2 +mov qword ptr [rsp+64], r8 +mov qword ptr [rsp+72], r9 +mov qword ptr [rsp+80], r10 +mov qword ptr [rsp+88], r11 + +mov dword ptr [rsp+352], 0 +lea r12, [invtable+REG_REL] + +inv_loop: +;# ecx = current instruction +mov ecx, dword ptr [r12] +test cl, cl +jz skip_square + +square_loop: +mov rax, r11 +mul r11 +mov r12, rax +xor r13d, r13d +mov rax, 38 +mul rdx +mov r14, rax +mov r15, rdx +shl r11, 1 +mov rax, r9 +mul r11 +mov rbx, rax +xor ebp, ebp +mov rdi, rdx +xor esi, esi +mov rax, r10 +mul r10 +add rbx, rax +adc rbp, 0 +add rdi, rdx +adc rsi, 0 +mov rax, r10 +mul r11 +add rdi, rax +adc rsi, 0 +add r12, rdx +adc r13, 0 +mov rax, 38 +mul rdi +imul rsi, rsi, 38 +mov rdi, rax +add rsi, rdx +mov rax, 38 +mul r12 +imul r13, r13, 38 +mov r12, rax +add r13, rdx +mov rax, r8 +mul r11 +add r14, rax +adc r15, 0 +add rbx, rdx +adc rbp, 0 +mov rax, r9 +mul r10 +add r14, rax +adc r15, 0 +add rbx, rdx +adc rbp, 0 +add r14, rax +adc r15, 0 +add rbx, rdx +adc rbp, 0 +mov rax, 38 +mul rbx +imul rbp, rbp, 38 +mov rbx, rax +add rbp, rdx +mov rax, r8 +mul r8 +add rbx, rax +adc rbp, 0 +add rdi, rdx +adc rsi, 0 +mov rax, r8 +mul r9 +add rdi, rax +adc rsi, 0 +add r12, rdx +adc r13, 0 +add rdi, rax +adc rsi, 0 +add r12, rdx +adc r13, 0 +mov rax, r8 +mul r10 +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov rax, r9 +mul r9 +add r12, rax +adc r13, 0 +add r14, rdx +adc r15, 0 +mov r9, rdi +add r9, rbp +adc rsi, 0 +mov r10, r12 +add r10, rsi +adc r13, 0 +mov r11, r14 +add r11, r13 +adc r15, 0 +shld r15, r11, 1 +imul r15, r15, 19 +btr r11, 63 +add rbx, r15 +adc r9, 0 +adc r10, 0 +adc r11, 0 +mov r8, 0 +adc r8, 0 +shld r8, r11, 1 +btr r11, 63 +imul r8, r8, 19 +add r8, rbx +sub cl, 1 +jnz square_loop + +skip_square: +shr ecx, 8 +test cl, cl +jnz skip_mul + +;# T4 ← result +lea rbx, [rsp+128] +mov qword ptr [rbx], r8 +mov qword ptr [rbx+8], r9 +mov qword ptr [rbx+16], r10 +mov qword ptr [rbx+24], r11 + +movzx edi, ch ;# mul_source +shl edi, 5 +add rdi, rsp + +;# result ← T4 · mul_source +mov rax, qword ptr [rbx+8] +mul qword ptr [rdi+24] +mov r8, rax +xor esi, esi +mov r9, rdx +xor ebp, ebp +mov rax, qword ptr [rbx+16] +mul qword ptr [rdi+16] +add r8, rax +adc rsi, 0 +add r9, rdx +adc rbp, 0 +mov rax, qword ptr [rbx+24] +mul qword ptr [rdi+8] +add r8, rax +adc rsi, 0 +add r9, rdx +adc rbp, 0 +mov rax, qword ptr [rbx+16] +mul qword ptr [rdi+24] +add r9, rax +adc rbp, 0 +mov r10, rdx +xor r13d, r13d +mov rax, qword ptr [rbx+24] +mul qword ptr [rdi+16] +add r9, rax +adc rbp, 0 +add r10, rdx +adc r13, 0 +mov rax, 38 +mul r9 +imul rbp, rbp, 38 +mov r9, rax +add rbp, rdx +mov rax, qword ptr [rbx+24] +mul qword ptr [rdi+24] +add r10, rax +adc r13, 0 +mov rax, 38 +mul rdx +mov r11, rax +mov r15, rdx +mov rax, 38 +mul r10 +imul r13, r13, 38 +mov r10, rax +add r13, rdx +mov rax, qword ptr [rbx] +mul qword ptr [rdi+24] +add r11, rax +adc r15, 0 +add r8, rdx +adc rsi, 0 +mov rax, qword ptr [rbx+8] +mul qword ptr [rdi+16] +add r11, rax +adc r15, 0 +add r8, rdx +adc rsi, 0 +mov rax, qword ptr [rbx+16] +mul qword ptr [rdi+8] +add r11, rax +adc r15, 0 +add r8, rdx +adc rsi, 0 +mov rax, qword ptr [rbx+24] +mul qword ptr [rdi] +add r11, rax +adc r15, 0 +add r8, rdx +adc rsi, 0 +mov rax, 38 +mul r8 +imul rsi, rsi, 38 +mov r8, rax +add rsi, rdx +mov rax, qword ptr [rbx] +mul qword ptr [rdi] +add r8, rax +adc rsi, 0 +add r9, rdx +adc rbp, 0 +mov rax, qword ptr [rbx] +mul qword ptr [rdi+8] +add r9, rax +adc rbp, 0 +add r10, rdx +adc r13, 0 +mov rax, qword ptr [rbx+8] +mul qword ptr [rdi] +add r9, rax +adc rbp, 0 +add r10, rdx +adc r13, 0 +mov rax, qword ptr [rbx] +mul qword ptr [rdi+16] +add r10, rax +adc r13, 0 +add r11, rdx +adc r15, 0 +mov rax, qword ptr [rbx+8] +mul qword ptr [rdi+8] +add r10, rax +adc r13, 0 +add r11, rdx +adc r15, 0 +mov rax, qword ptr [rbx+16] +mul qword ptr [rdi] +add r10, rax +adc r13, 0 +add r11, rdx +adc r15, 0 +add r9, rsi +adc rbp, 0 +add r10, rbp +adc r13, 0 +add r11, r13 +adc r15, 0 +shld r15, r11, 1 +btr r11, 63 +imul r15, r15, 19 +add r8, r15 +adc r9, 0 +adc r10, 0 +adc r11, 0 + +skip_mul: +shr ecx, 11 +jz skip_store + +add rcx, rsp + +;# store temp T[1-3] +mov qword ptr [rcx], r8 +mov qword ptr [rcx+8], r9 +mov qword ptr [rcx+16], r10 +mov qword ptr [rcx+24], r11 + +skip_store: +mov eax, dword ptr [rsp+352] +add eax, 4 +lea r12, [invtable+REG_REL] +add r12, rax +mov dword ptr [rsp+352], eax + +cmp eax, 52 +jb inv_loop + +;# final reduction + +xor r12d, r12d +btr r11, 63 +adc r12, 0 +imul r12, r12, 19 +add r8, r12 +adc r9, 0 + +mov rax, r8 +mov rcx, r9 +mov rdx, r10 +mov rsi, r11 +sub r8, qword ptr [p0+REG_REL] +sbb r9, qword ptr [p12+REG_REL] +sbb r10, qword ptr [p12+REG_REL] +sbb r11, qword ptr [p3+REG_REL] +bt r11, 63 +cmovb r8, rax +cmovb r9, rcx +cmovb r10, rdx +cmovb r11, rsi + +mov rdi, qword ptr [rsp+304] + +;# output the result +mov qword ptr [rdi], r8 +mov qword ptr [rdi+8], r9 +mov qword ptr [rdi+16], r10 +mov qword ptr [rdi+24], r11 + +;# restore registers +mov rbx, qword ptr [rsp+288] +mov rbp, qword ptr [rsp+296] +;# mov rdi, qword ptr [rsp+304] +;# mov rsi, qword ptr [rsp+312] +mov r12, qword ptr [rsp+320] +mov r13, qword ptr [rsp+328] +mov r14, qword ptr [rsp+336] +mov r15, qword ptr [rsp+344] + +add rsp, 392 diff --git a/external/src/mx25519/amd64/scalarmult_gnu.S b/external/src/mx25519/amd64/scalarmult_gnu.S new file mode 100644 index 0000000..0f2b57b --- /dev/null +++ b/external/src/mx25519/amd64/scalarmult_gnu.S @@ -0,0 +1,64 @@ +# Copyright (c) 2022 tevador +# +# This file is part of mx25519, which is released under LGPLv3. +# See LICENSE for full license details. + +.intel_syntax noprefix +#if defined(__APPLE__) +.text +#define DECL(x) _##x +#else +.section .text +#define DECL(x) x +#endif + +#if defined(__WIN32__) || defined(__CYGWIN__) +#define WINABI +#endif + +#define REG_REL rip +#define db .byte +#define ALIGN .balign + +.global DECL(mx25519_scalarmult_amd64x) +.global DECL(mx25519_scalarmult_amd64) + +#include "constants.inc" + +ALIGN 32 +DECL(mx25519_scalarmult_amd64x): +#ifdef WINABI + mov qword ptr [rsp+8], rdi + mov qword ptr [rsp+16], rsi + mov rdi, rcx + mov rsi, rdx + mov rdx, r8 +#endif + +#include "scalarmult_mulx_adx.inc" + +#ifdef WINABI + mov rdi, qword ptr [rsp+8] + mov rsi, qword ptr [rsp+16] +#endif + + ret + +ALIGN 32 +DECL(mx25519_scalarmult_amd64): +#ifdef WINABI + mov qword ptr [rsp+8], rdi + mov qword ptr [rsp+16], rsi + mov rdi, rcx + mov rsi, rdx + mov rdx, r8 +#endif + +#include "scalarmult_compat.inc" + +#ifdef WINABI + mov rdi, qword ptr [rsp+8] + mov rsi, qword ptr [rsp+16] +#endif + + ret diff --git a/external/src/mx25519/amd64/scalarmult_masm.asm b/external/src/mx25519/amd64/scalarmult_masm.asm new file mode 100644 index 0000000..c5bf759 --- /dev/null +++ b/external/src/mx25519/amd64/scalarmult_masm.asm @@ -0,0 +1,49 @@ +; Copyright (c) 2022 tevador +; +; This file is part of mx25519, which is released under LGPLv3. +; See LICENSE for full license details. + +IFDEF RAX + +REG_REL EQU 0 + +MX25519_SCALARMULT SEGMENT PAGE READ EXECUTE + +PUBLIC mx25519_scalarmult_amd64x +PUBLIC mx25519_scalarmult_amd64 + +include constants.inc + +mx25519_scalarmult_amd64x PROC + mov qword ptr [rsp+8], rdi + mov qword ptr [rsp+16], rsi + mov rdi, rcx + mov rsi, rdx + mov rdx, r8 + +include scalarmult_mulx_adx.inc + + mov rdi, qword ptr [rsp+8] + mov rsi, qword ptr [rsp+16] + ret +mx25519_scalarmult_amd64x ENDP + +mx25519_scalarmult_amd64 PROC + mov qword ptr [rsp+8], rdi + mov qword ptr [rsp+16], rsi + mov rdi, rcx + mov rsi, rdx + mov rdx, r8 + +include scalarmult_compat.inc + + mov rdi, qword ptr [rsp+8] + mov rsi, qword ptr [rsp+16] + ret +mx25519_scalarmult_amd64 ENDP + +MX25519_SCALARMULT ENDS + +ENDIF + +END diff --git a/external/src/mx25519/amd64/scalarmult_mulx_adx.inc b/external/src/mx25519/amd64/scalarmult_mulx_adx.inc new file mode 100644 index 0000000..a7d7841 --- /dev/null +++ b/external/src/mx25519/amd64/scalarmult_mulx_adx.inc @@ -0,0 +1,1449 @@ +;# Copyright (c) 2022 tevador +;# +;# This file is part of mx25519, which is released under LGPLv3. +;# See LICENSE for full license details. +;# +;# +;# Parts of this file are derived from a work with the following license: +;# +;# Copyright (c) 2020, Kaushik Nath and Palash Sarkar. +;# +;# Permission to use this code is granted. +;# +;# Redistribution and use in source and binary forms, with or without +;# modification, are permitted provided that the following conditions are +;# met: +;# +;# * Redistributions of source code must retain the above copyright notice, +;# this list of conditions and the following disclaimer. +;# +;# * Redistributions in binary form must reproduce the above copyright +;# notice, this list of conditions and the following disclaimer in the +;# documentation and/or other materials provided with the distribution. +;# +;# * The names of the contributors may not be used to endorse or promote +;# products derived from this software without specific prior written +;# permission. +;# +;# THIS SOFTWARE IS PROVIDED BY THE AUTHORS ""AS IS"" AND ANY EXPRESS OR +;# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +;# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +;# IN NO EVENT SHALL THE CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +;# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +;# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +;# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +;# THEORY LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +;# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +;# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +;# arguments: +;# rdi -> public key address (out) +;# rsi -> private key address (in) +;# rdx -> base point address (in) + +sub rsp, 392 + +;# stack layout (offsets from rsp): +;# 0 T1 +;# 32 T2 +;# 64 T3 +;# 96 T4 +;# 128 X1 +;# 160 X2 +;# 192 X3 +;# 224 Z2 +;# 256 Z3 +;# 288 saved registers: rbx, rbp, rdi, rsi, r12, r13, r14, r15 +;# 352 key bit index (1 byte) +;# 353 swap flag (1 byte) +;# 354 current key byte value (1 byte) +;# 356 key byte index (4 bytes) + +mov qword ptr [rsp+288], rbx +mov qword ptr [rsp+296], rbp +mov qword ptr [rsp+304], rdi +mov qword ptr [rsp+312], rsi +mov qword ptr [rsp+320], r12 +mov qword ptr [rsp+328], r13 +mov qword ptr [rsp+336], r14 +mov qword ptr [rsp+344], r15 + +;# starting from bit 6 of the 31st byte +mov eax, 31 +mov byte ptr [rsp+352], 6 +mov byte ptr [rsp+353], 0 +mov dword ptr [rsp+356], eax + +;# load XP +mov r8, qword ptr [rdx] +mov r9, qword ptr [rdx+8] +mov r10, qword ptr [rdx+16] +mov r11, qword ptr [rdx+24] + +;# reduce XP mod 2^255-19 +btr r11, 63 +mov rbx, r8 +mov rcx, r9 +mov rdx, r10 +mov rdi, r11 +sub r8, qword ptr [p0+REG_REL] +sbb r9, qword ptr [p12+REG_REL] +sbb r10, qword ptr [p12+REG_REL] +sbb r11, qword ptr [p3+REG_REL] +bt r11, 63 +cmovb r8, rbx +cmovb r9, rcx +cmovb r10, rdx +cmovb r11, rdi + +;# X1 ← XP, X3 ← XP +mov qword ptr [rsp+128], r8 +mov qword ptr [rsp+192], r8 +mov qword ptr [rsp+136], r9 +mov qword ptr [rsp+200], r9 +mov qword ptr [rsp+144], r10 +mov qword ptr [rsp+208], r10 +mov qword ptr [rsp+152], r11 +mov qword ptr [rsp+216], r11 + +;# X2 ← 1 +mov qword ptr [rsp+160], 1 +mov qword ptr [rsp+168], 0 +mov qword ptr [rsp+176], 0 +mov qword ptr [rsp+184], 0 + +;# Z2 ← 0 +mov qword ptr [rsp+224], 0 +mov qword ptr [rsp+232], 0 +mov qword ptr [rsp+240], 0 +mov qword ptr [rsp+248], 0 + +;# Z3 ← 1 +mov qword ptr [rsp+256], 1 +mov qword ptr [rsp+264], 0 +mov qword ptr [rsp+272], 0 +mov qword ptr [rsp+280], 0 + +;# Montgomery ladder loop + +ALIGN 16 +loop_byte_x: + +;# rsi = private key address +;# eax = key byte index +movzx ecx, byte ptr [rsi+rax] +mov byte ptr [rsp+354], cl + +;# +;# Montgomery ladder step +;# +;# Reduction ideas for addition and subtraction are taken from the 64-bit implementation +;# "amd64-64" of the work "https://link.springer.com/article/10.1007/s13389-012-0027-1" +;# +;# T1 ← X2 + Z2 +;# T2 ← X2 - Z2 +;# T3 ← X3 + Z3 +;# T4 ← X3 - Z3 +;# Z3 ← T2 · T3 +;# X3 ← T1 · T4 +;# +;# bit ← n[i] +;# select ← bit ⊕ prevbit +;# prevbit ← bit +;# CSelect(T1,T3,select): if (select == 1) {T1 = T3} +;# CSelect(T2,T4,select): if (select == 1) {T2 = T4} +;# +;# T2 ← T2^2 +;# T1 ← T1^2 +;# T3 ← X3 + Z3 +;# Z3 ← X3 - Z3 +;# Z3 ← Z3^2 +;# X3 ← T3^2 +;# T3 ← T1 - T2 +;# T4 ← ((A + 2)/4) · T3 +;# T4 ← T4 + T2 +;# X2 ← T1 · T2 +;# Z2 ← T3 · T4 +;# Z3 ← Z3 · X1 +;# + +ALIGN 16 +loop_bit_x: + +;# X2 +mov r8, qword ptr [rsp+160] +mov r9, qword ptr [rsp+168] +mov r10, qword ptr [rsp+176] +mov r11, qword ptr [rsp+184] + +;# copy X2 +mov rax, r8 +mov rbx, r9 +mov rbp, r10 +mov rsi, r11 + +;# T1 ← X2 + Z2 +add r8, qword ptr [rsp+224] +adc r9, qword ptr [rsp+232] +adc r10, qword ptr [rsp+240] +adc r11, qword ptr [rsp+248] + +mov rdi, 0 +mov rcx, 38 +cmovae rcx, rdi + +add r8, rcx +adc r9, rdi +adc r10, rdi +adc r11, rdi + +cmovb rdi, rcx +add r8, rdi + +mov qword ptr [rsp+0], r8 +mov qword ptr [rsp+8], r9 +mov qword ptr [rsp+16], r10 +mov qword ptr [rsp+24], r11 + +;# T2 ← X2 - Z2 +sub rax, qword ptr [rsp+224] +sbb rbx, qword ptr [rsp+232] +sbb rbp, qword ptr [rsp+240] +sbb rsi, qword ptr [rsp+248] +mov edi, 0 +mov rcx, 38 +cmovae rcx, rdi +sub rax, rcx +sbb rbx, rdi +sbb rbp, rdi +sbb rsi, rdi +cmovb rdi, rcx +sub rax, rdi +mov qword ptr [rsp+32], rax +mov qword ptr [rsp+40], rbx +mov qword ptr [rsp+48], rbp +mov qword ptr [rsp+56], rsi + +;# X3 +mov r8, qword ptr [rsp+192] +mov r9, qword ptr [rsp+200] +mov r10, qword ptr [rsp+208] +mov r11, qword ptr [rsp+216] + +;# copy X3 +mov rax, r8 +mov rbx, r9 +mov rbp, r10 +mov rsi, r11 + +;# T3 ← X3 + Z3 +add r8, qword ptr [rsp+256] +adc r9, qword ptr [rsp+264] +adc r10, qword ptr [rsp+272] +adc r11, qword ptr [rsp+280] +mov edi, 0 +mov rcx, 38 +cmovae rcx, rdi +add r8, rcx +adc r9, rdi +adc r10, rdi +adc r11, rdi +cmovb rdi, rcx +add r8, rdi +mov qword ptr [rsp+64], r8 +mov qword ptr [rsp+72], r9 +mov qword ptr [rsp+80], r10 +mov qword ptr [rsp+88], r11 + +;# T4 ← X3 - Z3 +sub rax, qword ptr [rsp+256] +sbb rbx, qword ptr [rsp+264] +sbb rbp, qword ptr [rsp+272] +sbb rsi, qword ptr [rsp+280] +mov edi, 0 +mov rcx, 38 +cmovae rcx, rdi +sub rax, rcx +sbb rbx, rdi +sbb rbp, rdi +sbb rsi, rdi +cmovb rdi, rcx +sub rax, rdi +mov qword ptr [rsp+96], rax +mov qword ptr [rsp+104], rbx +mov qword ptr [rsp+112], rbp +mov qword ptr [rsp+120], rsi + +;# Z3 ← T2 · T3 +xor r13d, r13d +mov rdx, qword ptr [rsp+32] +mulx r9, r8, qword ptr [rsp+64] +mulx r10, rcx, qword ptr [rsp+72] +adcx r9, rcx +mulx r11, rcx, qword ptr [rsp+80] +adcx r10, rcx +mulx r12, rcx, qword ptr [rsp+88] +adcx r11, rcx +adcx r12, r13 +xor r14d, r14d +mov rdx, qword ptr [rsp+40] +mulx rbp, rcx, qword ptr [rsp+64] +adcx r9, rcx +adox r10, rbp +mulx rbp, rcx, qword ptr [rsp+72] +adcx r10, rcx +adox r11, rbp +mulx rbp, rcx, qword ptr [rsp+80] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+88] +adcx r12, rcx +adox r13, rbp +adcx r13, r14 +xor r15d, r15d +mov rdx, qword ptr [rsp+48] +mulx rbp, rcx, qword ptr [rsp+64] +adcx r10, rcx +adox r11, rbp +mulx rbp, rcx, qword ptr [rsp+72] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+80] +adcx r12, rcx +adox r13, rbp +mulx rbp, rcx, qword ptr [rsp+88] +adcx r13, rcx +adox r14, rbp +adcx r14, r15 +xor eax, eax +mov rdx, qword ptr [rsp+56] +mulx rbp, rcx, qword ptr [rsp+64] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+72] +adcx r12, rcx +adox r13, rbp +mulx rbp, rcx, qword ptr [rsp+80] +adcx r13, rcx +adox r14, rbp +mulx rbp, rcx, qword ptr [rsp+88] +adcx r14, rcx +adox r15, rbp +adcx r15, rax +xor ebp, ebp +mov rdx, 38 +mulx r12, rax, r12 +adcx r8, rax +adox r9, r12 +mulx r13, rcx, r13 +adcx r9, rcx +adox r10, r13 +mulx r14, rcx, r14 +adcx r10, rcx +adox r11, r14 +mulx r15, rcx, r15 +adcx r11, rcx +adox r15, rbp ;# zero +adcx r15, rbp ;# zero +shld r15, r11, 1 +btr r11, 63 +imul r15, r15, 19 +add r8, r15 +adc r9, 0 +adc r10, 0 +adc r11, 0 +mov qword ptr [rsp+256], r8 +mov qword ptr [rsp+264], r9 +mov qword ptr [rsp+272], r10 +mov qword ptr [rsp+280], r11 + +;# X3 ← T1 · T4 +xor r13d, r13d +mov rdx, qword ptr [rsp+0] +mulx r9, r8, qword ptr [rsp+96] +mulx r10, rcx, qword ptr [rsp+104] +adcx r9, rcx +mulx r11, rcx, qword ptr [rsp+112] +adcx r10, rcx +mulx r12, rcx, qword ptr [rsp+120] +adcx r11, rcx +adcx r12, r13 +xor r14d, r14d +mov rdx, qword ptr [rsp+8] +mulx rbp, rcx, qword ptr [rsp+96] +adcx r9, rcx +adox r10, rbp +mulx rbp, rcx, qword ptr [rsp+104] +adcx r10, rcx +adox r11, rbp +mulx rbp, rcx, qword ptr [rsp+112] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+120] +adcx r12, rcx +adox r13, rbp +adcx r13, r14 +xor r15d, r15d +mov rdx, qword ptr [rsp+16] +mulx rbp, rcx, qword ptr [rsp+96] +adcx r10, rcx +adox r11, rbp +mulx rbp, rcx, qword ptr [rsp+104] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+112] +adcx r12, rcx +adox r13, rbp +mulx rbp, rcx, qword ptr [rsp+120] +adcx r13, rcx +adox r14, rbp +adcx r14, r15 +xor eax, eax +mov rdx, qword ptr [rsp+24] +mulx rbp, rcx, qword ptr [rsp+96] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+104] +adcx r12, rcx +adox r13, rbp +mulx rbp, rcx, qword ptr [rsp+112] +adcx r13, rcx +adox r14, rbp +mulx rbp, rcx, qword ptr [rsp+120] +adcx r14, rcx +adox r15, rbp +adcx r15, rax +xor ebp, ebp +mov rdx, 38 +mulx r12, rax, r12 +adcx r8, rax +adox r9, r12 +mulx r13, rcx, r13 +adcx r9, rcx +adox r10, r13 +mulx r14, rcx, r14 +adcx r10, rcx +adox r11, r14 +mulx r15, rcx, r15 +adcx r11, rcx +adox r15, rbp ;# zero +adcx r15, rbp ;# zero +shld r15, r11, 1 +btr r11, 63 +imul r15, r15, 19 +add r8, r15 +adc r9, 0 +adc r10, 0 +adc r11, 0 +mov qword ptr [rsp+192], r8 +mov qword ptr [rsp+200], r9 +mov qword ptr [rsp+208], r10 +mov qword ptr [rsp+216], r11 + +movzx ecx, byte ptr [rsp+352] +movzx ebx, byte ptr [rsp+354] +shr bl, cl +and bl, 1 +mov cl, bl +xor bl, byte ptr [rsp+353] +mov byte ptr [rsp+353], cl +cmp bl, 1 + +;# CSelect(T1,T3,select) +mov r8, qword ptr [rsp+0] +mov r9, qword ptr [rsp+8] +mov r10, qword ptr [rsp+16] +mov r11, qword ptr [rsp+24] +mov r12, qword ptr [rsp+64] +mov r13, qword ptr [rsp+72] +mov r14, qword ptr [rsp+80] +mov r15, qword ptr [rsp+88] +cmove r8, r12 +cmove r9, r13 +cmove r10, r14 +cmove r11, r15 +mov qword ptr [rsp+0], r8 +mov qword ptr [rsp+8], r9 +mov qword ptr [rsp+16], r10 +mov qword ptr [rsp+24], r11 + +;# CSelect(T2,T4,select) +mov rax, qword ptr [rsp+32] +mov rbx, qword ptr [rsp+40] +mov rbp, qword ptr [rsp+48] +mov rsi, qword ptr [rsp+56] +mov r12, qword ptr [rsp+96] +mov r13, qword ptr [rsp+104] +mov r14, qword ptr [rsp+112] +mov r15, qword ptr [rsp+120] +cmove rax, r12 +cmove rbx, r13 +cmove rbp, r14 +cmove rsi, r15 + +;# T2 ← T2^2 +xor r13d, r13d +mov rdx, rax +mulx r10, r9, rbx +mulx r11, rcx, rbp +adcx r10, rcx +mulx r12, rcx, rsi +adcx r11, rcx +adcx r12, r13 +xor r14d, r14d +mov rdx, rbx +mulx rdi, rcx, rbp +adcx r11, rcx +adox r12, rdi +mulx rdi, rcx, rsi +adcx r12, rcx +adox r13, rdi +adcx r13, r14 +xor r15d, r15d +mov rdx, rbp +mulx r14, rcx, rsi +adcx r13, rcx +adcx r14, r15 +shld r15, r14, 1 +shld r14, r13, 1 +shld r13, r12, 1 +shld r12, r11, 1 +shld r11, r10, 1 +shld r10, r9, 1 +shl r9, 1 +xor edx, edx +mov rdx, rax +mulx rdx, r8, rdx +adcx r9, rdx +mov rdx, rbx +mulx rdx, rcx, rdx +adcx r10, rcx +adcx r11, rdx +mov rdx, rbp +mulx rdx, rcx, rdx +adcx r12, rcx +adcx r13, rdx +mov rdx, rsi +mulx rdx, rcx, rdx +adcx r14, rcx +adcx r15, rdx +xor edi, edi +mov rdx, 38 +mulx rbp, rbx, r12 +adcx rbx, r8 +adox rbp, r9 +mulx rax, rcx, r13 +adcx rbp, rcx +adox rax, r10 +mulx rsi, rcx, r14 +adcx rax, rcx +adox rsi, r11 +mulx r15, rcx, r15 +adcx rsi, rcx +adox r15, rdi ;# zero +adcx r15, rdi ;# zero +shld r15, rsi, 1 +btr rsi, 63 +imul r15, r15, 19 +add rbx, r15 +adc rbp, 0 +adc rax, 0 +adc rsi, 0 +mov qword ptr [rsp+32], rbx +mov qword ptr [rsp+40], rbp +mov qword ptr [rsp+48], rax +mov qword ptr [rsp+56], rsi + +;# T1 ← T1^2 +xor r13d, r13d +mov rdx, qword ptr [rsp+0] +mulx r10, r9, qword ptr [rsp+8] +mulx r11, rcx, qword ptr [rsp+16] +adcx r10, rcx +mulx r12, rcx, qword ptr [rsp+24] +adcx r11, rcx +adcx r12, r13 +xor r14d, r14d +mov rdx, qword ptr [rsp+8] +mulx rdi, rcx, qword ptr [rsp+16] +adcx r11, rcx +adox r12, rdi +mulx rdi, rcx, qword ptr [rsp+24] +adcx r12, rcx +adox r13, rdi +adcx r13, r14 +xor r15d, r15d +mov rdx, qword ptr [rsp+16] +mulx r14, rcx, qword ptr [rsp+24] +adcx r13, rcx +adcx r14, r15 +shld r15, r14, 1 +shld r14, r13, 1 +shld r13, r12, 1 +shld r12, r11, 1 +shld r11, r10, 1 +shld r10, r9, 1 +shl r9, 1 +xor edx, edx +mov rdx, qword ptr [rsp+0] +mulx rdx, r8, rdx +adcx r9, rdx +mov rdx, qword ptr [rsp+8] +mulx rdx, rcx, rdx +adcx r10, rcx +adcx r11, rdx +mov rdx, qword ptr [rsp+16] +mulx rdx, rcx, rdx +adcx r12, rcx +adcx r13, rdx +mov rdx, qword ptr [rsp+24] +mulx rdx, rcx, rdx +adcx r14, rcx +adcx r15, rdx +xor edi, edi +mov rdx, 38 +mulx rbp, rbx, r12 +adcx rbx, r8 +adox rbp, r9 +mulx rax, rcx, r13 +adcx rbp, rcx +adox rax, r10 +mulx rsi, rcx, r14 +adcx rax, rcx +adox rsi, r11 +mulx r15, rcx, r15 +adcx rsi, rcx +adox r15, rdi ;# zero +adcx r15, rdi ;# zero +shld r15, rsi, 1 +btr rsi, 63 +imul r15, r15, 19 +add rbx, r15 +adc rbp, 0 +adc rax, 0 +adc rsi, 0 +mov qword ptr [rsp+0], rbx +mov qword ptr [rsp+8], rbp +mov qword ptr [rsp+16], rax +mov qword ptr [rsp+24], rsi + +;# X3 +mov r8, qword ptr [rsp+192] +mov r9, qword ptr [rsp+200] +mov r10, qword ptr [rsp+208] +mov r11, qword ptr [rsp+216] + +;# copy X3 +mov rax, r8 +mov rbx, r9 +mov rbp, r10 +mov rsi, r11 + +;# T3 ← X3 + Z3 +add r8, qword ptr [rsp+256] +adc r9, qword ptr [rsp+264] +adc r10, qword ptr [rsp+272] +adc r11, qword ptr [rsp+280] +mov rdi, 0 +mov rcx, 38 +cmovae rcx, rdi +add r8, rcx +adc r9, rdi +adc r10, rdi +adc r11, rdi +cmovb rdi, rcx +add r8, rdi +mov qword ptr [rsp+64], r8 +mov qword ptr [rsp+72], r9 +mov qword ptr [rsp+80], r10 +mov qword ptr [rsp+88], r11 + +;# Z3 ← X3 - Z3 +sub rax, qword ptr [rsp+256] +sbb rbx, qword ptr [rsp+264] +sbb rbp, qword ptr [rsp+272] +sbb rsi, qword ptr [rsp+280] +mov edi, 0 +mov rcx, 38 +cmovae rcx, rdi +sub rax, rcx +sbb rbx, rdi +sbb rbp, rdi +sbb rsi, rdi +cmovb rdi, rcx +sub rax, rdi + +;# Z3 ← Z3^2 +xor r13d, r13d +mov rdx, rax +mulx r10, r9, rbx +mulx r11, rcx, rbp +adcx r10, rcx +mulx r12, rcx, rsi +adcx r11, rcx +adcx r12, r13 +xor r14d, r14d +mov rdx, rbx +mulx rdi, rcx, rbp +adcx r11, rcx +adox r12, rdi +mulx rdi, rcx, rsi +adcx r12, rcx +adox r13, rdi +adcx r13, r14 +xor r15d, r15d +mov rdx, rbp +mulx r14, rcx, rsi +adcx r13, rcx +adcx r14, r15 +shld r15, r14, 1 +shld r14, r13, 1 +shld r13, r12, 1 +shld r12, r11, 1 +shld r11, r10, 1 +shld r10, r9, 1 +shl r9, 1 +xor edx, edx +mov rdx, rax +mulx rdx, r8, rdx +adcx r9, rdx +mov rdx, rbx +mulx rdx, rcx, rdx +adcx r10, rcx +adcx r11, rdx +mov rdx, rbp +mulx rdx, rcx, rdx +adcx r12, rcx +adcx r13, rdx +mov rdx, rsi +mulx rdx, rcx, rdx +adcx r14, rcx +adcx r15, rdx +xor edi, edi +mov rdx, 38 +mulx rbp, rbx, r12 +adcx rbx, r8 +adox rbp, r9 +mulx rax, rcx, r13 +adcx rbp, rcx +adox rax, r10 +mulx rsi, rcx, r14 +adcx rax, rcx +adox rsi, r11 +mulx r15, rcx, r15 +adcx rsi, rcx +adox r15, rdi ;# zero +adcx r15, rdi ;# zero +shld r15, rsi, 1 +btr rsi, 63 +imul r15, r15, 19 +add rbx, r15 +adc rbp, 0 +adc rax, 0 +adc rsi, 0 +mov qword ptr [rsp+256], rbx +mov qword ptr [rsp+264], rbp +mov qword ptr [rsp+272], rax +mov qword ptr [rsp+280], rsi + +;# T3 ← X3^2 +xor r13d, r13d +mov rdx, qword ptr [rsp+64] +mulx r10, r9, qword ptr [rsp+72] +mulx r11, rcx, qword ptr [rsp+80] +adcx r10, rcx +mulx r12, rcx, qword ptr [rsp+88] +adcx r11, rcx +adcx r12, r13 +xor r14d, r14d +mov rdx, qword ptr [rsp+72] +mulx rdi, rcx, qword ptr [rsp+80] +adcx r11, rcx +adox r12, rdi +mulx rdi, rcx, qword ptr [rsp+88] +adcx r12, rcx +adox r13, rdi +adcx r13, r14 +xor r15d, r15d +mov rdx, qword ptr [rsp+80] +mulx r14, rcx, qword ptr [rsp+88] +adcx r13, rcx +adcx r14, r15 +shld r15, r14, 1 +shld r14, r13, 1 +shld r13, r12, 1 +shld r12, r11, 1 +shld r11, r10, 1 +shld r10, r9, 1 +shl r9, 1 +xor edx, edx +mov rdx, qword ptr [rsp+64] +mulx rdx, r8, rdx +adcx r9, rdx +mov rdx, qword ptr [rsp+72] +mulx rdx, rcx, rdx +adcx r10, rcx +adcx r11, rdx +mov rdx, qword ptr [rsp+80] +mulx rdx, rcx, rdx +adcx r12, rcx +adcx r13, rdx +mov rdx, qword ptr [rsp+88] +mulx rdx, rcx, rdx +adcx r14, rcx +adcx r15, rdx +xor edi, edi +mov rdx, 38 +mulx rbp, rbx, r12 +adcx rbx, r8 +adox rbp, r9 +mulx rax, rcx, r13 +adcx rbp, rcx +adox rax, r10 +mulx rsi, rcx, r14 +adcx rax, rcx +adox rsi, r11 +mulx r15, rcx, r15 +adcx rsi, rcx +adox r15, rdi ;# zero +adcx r15, rdi ;# zero +shld r15, rsi, 1 +btr rsi, 63 +imul r15, r15, 19 +add rbx, r15 +adc rbp, 0 +adc rax, 0 +adc rsi, 0 + +;# update X3 +mov qword ptr [rsp+192], rbx +mov qword ptr [rsp+200], rbp +mov qword ptr [rsp+208], rax +mov qword ptr [rsp+216], rsi + +;# T3 ← T1 - T2 +mov rax, qword ptr [rsp+0] +mov rbx, qword ptr [rsp+8] +mov rsi, qword ptr [rsp+16] +mov rdi, qword ptr [rsp+24] +sub rax, qword ptr [rsp+32] +sbb rbx, qword ptr [rsp+40] +sbb rsi, qword ptr [rsp+48] +sbb rdi, qword ptr [rsp+56] +mov ebp, 0 +mov rcx, 38 +cmovae rcx, rbp +sub rax, rcx +sbb rbx, rbp +sbb rsi, rbp +sbb rdi, rbp +cmovb rbp, rcx +sub rax, rbp +mov qword ptr [rsp+64], rax +mov qword ptr [rsp+72], rbx +mov qword ptr [rsp+80], rsi +mov qword ptr [rsp+88], rdi + +;# T4 ← ((A + 2)/4) · T3 +xor r12d, r12d +mov rdx, 121666 +mulx rbp, rax, rax +mulx rcx, rbx, rbx +adcx rbx, rbp +mulx rbp, rsi, rsi +adcx rsi, rcx +mulx rcx, rdi, rdi +adcx rdi, rbp +adcx rcx, r12 +shld rcx, rdi, 1 +btr rdi, 63 +imul rcx, rcx, 19 +add rax, rcx +adc rbx, 0 +adc rsi, 0 +adc rdi, 0 + +;# T4 ← T4 + T2 +add rax, qword ptr [rsp+32] +adc rbx, qword ptr [rsp+40] +adc rsi, qword ptr [rsp+48] +adc rdi, qword ptr [rsp+56] +mov ebp, 0 +mov rcx, 38 +cmovae rcx, rbp +add rax, rcx +adc rbx, rbp +adc rsi, rbp +adc rdi, rbp +cmovb rbp, rcx +add rax, rbp +mov qword ptr [rsp+96], rax +mov qword ptr [rsp+104], rbx +mov qword ptr [rsp+112], rsi +mov qword ptr [rsp+120], rdi + +;# X2 ← T1 · T2 +xor r13d, r13d +mov rdx, qword ptr [rsp+0] +mulx r9, r8, qword ptr [rsp+32] +mulx r10, rcx, qword ptr [rsp+40] +adcx r9, rcx +mulx r11, rcx, qword ptr [rsp+48] +adcx r10, rcx +mulx r12, rcx, qword ptr [rsp+56] +adcx r11, rcx +adcx r12, r13 +xor r14d, r14d +mov rdx, qword ptr [rsp+8] +mulx rbp, rcx, qword ptr [rsp+32] +adcx r9, rcx +adox r10, rbp +mulx rbp, rcx, qword ptr [rsp+40] +adcx r10, rcx +adox r11, rbp +mulx rbp, rcx, qword ptr [rsp+48] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+56] +adcx r12, rcx +adox r13, rbp +adcx r13, r14 +xor r15d, r15d +mov rdx, qword ptr [rsp+16] +mulx rbp, rcx, qword ptr [rsp+32] +adcx r10, rcx +adox r11, rbp +mulx rbp, rcx, qword ptr [rsp+40] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+48] +adcx r12, rcx +adox r13, rbp +mulx rbp, rcx, qword ptr [rsp+56] +adcx r13, rcx +adox r14, rbp +adcx r14, r15 +xor eax, eax +mov rdx, qword ptr [rsp+24] +mulx rbp, rcx, qword ptr [rsp+32] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+40] +adcx r12, rcx +adox r13, rbp +mulx rbp, rcx, qword ptr [rsp+48] +adcx r13, rcx +adox r14, rbp +mulx rbp, rcx, qword ptr [rsp+56] +adcx r14, rcx +adox r15, rbp +adcx r15, rax +xor ebp, ebp +mov rdx, 38 +mulx r12, rax, r12 +adcx r8, rax +adox r9, r12 +mulx r13, rcx, r13 +adcx r9, rcx +adox r10, r13 +mulx r14, rcx, r14 +adcx r10, rcx +adox r11, r14 +mulx r15, rcx, r15 +adcx r11, rcx +adox r15, rbp ;# zero +adcx r15, rbp ;# zero +shld r15, r11, 1 +btr r11, 63 +imul r15, r15, 19 +add r8, r15 +adc r9, 0 +adc r10, 0 +adc r11, 0 + +;# update X2 +mov qword ptr [rsp+160], r8 +mov qword ptr [rsp+168], r9 +mov qword ptr [rsp+176], r10 +mov qword ptr [rsp+184], r11 + +;# Z2 ← T3 · T4 +xor r13d, r13d +mov rdx, qword ptr [rsp+96] +mulx r9, r8, qword ptr [rsp+64] +mulx r10, rcx, qword ptr [rsp+72] +adcx r9, rcx +mulx r11, rcx, qword ptr [rsp+80] +adcx r10, rcx +mulx r12, rcx, qword ptr [rsp+88] +adcx r11, rcx +adcx r12, r13 +xor r14d, r14d +mov rdx, qword ptr [rsp+104] +mulx rbp, rcx, qword ptr [rsp+64] +adcx r9, rcx +adox r10, rbp +mulx rbp, rcx, qword ptr [rsp+72] +adcx r10, rcx +adox r11, rbp +mulx rbp, rcx, qword ptr [rsp+80] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+88] +adcx r12, rcx +adox r13, rbp +adcx r13, r14 +xor r15d, r15d +mov rdx, qword ptr [rsp+112] +mulx rbp, rcx, qword ptr [rsp+64] +adcx r10, rcx +adox r11, rbp +mulx rbp, rcx, qword ptr [rsp+72] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+80] +adcx r12, rcx +adox r13, rbp +mulx rbp, rcx, qword ptr [rsp+88] +adcx r13, rcx +adox r14, rbp +adcx r14, r15 +xor eax, eax +mov rdx, qword ptr [rsp+120] +mulx rbp, rcx, qword ptr [rsp+64] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+72] +adcx r12, rcx +adox r13, rbp +mulx rbp, rcx, qword ptr [rsp+80] +adcx r13, rcx +adox r14, rbp +mulx rbp, rcx, qword ptr [rsp+88] +adcx r14, rcx +adox r15, rbp +adcx r15, rax +xor ebp, ebp +mov rdx, 38 +mulx r12, rax, r12 +adcx r8, rax +adox r9, r12 +mulx r13, rcx, r13 +adcx r9, rcx +adox r10, r13 +mulx r14, rcx, r14 +adcx r10, rcx +adox r11, r14 +mulx r15, rcx, r15 +adcx r11, rcx +adox r15, rbp ;# zero +adcx r15, rbp ;# zero +shld r15, r11, 1 +btr r11, 63 +imul r15, r15, 19 +add r8, r15 +adc r9, 0 +adc r10, 0 +adc r11, 0 + +;# update Z2 +mov qword ptr [rsp+224], r8 +mov qword ptr [rsp+232], r9 +mov qword ptr [rsp+240], r10 +mov qword ptr [rsp+248], r11 + +;# Z3 ← Z3 · X1 +xor r13d, r13d +mov rdx, qword ptr [rsp+256] +mulx r9, r8, qword ptr [rsp+128] +mulx r10, rcx, qword ptr [rsp+136] +adcx r9, rcx +mulx r11, rcx, qword ptr [rsp+144] +adcx r10, rcx +mulx r12, rcx, qword ptr [rsp+152] +adcx r11, rcx +adcx r12, r13 +xor r14d, r14d +mov rdx, qword ptr [rsp+264] +mulx rbp, rcx, qword ptr [rsp+128] +adcx r9, rcx +adox r10, rbp +mulx rbp, rcx, qword ptr [rsp+136] +adcx r10, rcx +adox r11, rbp +mulx rbp, rcx, qword ptr [rsp+144] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+152] +adcx r12, rcx +adox r13, rbp +adcx r13, r14 +xor r15d, r15d +mov rdx, qword ptr [rsp+272] +mulx rbp, rcx, qword ptr [rsp+128] +adcx r10, rcx +adox r11, rbp +mulx rbp, rcx, qword ptr [rsp+136] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+144] +adcx r12, rcx +adox r13, rbp +mulx rbp, rcx, qword ptr [rsp+152] +adcx r13, rcx +adox r14, rbp +adcx r14, r15 +xor eax, eax +mov rdx, qword ptr [rsp+280] +mulx rbp, rcx, qword ptr [rsp+128] +adcx r11, rcx +adox r12, rbp +mulx rbp, rcx, qword ptr [rsp+136] +adcx r12, rcx +adox r13, rbp +mulx rbp, rcx, qword ptr [rsp+144] +adcx r13, rcx +adox r14, rbp +mulx rbp, rcx, qword ptr [rsp+152] +adcx r14, rcx +adox r15, rbp +adcx r15, rax +xor ebp, ebp +mov rdx, 38 +mulx r12, rax, r12 +adcx r8, rax +adox r9, r12 +mulx r13, rcx, r13 +adcx r9, rcx +adox r10, r13 +mulx r14, rcx, r14 +adcx r10, rcx +adox r11, r14 +mulx r15, rcx, r15 +adcx r11, rcx +adox r15, rbp ;# zero +adcx r15, rbp ;# zero +shld r15, r11, 1 +btr r11, 63 +imul r15, r15, 19 +add r8, r15 +adc r9, 0 +adc r10, 0 +adc r11, 0 + +;# update Z3 +mov qword ptr [rsp+256], r8 +mov qword ptr [rsp+264], r9 +mov qword ptr [rsp+272], r10 +mov qword ptr [rsp+280], r11 + +movzx ecx, byte ptr [rsp+352] +sub ecx, 1 +mov byte ptr [rsp+352], cl +;# cmp cl, 0 +jge loop_bit_x + +mov byte ptr [rsp+352], 7 +mov rsi, qword ptr [rsp+312] +mov eax, dword ptr [rsp+356] +sub eax, 1 +mov dword ptr [rsp+356], eax +;# cmp eax, 0 +jge loop_byte_x + +;# load select, AKA the LSB +mov bl, byte ptr [rsp+353] +cmp bl, 1 + +;# CSelect(X2,X3,select) +mov rsi, qword ptr [rsp+160] +mov rbp, qword ptr [rsp+168] +mov rcx, qword ptr [rsp+176] +mov rax, qword ptr [rsp+184] +mov r12, qword ptr [rsp+192] +mov r13, qword ptr [rsp+200] +mov r14, qword ptr [rsp+208] +mov r15, qword ptr [rsp+216] +cmove rsi, r12 +cmove rbp, r13 +cmove rcx, r14 +cmove rax, r15 + +;# CSelect(Z2,Z3,select) +mov r8, qword ptr [rsp+224] +mov r9, qword ptr [rsp+232] +mov r10, qword ptr [rsp+240] +mov r11, qword ptr [rsp+248] +mov r12, qword ptr [rsp+256] +mov r13, qword ptr [rsp+264] +mov r14, qword ptr [rsp+272] +mov r15, qword ptr [rsp+280] +cmove r8, r12 +cmove r9, r13 +cmove r10, r14 +cmove r11, r15 + +;# result ← X2 · Z2^(-1) + +;# result in: r8, r9, r10, r11 +;# Stack variables: +;# 0 T0 +;# 32 T1 +;# 64 T2 +;# 96 T3 +;# 128 T4 +;# 352 invtable position (4 bytes) + +;# T0 ← X2 +mov qword ptr [rsp+0], rsi +mov qword ptr [rsp+8], rbp +mov qword ptr [rsp+16], rcx +mov qword ptr [rsp+24], rax + +;# T2 ← Z2 +mov qword ptr [rsp+64], r8 +mov qword ptr [rsp+72], r9 +mov qword ptr [rsp+80], r10 +mov qword ptr [rsp+88], r11 + +mov dword ptr [rsp+352], 0 +lea r12, [invtable+REG_REL] + +inv_loop_x: +;# ecx = current instruction +mov ecx, dword ptr [r12] +test cl, cl +jz skip_square_x + +square_loop_x: +xor r13d, r13d +mov rdx, r8 +mulx rax, rbp, r9 +mulx rsi, rdi, r10 +adcx rax, rdi +mulx r12, rdi, r11 +adcx rsi, rdi +adcx r12, r13 +mov rdx, r9 +xor r14d, r14d +mulx rdx, rdi, r10 +adcx rsi, rdi +adox r12, rdx +mov rdx, r9 +mulx rdx, rdi, r11 +adcx r12, rdi +adox r13, rdx +adcx r13, r14 +xor r15d, r15d +mov rdx, r10 +mulx r14, rdi, r11 +adcx r13, rdi +adcx r14, r15 +shld r15, r14, 1 +shld r14, r13, 1 +shld r13, r12, 1 +shld r12, rsi, 1 +shld rsi, rax, 1 +shld rax, rbp, 1 +shl rbp, 1 +xor edx, edx +mov rdx, r8 +mulx rdx, rbx, rdx +adcx rbp, rdx +mov rdx, r9 +mulx rdx, rdi, rdx +adcx rax, rdi +adcx rsi, rdx +mov rdx, r10 +mulx rdx, rdi, rdx +adcx r12, rdi +adcx r13, rdx +mov rdx, r11 +mulx rdx, rdi, rdx +adcx r14, rdi +adcx r15, rdx +;# xor r9d, r9d +mov rdx, 38 +mulx r9, r8, r12 +xor r12d, r12d +adcx r8, rbx +adox r9, rbp +mulx r10, rdi, r13 +adcx r9, rdi +adox r10, rax +mulx r11, rdi, r14 +adcx r10, rdi +adox r11, rsi +mulx r15, rdi, r15 +adcx r11, rdi +adox r15, r12 ;# zero +adcx r15, r12 ;# zero +shld r15, r11, 1 +btr r11, 63 +imul r15, r15, 19 +add r8, r15 +adc r9, 0 +adc r10, 0 +adc r11, 0 +sub cl, 1 +jnz square_loop_x + +skip_square_x: +shr ecx, 8 +test cl, cl +jnz skip_mul_x + +;# T4 ← result +lea rbx, [rsp+128] +mov qword ptr [rbx], r8 +mov qword ptr [rbx+8], r9 +mov qword ptr [rbx+16], r10 +mov qword ptr [rbx+24], r11 + +movzx edi, ch ;# mul_source +shl edi, 5 +add rdi, rsp + +;# result ← T4 · mul_source +xor r13d, r13d +mov rdx, qword ptr [rbx] +mulx r9, r8, qword ptr [rdi] +mulx r10, rsi, qword ptr [rdi+8] +adcx r9, rsi +mulx r11, rsi, qword ptr [rdi+16] +adcx r10, rsi +mulx r12, rsi, qword ptr [rdi+24] +adcx r11, rsi +adcx r12, r13 ;# zero +xor r14d, r14d +mov rdx, qword ptr [rbx+8] +mulx rbp, rsi, qword ptr [rdi] +adcx r9, rsi +adox r10, rbp +mulx rbp, rsi, qword ptr [rdi+8] +adcx r10, rsi +adox r11, rbp +mulx rbp, rsi, qword ptr [rdi+16] +adcx r11, rsi +adox r12, rbp +mulx rbp, rsi, qword ptr [rdi+24] +adcx r12, rsi +adox r13, rbp +adcx r13, r14 ;#zero +xor r15d, r15d +mov rdx, qword ptr [rbx+16] +mulx rbp, rsi, qword ptr [rdi] +adcx r10, rsi +adox r11, rbp +mulx rbp, rsi, qword ptr [rdi+8] +adcx r11, rsi +adox r12, rbp +mulx rbp, rsi, qword ptr [rdi+16] +adcx r12, rsi +adox r13, rbp +mulx rbp, rsi, qword ptr [rdi+24] +adcx r13, rsi +adox r14, rbp +adcx r14, r15 +xor eax, eax +mov rdx, qword ptr [rbx+24] +mulx rbp, rsi, qword ptr [rdi] +adcx r11, rsi +adox r12, rbp +mulx rbp, rsi, qword ptr [rdi+8] +adcx r12, rsi +adox r13, rbp +mulx rbp, rsi, qword ptr [rdi+16] +adcx r13, rsi +adox r14, rbp +mulx rbp, rsi, qword ptr [rdi+24] +adcx r14, rsi +adox r15, rbp +adcx r15, rax ;# zero +xor ebp, ebp +mov rdx, 38 +mulx r12, rax, r12 +adcx r8, rax +adox r9, r12 +mulx r13, rsi, r13 +adcx r9, rsi +adox r10, r13 +mulx r14, rsi, r14 +adcx r10, rsi +adox r11, r14 +mulx r15, rsi, r15 +adcx r11, rsi +adox r15, rbp ;# zero +adcx r15, rbp ;# zero +shld r15, r11, 1 +btr r11, 63 +imul r15, r15, 19 +add r8, r15 +adc r9, 0 +adc r10, 0 +adc r11, 0 + +skip_mul_x: +shr ecx, 11 +jz skip_store_x + +add rcx, rsp + +;# store temp T[1-3] +mov qword ptr [rcx], r8 +mov qword ptr [rcx+8], r9 +mov qword ptr [rcx+16], r10 +mov qword ptr [rcx+24], r11 + +skip_store_x: +mov ebx, dword ptr [rsp+352] +add ebx, 4 +lea r12, [invtable+REG_REL] +add r12, rbx +mov dword ptr [rsp+352], ebx + +cmp ebx, 52 +jb inv_loop_x + +;# final reduction + +xor r12d, r12d +btr r11, 63 +adc r12, 0 +imul r12, r12, 19 +add r8, r12 +adc r9, 0 + +mov rax, r8 +mov rcx, r9 +mov rdx, r10 +mov rsi, r11 +sub r8, qword ptr [p0+REG_REL] +sbb r9, qword ptr [p12+REG_REL] +sbb r10, qword ptr [p12+REG_REL] +sbb r11, qword ptr [p3+REG_REL] +bt r11, 63 +cmovb r8, rax +cmovb r9, rcx +cmovb r10, rdx +cmovb r11, rsi + +mov rdi, qword ptr [rsp+304] + +;# output the result +mov qword ptr [rdi], r8 +mov qword ptr [rdi+8], r9 +mov qword ptr [rdi+16], r10 +mov qword ptr [rdi+24], r11 + +;# restore registers +mov rbx, qword ptr [rsp+288] +mov rbp, qword ptr [rsp+296] +;# mov rdi, qword ptr [rsp+304] +;# mov rsi, qword ptr [rsp+312] +mov r12, qword ptr [rsp+320] +mov r13, qword ptr [rsp+328] +mov r14, qword ptr [rsp+336] +mov r15, qword ptr [rsp+344] + +add rsp, 392 diff --git a/external/src/mx25519/arm64/scalarmult.S b/external/src/mx25519/arm64/scalarmult.S new file mode 100644 index 0000000..d136c61 --- /dev/null +++ b/external/src/mx25519/arm64/scalarmult.S @@ -0,0 +1,1650 @@ +/* Copyright (c) 2021-2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#if defined(__APPLE__) +#define DECL(x) _##x +#else +#define DECL(x) x +#endif + +/* X25519-AArch64 by Emil Lenngren (2018) + * + * To the extent possible under law, the person who associated CC0 with + * X25519-AArch64 has waived all copyright and related or neighboring rights + * to X25519-AArch64. + * + * You should have received a copy of the CC0 legalcode along with this + * work. If not, see . + */ + +/* + * This is an AArch64 implementation of X25519. + * It follows the reference implementation where the representation of + * a field element [0..2^255-19) is represented by a 256-bit little endian integer, + * reduced modulo 2^256-38, and may possibly be in the range [2^256-38..2^256). + * The scalar is a 256-bit integer where certain bits are hardcoded per specification. + * + * The implementation runs in constant time (~145k cycles on Cortex-A53), + * and no conditional branches or memory access pattern depend on secret data. + */ + +.cpu generic+fp+simd + .text + .align 2 + + // in: x0: pointer + // out: x0: loaded value + //.type load64unaligned, %function +load64unaligned: + ldrb w1, [x0] + ldrb w2, [x0, #1] + ldrb w3, [x0, #2] + ldrb w4, [x0, #3] + ldrb w5, [x0, #4] + ldrb w6, [x0, #5] + ldrb w7, [x0, #6] + ldrb w8, [x0, #7] + + orr w1, w1, w2, lsl #8 + orr w3, w3, w4, lsl #8 + orr w5, w5, w6, lsl #8 + orr w7, w7, w8, lsl #8 + + orr w1, w1, w3, lsl #16 + orr w5, w5, w7, lsl #16 + + orr x0, x1, x5, lsl #32 + + ret + //.size load64unaligned, .-load64unaligned + + // in: x0: pointer + // out: x0-x3: loaded value + //.type load256unaligned, %function +load256unaligned: + stp x29, x30, [sp, #-64]! + mov x29, sp + stp x19, x20, [sp, #16] + stp x21, x22, [sp, #32] + + mov x19, x0 + bl load64unaligned + mov x20, x0 + add x0, x19, #8 + bl load64unaligned + mov x21, x0 + add x0, x19, #16 + bl load64unaligned + mov x22, x0 + add x0, x19, #24 + bl load64unaligned + mov x3, x0 + + mov x0, x20 + mov x1, x21 + mov x2, x22 + + ldp x19, x20, [sp, #16] + ldp x21, x22, [sp, #32] + ldp x29, x30, [sp], #64 + ret + //.size load256unaligned, .-load256unaligned + + // in: x1: scalar pointer, x2: base point pointer + // out: x0: result pointer + .global DECL(mx25519_scalarmult_arm64) + //.type mx25519_scalarmult_arm64, %function +DECL(mx25519_scalarmult_arm64): + stp x29, x30, [sp, #-160]! + mov x29, sp + stp x19, x20, [sp, #16] + stp x21, x22, [sp, #32] + stp x23, x24, [sp, #48] + stp x25, x26, [sp, #64] + stp x27, x28, [sp, #80] + stp d8, d9, [sp, #96] + stp d10, d11, [sp, #112] + stp d12, d13, [sp, #128] + stp d14, d15, [sp, #144] + sub sp, sp, 192 + + // 0: mask1, 8: mask2, 16: AA, 56: B/BB, 96: counter, 100: lastbit, 104: scalar, 136: X1, 176: outptr, 184: padding, 192: fp, 200: lr + + str x0, [sp, #176] // outptr + mov x19, x2 // point + + mov x0, x1 // scalar + bl load256unaligned + + and x3, x3, #0x7fffffffffffffff + //and x0, x0, #0xfffffffffffffff8 + //orr x3, x3, #0x4000000000000000 //do not set bit 254 + + stp x0, x1, [sp, #104] + stp x2, x3, [sp, #104+16] + + mov x0, x19 // point + bl load256unaligned + + // Unpack point (discard most significant bit) + lsr x12, x0, #51 + lsr x17, x2, #51 + orr w12, w12, w1, lsl #13 + orr w17, w17, w3, lsl #13 + ubfx x8, x3, #12, #26 + ubfx x9, x3, #38, #25 + ubfx x11, x0, #26, #25 + ubfx x13, x1, #13, #25 + lsr x14, x1, #38 + ubfx x16, x2, #25, #26 + and w10, w0, #0x3ffffff + and w12, w12, #0x3ffffff + and w15, w2, #0x1ffffff + and w17, w17, #0x1ffffff + stp w10, w11, [sp, #136] + stp w12, w13, [sp, #136+8] + stp w14, w15, [sp, #136+16] + stp w16, w17, [sp, #136+24] + stp w8, w9, [sp, #136+32] + + // X2 (initially set to 1) + mov x1, #1 + mov v0.d[0], x1 + mov v2.d[0], xzr + mov v4.d[0], xzr + mov v6.d[0], xzr + mov v8.d[0], xzr + + // Z2 (initially set to 0) + mov v1.d[0], xzr + mov v3.d[0], xzr + mov v5.d[0], xzr + mov v7.d[0], xzr + mov v9.d[0], xzr + + // X3 (initially set to X1) + mov v10.s[0], w10 + mov v10.s[1], w11 + mov v12.s[0], w12 + mov v12.s[1], w13 + mov v14.s[0], w14 + mov v14.s[1], w15 + mov v16.s[0], w16 + mov v16.s[1], w17 + mov v18.s[0], w8 + mov v18.s[1], w9 + + // Z3 (initially set to 1) + mov v11.d[0], x1 + mov v13.d[0], xzr + mov v15.d[0], xzr + mov v17.d[0], xzr + mov v19.d[0], xzr + + mov x0, #255-1 // 255 iterations + str w0, [sp, #96] + + + mov w30, #19 + dup v31.2s, w30 + mov x0, #(1<<26)-1 + dup v30.2d, x0 + ldr x0, =0x07fffffe07fffffc + stp x0, x0, [sp] + sub x1, x0, #0xfc-0xb4 + stp x1, x0, [sp, #8] + + ldr d28, [sp, #8] + ldr d29, [sp] + + ldrb w1, [sp, #135] + lsr w1, w1, #6 + str w1, [sp, #100] + +.Lmainloop: + tst w1, #1 + + // (2^255-19)*4 - Z_ + sub v20.2s, v28.2s, v1.2s + sub v21.2s, v29.2s, v3.2s + sub v22.2s, v29.2s, v5.2s + sub v23.2s, v29.2s, v7.2s + sub v24.2s, v29.2s, v9.2s + + sub v25.2s, v28.2s, v11.2s + sub v26.2s, v29.2s, v13.2s + sub v27.2s, v29.2s, v15.2s + + sub v28.2s, v29.2s, v17.2s + sub v29.2s, v29.2s, v19.2s + + // ... + X_ + add v20.2s, v0.2s, v20.2s + add v21.2s, v2.2s, v21.2s + add v22.2s, v4.2s, v22.2s + add v23.2s, v6.2s, v23.2s + add v24.2s, v8.2s, v24.2s + + add v25.2s, v10.2s, v25.2s + add v26.2s, v12.2s, v26.2s + add v27.2s, v14.2s, v27.2s + add v28.2s, v16.2s, v28.2s + add v29.2s, v18.2s, v29.2s + + // X_ + Z_ + add v0.2s, v0.2s, v1.2s + add v10.2s, v10.2s, v11.2s + add v2.2s, v2.2s, v3.2s + add v12.2s, v12.2s, v13.2s + fcsel d1, d0, d10, eq + add v4.2s, v4.2s, v5.2s + fcsel d3, d2, d12, eq + add v14.2s, v14.2s, v15.2s + add v6.2s, v6.2s, v7.2s + add v16.2s, v16.2s, v17.2s + mov x0, v1.d[0] + add v8.2s, v8.2s, v9.2s + mov x2, v3.d[0] + add v18.2s, v18.2s, v19.2s + fcsel d5, d4, d14, eq + + // [B A] + trn2 v1.2s, v0.2s, v20.2s + fcsel d7, d6, d16, eq + trn1 v0.2s, v0.2s, v20.2s + fcsel d9, d8, d18, eq + trn2 v3.2s, v2.2s, v21.2s + mov x4, v5.d[0] + trn1 v2.2s, v2.2s, v21.2s + mov x6, v7.d[0] + trn2 v5.2s, v4.2s, v22.2s + mov x8, v9.d[0] + trn1 v4.2s, v4.2s, v22.2s + fcsel d20, d20, d25, eq + trn2 v7.2s, v6.2s, v23.2s + fcsel d21, d21, d26, eq + trn1 v6.2s, v6.2s, v23.2s + fcsel d22, d22, d27, eq + trn2 v9.2s, v8.2s, v24.2s + fcsel d23, d23, d28, eq + trn1 v8.2s, v8.2s, v24.2s + fcsel d24, d24, d29, eq + + // [C D] + trn2 v11.2s, v25.2s, v10.2s + trn1 v10.2s, v25.2s, v10.2s + trn2 v13.2s, v26.2s, v12.2s + trn1 v12.2s, v26.2s, v12.2s + trn2 v15.2s, v27.2s, v14.2s + trn1 v14.2s, v27.2s, v14.2s + trn2 v17.2s, v28.2s, v16.2s + trn1 v16.2s, v28.2s, v16.2s + stp d20, d21, [sp, #56] + trn2 v19.2s, v29.2s, v18.2s + stp d22, d23, [sp, #56+16] + trn1 v18.2s, v29.2s, v18.2s + str d24, [sp, #56+32] + + + //v0-v9: [B A] + //v10-v19: [C D] + + + umull v29.2d, v0.2s, v19.2s + lsr x1, x0, #32 + umlal v29.2d, v2.2s, v17.2s + lsr x3, x2, #32 + umlal v29.2d, v4.2s, v15.2s + lsr x5, x4, #32 + umlal v29.2d, v6.2s, v13.2s + lsr x7, x6, #32 + umlal v29.2d, v8.2s, v11.2s + lsr x9, x8, #32 + mul v19.2s, v19.2s, v31.2s + add x21, x9, x9 + umull v28.2d, v1.2s, v17.2s + add x17, x8, x8 + umlal v28.2d, v3.2s, v15.2s + add x16, x7, x7 + umlal v28.2d, v5.2s, v13.2s + add x15, x6, x6 + umlal v28.2d, v7.2s, v11.2s + add x14, x5, x5 + umlal v28.2d, v9.2s, v19.2s + add x13, x4, x4 + umlal v29.2d, v1.2s, v18.2s + add x12, x3, x3 + umlal v29.2d, v3.2s, v16.2s + add x11, x2, x2 + umlal v29.2d, v5.2s, v14.2s + add x10, x1, x1 + umlal v29.2d, v7.2s, v12.2s + umull x28, w4, w4 + umlal v29.2d, v9.2s, v10.2s + umull x19, w4, w14 + shl v28.2d, v28.2d, #1 + mul w9, w9, w30 + umull v27.2d, v0.2s, v17.2s + mul w7, w7, w30 + umlal v27.2d, v2.2s, v15.2s + mul w5, w5, w30 + umlal v27.2d, v4.2s, v13.2s + umaddl x28, w9, w21, x28 + umlal v27.2d, v6.2s, v11.2s + umaddl x19, w0, w21, x19 + umlal v27.2d, v8.2s, v19.2s + umull x20, w0, w0 + mul v17.2s, v17.2s, v31.2s + umull x21, w0, w10 + umlal v28.2d, v0.2s, v18.2s + umull x22, w0, w11 + umlal v28.2d, v2.2s, v16.2s + umull x23, w0, w12 + umlal v28.2d, v4.2s, v14.2s + umull x24, w0, w13 + umlal v28.2d, v6.2s, v12.2s + umull x25, w0, w14 + umlal v28.2d, v8.2s, v10.2s + umull x26, w0, w15 + mul v18.2s, v18.2s, v31.2s + umull x27, w0, w16 + umull v26.2d, v1.2s, v15.2s + umaddl x28, w0, w17, x28 + umlal v26.2d, v3.2s, v13.2s + mul w0, w6, w30 + umlal v26.2d, v5.2s, v11.2s + umaddl x22, w1, w10, x22 + umlal v26.2d, v7.2s, v19.2s + umaddl x23, w1, w11, x23 + umlal v26.2d, v9.2s, v17.2s + umaddl x24, w10, w12, x24 + umlal v27.2d, v1.2s, v16.2s + umaddl x25, w1, w13, x25 + umlal v27.2d, v3.2s, v14.2s + umaddl x26, w10, w14, x26 + umlal v27.2d, v5.2s, v12.2s + umaddl x27, w1, w15, x27 + umlal v27.2d, v7.2s, v10.2s + umaddl x28, w10, w16, x28 + umlal v27.2d, v9.2s, v18.2s + umaddl x19, w1, w17, x19 + shl v26.2d, v26.2d, #1 + mul w1, w8, w30 + umull v25.2d, v0.2s, v15.2s + umaddl x24, w2, w2, x24 + umlal v25.2d, v2.2s, v13.2s + umaddl x25, w2, w12, x25 + umlal v25.2d, v4.2s, v11.2s + umaddl x26, w2, w13, x26 + umlal v25.2d, v6.2s, v19.2s + umaddl x27, w2, w14, x27 + umlal v25.2d, v8.2s, v17.2s + umaddl x28, w2, w15, x28 + mul v15.2s, v15.2s, v31.2s + umaddl x19, w2, w16, x19 + umlal v26.2d, v0.2s, v16.2s + umaddl x26, w3, w12, x26 + umlal v26.2d, v2.2s, v14.2s + umaddl x27, w3, w13, x27 + umlal v26.2d, v4.2s, v12.2s + umaddl x28, w12, w14, x28 + umlal v26.2d, v6.2s, v10.2s + umaddl x19, w3, w15, x19 + umlal v26.2d, v8.2s, v18.2s + umaddl x26, w1, w8, x26 + mul v16.2s, v16.2s, v31.2s + umaddl x22, w0, w6, x22 + umull v24.2d, v1.2s, v13.2s + add x19, x19, x28, lsr #26 + umlal v24.2d, v3.2s, v11.2s + umaddl x20, w5, w14, x20 + umlal v24.2d, v5.2s, v19.2s + add x20, x20, x19, lsr #25 + umlal v24.2d, v7.2s, v17.2s + bic x2, x19, #0x1ffffff + umlal v24.2d, v9.2s, v15.2s + add x20, x20, x2, lsr #24 + umlal v25.2d, v1.2s, v14.2s + and x19, x19, #0x1ffffff + umlal v25.2d, v3.2s, v12.2s + add x20, x20, x2, lsr #21 + umlal v25.2d, v5.2s, v10.2s + umaddl x24, w7, w16, x24 + umlal v25.2d, v7.2s, v18.2s + add x2, x10, x10 + umlal v25.2d, v9.2s, v16.2s + add x3, x12, x12 + shl v24.2d, v24.2d, #1 + add x4, x14, x14 + umull v23.2d, v0.2s, v13.2s + add x5, x16, x16 + umlal v23.2d, v2.2s, v11.2s + umaddl x20, w0, w13, x20 + umlal v23.2d, v4.2s, v19.2s + umaddl x21, w0, w14, x21 + umlal v23.2d, v6.2s, v17.2s + and x28, x28, #0x3ffffff + umlal v23.2d, v8.2s, v15.2s + umaddl x20, w7, w3, x20 + mul v13.2s, v13.2s, v31.2s + umaddl x21, w7, w13, x21 + umlal v24.2d, v0.2s, v14.2s + umaddl x22, w7, w4, x22 + umlal v24.2d, v2.2s, v12.2s + umaddl x23, w7, w15, x23 + umlal v24.2d, v4.2s, v10.2s + umaddl x20, w1, w11, x20 + umlal v24.2d, v6.2s, v18.2s + umaddl x21, w1, w12, x21 + umlal v24.2d, v8.2s, v16.2s + umaddl x22, w1, w13, x22 + mul v14.2s, v14.2s, v31.2s + umaddl x23, w1, w14, x23 + umull v22.2d, v1.2s, v11.2s + umaddl x24, w1, w15, x24 + umlal v22.2d, v3.2s, v19.2s + umaddl x25, w1, w16, x25 + umlal v22.2d, v5.2s, v17.2s + umaddl x20, w9, w2, x20 + umlal v22.2d, v7.2s, v15.2s + umaddl x21, w9, w11, x21 + umlal v22.2d, v9.2s, v13.2s + umaddl x22, w9, w3, x22 + umlal v23.2d, v1.2s, v12.2s + umaddl x23, w9, w13, x23 + umlal v23.2d, v3.2s, v10.2s + umaddl x24, w9, w4, x24 + umlal v23.2d, v5.2s, v18.2s + umaddl x25, w9, w15, x25 + umlal v23.2d, v7.2s, v16.2s + umaddl x26, w9, w5, x26 + umlal v23.2d, v9.2s, v14.2s + umaddl x27, w9, w17, x27 + shl v22.2d, v22.2d, #1 + add x21, x21, x20, lsr #26 + umull v21.2d, v0.2s, v11.2s + and x20, x20, #0x3ffffff + umlal v21.2d, v2.2s, v19.2s + add x22, x22, x21, lsr #25 + umlal v21.2d, v4.2s, v17.2s + bfi x20, x21, #32, #25 + umlal v21.2d, v6.2s, v15.2s + add x23, x23, x22, lsr #26 + umlal v21.2d, v8.2s, v13.2s + and x22, x22, #0x3ffffff + mul v11.2s, v11.2s, v31.2s + add x24, x24, x23, lsr #25 + umlal v22.2d, v0.2s, v12.2s + bfi x22, x23, #32, #25 + umlal v22.2d, v2.2s, v10.2s + add x25, x25, x24, lsr #26 + umlal v22.2d, v4.2s, v18.2s + and x24, x24, #0x3ffffff + umlal v22.2d, v6.2s, v16.2s + add x26, x26, x25, lsr #25 + umlal v22.2d, v8.2s, v14.2s + bfi x24, x25, #32, #25 + mul v12.2s, v12.2s, v31.2s + add x27, x27, x26, lsr #26 + umull v20.2d, v1.2s, v19.2s + and x26, x26, #0x3ffffff + umlal v20.2d, v3.2s, v17.2s + add x28, x28, x27, lsr #25 + umlal v20.2d, v5.2s, v15.2s + bfi x26, x27, #32, #25 + ushr v15.2d, v30.2d, #1 + add x19, x19, x28, lsr #26 + usra v23.2d, v22.2d, #26 + and x28, x28, #0x3ffffff + and v22.16b, v22.16b, v30.16b + bfi x28, x19, #32, #26 + umlal v21.2d, v1.2s, v10.2s + stp x20, x22, [sp, #16] + usra v24.2d, v23.2d, #25 + stp x24, x26, [sp, #32] + and v23.16b, v23.16b, v15.16b + str x28, [sp, #48] + umlal v20.2d, v7.2s, v13.2s + ldr x8, [sp, #88] + usra v25.2d, v24.2d, #26 + ldr x6, [sp, #80] + and v24.16b, v24.16b, v30.16b + ldr x4, [sp, #72] + umlal v21.2d, v3.2s, v18.2s + ldr x2, [sp, #64] + usra v26.2d, v25.2d, #25 + lsr x9, x8, #32 + and v25.16b, v25.16b, v15.16b + ldr x0, [sp, #56] + umlal v20.2d, v9.2s, v11.2s + lsr x7, x6, #32 + usra v27.2d, v26.2d, #26 + lsr x5, x4, #32 + and v26.16b, v26.16b, v30.16b + lsr x3, x2, #32 + umlal v21.2d, v5.2s, v16.2s + lsr x1, x0, #32 + umlal v21.2d, v7.2s, v14.2s + add x21, x9, x9 + umlal v21.2d, v9.2s, v12.2s + add x17, x8, x8 + usra v28.2d, v27.2d, #25 + add x16, x7, x7 + and v27.16b, v27.16b, v15.16b + add x15, x6, x6 + shl v20.2d, v20.2d, #1 + add x14, x5, x5 + usra v29.2d, v28.2d, #26 + add x13, x4, x4 + and v28.16b, v28.16b, v30.16b + add x12, x3, x3 + umlal v20.2d, v0.2s, v10.2s + add x11, x2, x2 + umlal v20.2d, v2.2s, v18.2s + add x10, x1, x1 + umlal v20.2d, v4.2s, v16.2s + umull x28, w4, w4 + umlal v20.2d, v6.2s, v14.2s + umull x19, w4, w14 + umlal v20.2d, v8.2s, v12.2s + mul w9, w9, w30 + bic v19.16b, v29.16b, v15.16b + mul w7, w7, w30 + and v29.16b, v29.16b, v15.16b + mul w5, w5, w30 + usra v20.2d, v19.2d, #25 + umaddl x28, w9, w21, x28 + uzp1 v24.4s, v24.4s, v25.4s + umaddl x19, w0, w21, x19 + usra v20.2d, v19.2d, #24 + umull x20, w0, w0 + uzp1 v25.4s, v26.4s, v27.4s + umull x21, w0, w10 + usra v20.2d, v19.2d, #21 + umull x22, w0, w11 + ld1r {v19.2d}, [sp] + umull x23, w0, w12 + uzp1 v26.4s, v24.4s, v25.4s + umull x24, w0, w13 + usra v21.2d, v20.2d, #26 + umull x25, w0, w14 + and v20.16b, v20.16b, v30.16b + umull x26, w0, w15 + uzp2 v27.4s, v24.4s, v25.4s + umull x27, w0, w16 + usra v22.2d, v21.2d, #25 + umaddl x28, w0, w17, x28 + and v21.16b, v21.16b, v15.16b + mul w0, w6, w30 + trn1 v28.4s, v28.4s, v29.4s + umaddl x22, w1, w10, x22 + usra v23.2d, v22.2d, #26 + umaddl x23, w1, w11, x23 + and v22.16b, v22.16b, v30.16b + umaddl x24, w10, w12, x24 + ldr b0, [sp, #8] + umaddl x25, w1, w13, x25 + uzp1 v20.4s, v20.4s, v21.4s + umaddl x26, w10, w14, x26 + uzp1 v21.4s, v22.4s, v23.4s + umaddl x27, w1, w15, x27 + mov v29.d[0], v28.d[1] + umaddl x28, w10, w16, x28 + uzp1 v24.4s, v20.4s, v21.4s + umaddl x19, w1, w17, x19 + uzp2 v25.4s, v20.4s, v21.4s + mul w1, w8, w30 + add v11.4s, v26.4s, v19.4s + umaddl x24, w2, w2, x24 + add v12.2s, v28.2s, v19.2s + umaddl x25, w2, w12, x25 + mov v19.b[0], v0.b[0] + umaddl x26, w2, w13, x26 + add v20.4s, v24.4s, v25.4s + umaddl x27, w2, w14, x27 + add v21.4s, v26.4s, v27.4s + umaddl x28, w2, w15, x28 + add v22.2s, v28.2s, v29.2s + umaddl x19, w2, w16, x19 + add v10.4s, v24.4s, v19.4s + umaddl x26, w3, w12, x26 + sub v11.4s, v11.4s, v27.4s + umaddl x27, w3, w13, x27 + sub v10.4s, v10.4s, v25.4s + umaddl x28, w12, w14, x28 + sub v12.2s, v12.2s, v29.2s + umaddl x19, w3, w15, x19 + zip1 v0.4s, v10.4s, v20.4s + umaddl x26, w1, w8, x26 + zip2 v2.4s, v10.4s, v20.4s + umaddl x22, w0, w6, x22 + zip1 v4.4s, v11.4s, v21.4s + add x19, x19, x28, lsr #26 + zip2 v6.4s, v11.4s, v21.4s + umaddl x20, w5, w14, x20 + zip1 v8.2s, v12.2s, v22.2s + add x20, x20, x19, lsr #25 + zip2 v9.2s, v12.2s, v22.2s + bic x2, x19, #0x1ffffff + mov v1.d[0], v0.d[1] + add x20, x20, x2, lsr #24 + mov v3.d[0], v2.d[1] + and x19, x19, #0x1ffffff + mov v5.d[0], v4.d[1] + add x20, x20, x2, lsr #21 + mov v7.d[0], v6.d[1] + umaddl x24, w7, w16, x24 + shl v19.2s, v9.2s, #1 + add x2, x10, x10 + shl v18.2s, v8.2s, #1 + add x3, x12, x12 + shl v17.2s, v7.2s, #1 + add x4, x14, x14 + shl v16.2s, v6.2s, #1 + add x5, x16, x16 + shl v10.2s, v5.2s, #1 + umaddl x20, w0, w13, x20 + shl v14.2s, v4.2s, #1 + umaddl x21, w0, w14, x21 + shl v13.2s, v3.2s, #1 + and x28, x28, #0x3ffffff + shl v12.2s, v2.2s, #1 + umaddl x20, w7, w3, x20 + shl v11.2s, v1.2s, #1 + umaddl x21, w7, w13, x21 + umull v29.2d, v0.2s, v19.2s + umaddl x22, w7, w4, x22 + umlal v29.2d, v1.2s, v18.2s + umaddl x23, w7, w15, x23 + umlal v29.2d, v2.2s, v17.2s + umaddl x20, w1, w11, x20 + umlal v29.2d, v3.2s, v16.2s + umaddl x21, w1, w12, x21 + umlal v29.2d, v4.2s, v10.2s + umaddl x22, w1, w13, x22 + umull v28.2d, v0.2s, v18.2s + umaddl x23, w1, w14, x23 + umlal v28.2d, v11.2s, v17.2s + umaddl x24, w1, w15, x24 + umlal v28.2d, v2.2s, v16.2s + umaddl x25, w1, w16, x25 + umlal v28.2d, v13.2s, v10.2s + umaddl x20, w9, w2, x20 + umlal v28.2d, v4.2s, v4.2s + umaddl x21, w9, w11, x21 + mul v4.2s, v9.2s, v31.2s + umaddl x22, w9, w3, x22 + umull v27.2d, v0.2s, v17.2s + umaddl x23, w9, w13, x23 + umlal v27.2d, v1.2s, v16.2s + umaddl x24, w9, w4, x24 + umlal v27.2d, v2.2s, v10.2s + umaddl x25, w9, w15, x25 + umlal v27.2d, v3.2s, v14.2s + umaddl x26, w9, w5, x26 + umlal v28.2d, v4.2s, v19.2s + umaddl x27, w9, w17, x27 + umull v26.2d, v0.2s, v16.2s + add x1, x21, x20, lsr #26 + umlal v26.2d, v11.2s, v10.2s + and x0, x20, #0x3ffffff + umlal v26.2d, v2.2s, v14.2s + add x2, x22, x1, lsr #25 + umlal v26.2d, v13.2s, v3.2s + bfi x0, x1, #32, #25 + umull v25.2d, v0.2s, v10.2s + add x3, x23, x2, lsr #26 + umlal v25.2d, v1.2s, v14.2s + and x2, x2, #0x3ffffff + umlal v25.2d, v2.2s, v13.2s + add x4, x24, x3, lsr #25 + umull v24.2d, v0.2s, v14.2s + bfi x2, x3, #32, #25 + umlal v24.2d, v11.2s, v13.2s + add x5, x25, x4, lsr #26 + umlal v24.2d, v2.2s, v2.2s + and x4, x4, #0x3ffffff + umull v23.2d, v0.2s, v13.2s + add x6, x26, x5, lsr #25 + umlal v23.2d, v1.2s, v12.2s + bfi x4, x5, #32, #25 + umull v22.2d, v0.2s, v12.2s + add x7, x27, x6, lsr #26 + umlal v22.2d, v11.2s, v1.2s + and x6, x6, #0x3ffffff + umull v21.2d, v0.2s, v11.2s + add x8, x28, x7, lsr #25 + umull v20.2d, v0.2s, v0.2s + bfi x6, x7, #32, #25 + usra v29.2d, v28.2d, #26 + add x9, x19, x8, lsr #26 + and v28.16b, v28.16b, v30.16b + and x8, x8, #0x3ffffff + mul v3.2s, v8.2s, v31.2s + bfi x8, x9, #32, #26 + bic v19.16b, v29.16b, v15.16b + and x1, x1, #0x1ffffff + and v9.16b, v29.16b, v15.16b + and x3, x3, #0x1ffffff + usra v20.2d, v19.2d, #25 + and x5, x5, #0x1ffffff + mul v2.2s, v7.2s, v31.2s + and x7, x7, #0x1ffffff + usra v20.2d, v19.2d, #24 + stp x0, x2, [sp, #56] + mul v1.2s, v6.2s, v31.2s + stp x4, x6, [sp, #56+16] + usra v20.2d, v19.2d, #21 + str x8, [sp, #56+32] + mul v0.2s, v5.2s, v31.2s + ldr x21, =0x07fffffe07fffffc + shl v5.2s, v11.2s, #1 + ldr x10, [sp, #16] + shl v7.2s, v13.2s, #1 + ldr x12, [sp, #16+8] + shl v19.2s, v10.2s, #1 + ldr x14, [sp, #16+16] + shl v11.2s, v17.2s, #1 + ldr x16, [sp, #16+24] + umlal v20.2d, v0.2s, v10.2s + ldr x19, [sp, #16+32] + umlal v20.2d, v4.2s, v5.2s + add x12, x12, x21 + umlal v20.2d, v3.2s, v12.2s + add x14, x14, x21 + umlal v20.2d, v2.2s, v7.2s + add x16, x16, x21 + umlal v20.2d, v1.2s, v14.2s + add x19, x19, x21 + umlal v21.2d, v4.2s, v12.2s + movk x21, #0xffb4 + umlal v21.2d, v3.2s, v13.2s + add x10, x10, x21 + umlal v21.2d, v2.2s, v14.2s + sub x10, x10, x0 + umlal v21.2d, v1.2s, v10.2s + sub x12, x12, x2 + umlal v22.2d, v1.2s, v6.2s + sub x14, x14, x4 + umlal v22.2d, v4.2s, v7.2s + sub x16, x16, x6 + umlal v22.2d, v3.2s, v14.2s + sub x19, x19, x8 + umlal v22.2d, v2.2s, v19.2s + mov w0, w0 + usra v21.2d, v20.2d, #26 + mov w2, w2 + umlal v23.2d, v4.2s, v14.2s + mov w4, w4 + umlal v23.2d, v3.2s, v10.2s + mov w6, w6 + umlal v23.2d, v2.2s, v16.2s + mov w8, w8 + usra v22.2d, v21.2d, #25 + lsr x11, x10, #32 + umlal v24.2d, v2.2s, v17.2s + lsr x13, x12, #32 + umlal v24.2d, v4.2s, v19.2s + lsr x15, x14, #32 + umlal v24.2d, v3.2s, v16.2s + lsr x17, x16, #32 + usra v23.2d, v22.2d, #26 + lsr x20, x19, #32 + umlal v25.2d, v4.2s, v16.2s + ldr x21, =121666 + umlal v25.2d, v3.2s, v17.2s + umaddl x9, w20, w21, x9 + usra v24.2d, v23.2d, #25 + umaddl x0, w10, w21, x0 + umlal v26.2d, v3.2s, v8.2s + umaddl x1, w11, w21, x1 + umlal v26.2d, v4.2s, v11.2s + umaddl x2, w12, w21, x2 + usra v25.2d, v24.2d, #26 + lsr x22, x9, #25 + umlal v27.2d, v4.2s, v18.2s + umaddl x3, w13, w21, x3 + and v4.16b, v24.16b, v30.16b + and x9, x9, #0x1ffffff + usra v26.2d, v25.2d, #25 + umaddl x4, w14, w21, x4 + and v5.16b, v25.16b, v15.16b + add x0, x0, x22 + and v0.16b, v20.16b, v30.16b + umaddl x5, w15, w21, x5 + usra v27.2d, v26.2d, #26 + add x0, x0, x22, lsl #1 + and v6.16b, v26.16b, v30.16b + umaddl x6, w16, w21, x6 + and v1.16b, v21.16b, v15.16b + add x0, x0, x22, lsl #4 + usra v28.2d, v27.2d, #25 + umaddl x7, w17, w21, x7 + and v7.16b, v27.16b, v15.16b + umaddl x8, w19, w21, x8 + and v2.16b, v22.16b, v30.16b + add x26, sp, #136 // X1 for ASIMD part + usra v9.2d, v28.2d, #26 + add x27, sp, #16 // AA + and v8.16b, v28.16b, v30.16b + add x28, sp, #56 // BB + and v3.16b, v23.16b, v15.16b + add x1, x1, x0, lsr #26 + ld2 { v0.s, v1.s }[1], [x27], #8 // X1, AA, BB loaded from a64 part + and x0, x0, #0x3ffffff + ld2 { v10.s, v11.s }[1], [x28], #8 + add x2, x2, x1, lsr #25 + ld2 { v10.s, v11.s }[0], [x26], #8 + and x1, x1, #0x1ffffff + ld2 { v2.s, v3.s }[1], [x27], #8 + add x3, x3, x2, lsr #26 + ld2 { v12.s, v13.s }[1], [x28], #8 + and x2, x2, #0x3ffffff + ld2 { v12.s, v13.s }[0], [x26], #8 + add x4, x4, x3, lsr #25 + ld2 { v4.s, v5.s }[1], [x27], #8 + and x3, x3, #0x1ffffff + ld2 { v14.s, v15.s }[1], [x28], #8 + add x5, x5, x4, lsr #26 + ld2 { v14.s, v15.s }[0], [x26], #8 + and x4, x4, #0x3ffffff + ld2 { v6.s, v7.s }[1], [x27], #8 + add x6, x6, x5, lsr #25 + ld2 { v16.s, v17.s }[1], [x28], #8 + and x5, x5, #0x1ffffff + ld2 { v16.s, v17.s }[0], [x26], #8 + add x7, x7, x6, lsr #26 + ld2 { v8.s, v9.s }[1], [x27], #8 + and x6, x6, #0x3ffffff + ld2 { v18.s, v19.s }[1], [x28], #8 + add x8, x8, x7, lsr #25 + ld2 { v18.s, v19.s }[0], [x26], #8 + and x7, x7, #0x1ffffff + umull v29.2d, v6.2s, v13.2s + add x9, x9, x8, lsr #26 + umlal v29.2d, v4.2s, v15.2s + and x8, x8, #0x3ffffff + umlal v29.2d, v0.2s, v19.2s + umull x21, w1, w19 + umlal v29.2d, v2.2s, v17.2s + umull x22, w1, w17 + umlal v29.2d, v8.2s, v11.2s + umull x23, w1, w16 + mul v19.2s, v19.2s, v31.2s + umull x24, w1, w15 + umull v28.2d, v1.2s, v17.2s + umaddl x21, w3, w16, x21 + umlal v28.2d, v3.2s, v15.2s + umaddl x22, w3, w15, x22 + umlal v28.2d, v5.2s, v13.2s + umaddl x23, w3, w14, x23 + umlal v28.2d, v7.2s, v11.2s + umaddl x24, w3, w13, x24 + umlal v28.2d, v9.2s, v19.2s + umaddl x21, w5, w14, x21 + umlal v29.2d, v1.2s, v18.2s + umaddl x22, w5, w13, x22 + umlal v29.2d, v3.2s, v16.2s + umaddl x23, w5, w12, x23 + umlal v29.2d, v5.2s, v14.2s + umaddl x24, w5, w11, x24 + umlal v29.2d, v7.2s, v12.2s + umaddl x21, w7, w12, x21 + umlal v29.2d, v9.2s, v10.2s + umaddl x22, w7, w11, x22 + shl v28.2d, v28.2d, #1 + umaddl x23, w7, w10, x23 + umull v27.2d, v0.2s, v17.2s + mul w27, w7, w30 + umlal v27.2d, v2.2s, v15.2s + mul w25, w9, w30 + umlal v27.2d, v4.2s, v13.2s + mul w26, w8, w30 + umlal v27.2d, v6.2s, v11.2s + mul w28, w6, w30 + umlal v27.2d, v8.2s, v19.2s + umaddl x24, w27, w20, x24 + mul v17.2s, v17.2s, v31.2s + umaddl x21, w9, w10, x21 + umlal v28.2d, v0.2s, v18.2s + umaddl x22, w25, w20, x22 + umlal v28.2d, v2.2s, v16.2s + umaddl x23, w25, w19, x23 + umlal v28.2d, v4.2s, v14.2s + umaddl x24, w25, w17, x24 + umlal v28.2d, v6.2s, v12.2s + add x22, x22, x22 + umlal v28.2d, v8.2s, v10.2s + umaddl x21, w0, w20, x21 + mul v18.2s, v18.2s, v31.2s + add x24, x24, x24 + umull v26.2d, v1.2s, v15.2s + umaddl x22, w0, w19, x22 + umlal v26.2d, v3.2s, v13.2s + umaddl x23, w0, w17, x23 + umlal v26.2d, v5.2s, v11.2s + umaddl x24, w0, w16, x24 + umlal v26.2d, v7.2s, v19.2s + umaddl x21, w2, w17, x21 + umlal v26.2d, v9.2s, v17.2s + umaddl x22, w2, w16, x22 + umlal v27.2d, v1.2s, v16.2s + umaddl x23, w2, w15, x23 + umlal v27.2d, v3.2s, v14.2s + umaddl x24, w2, w14, x24 + umlal v27.2d, v5.2s, v12.2s + umaddl x21, w4, w15, x21 + umlal v27.2d, v7.2s, v10.2s + umaddl x22, w4, w14, x22 + umlal v27.2d, v9.2s, v18.2s + umaddl x23, w4, w13, x23 + shl v26.2d, v26.2d, #1 + umaddl x24, w4, w12, x24 + umull v25.2d, v0.2s, v15.2s + umaddl x21, w6, w13, x21 + umlal v25.2d, v2.2s, v13.2s + umaddl x22, w6, w12, x22 + umlal v25.2d, v4.2s, v11.2s + umaddl x23, w6, w11, x23 + umlal v25.2d, v6.2s, v19.2s + umaddl x24, w6, w10, x24 + umlal v25.2d, v8.2s, v17.2s + umaddl x21, w8, w11, x21 + mul v15.2s, v15.2s, v31.2s + umaddl x22, w8, w10, x22 + umlal v26.2d, v0.2s, v16.2s + umaddl x23, w26, w20, x23 + umlal v26.2d, v2.2s, v14.2s + umaddl x24, w26, w19, x24 + umlal v26.2d, v4.2s, v12.2s + umull x6, w25, w16 + umlal v26.2d, v6.2s, v10.2s + umull x7, w25, w15 + umlal v26.2d, v8.2s, v18.2s + umull x8, w25, w14 + mul v16.2s, v16.2s, v31.2s + umaddl x6, w5, w10, x6 + umull v24.2d, v1.2s, v13.2s + mul w5, w5, w30 + umlal v24.2d, v3.2s, v11.2s + umaddl x7, w27, w17, x7 + umlal v24.2d, v5.2s, v19.2s + umaddl x8, w27, w16, x8 + umlal v24.2d, v7.2s, v17.2s + umaddl x6, w27, w19, x6 + umlal v24.2d, v9.2s, v15.2s + umaddl x7, w5, w20, x7 + umlal v25.2d, v1.2s, v14.2s + umaddl x8, w5, w19, x8 + umlal v25.2d, v3.2s, v12.2s + umaddl x6, w3, w12, x6 + umlal v25.2d, v5.2s, v10.2s + umaddl x7, w3, w11, x7 + umlal v25.2d, v7.2s, v18.2s + umaddl x8, w3, w10, x8 + umlal v25.2d, v9.2s, v16.2s + umaddl x6, w1, w14, x6 + shl v24.2d, v24.2d, #1 + umaddl x7, w1, w13, x7 + umull v23.2d, v0.2s, v13.2s + umaddl x8, w1, w12, x8 + umlal v23.2d, v2.2s, v11.2s + mul w9, w4, w30 + umlal v23.2d, v4.2s, v19.2s + add x7, x7, x7 + umlal v23.2d, v6.2s, v17.2s + umaddl x6, w26, w17, x6 + umlal v23.2d, v8.2s, v15.2s + umaddl x7, w26, w16, x7 + mul v13.2s, v13.2s, v31.2s + umaddl x8, w26, w15, x8 + umlal v24.2d, v0.2s, v14.2s + umaddl x6, w28, w20, x6 + umlal v24.2d, v2.2s, v12.2s + umaddl x7, w28, w19, x7 + umlal v24.2d, v4.2s, v10.2s + umaddl x8, w28, w17, x8 + umlal v24.2d, v6.2s, v18.2s + umaddl x6, w4, w11, x6 + umlal v24.2d, v8.2s, v16.2s + umaddl x7, w4, w10, x7 + mul v14.2s, v14.2s, v31.2s + umaddl x8, w9, w20, x8 + umull v22.2d, v1.2s, v11.2s + umaddl x6, w2, w13, x6 + umlal v22.2d, v3.2s, v19.2s + umaddl x7, w2, w12, x7 + umlal v22.2d, v5.2s, v17.2s + umaddl x8, w2, w11, x8 + umlal v22.2d, v7.2s, v15.2s + umaddl x6, w0, w15, x6 + umlal v22.2d, v9.2s, v13.2s + umaddl x7, w0, w14, x7 + umlal v23.2d, v1.2s, v12.2s + umaddl x8, w0, w13, x8 + umlal v23.2d, v3.2s, v10.2s + mul w4, w3, w30 + umlal v23.2d, v5.2s, v18.2s + add x6, x6, x7, lsr #26 + umlal v23.2d, v7.2s, v16.2s + and x7, x7, #0x3ffffff + umlal v23.2d, v9.2s, v14.2s + add x24, x24, x6, lsr #25 + shl v22.2d, v22.2d, #1 + and x6, x6, #0x1ffffff + umull v21.2d, v0.2s, v11.2s + add x23, x23, x24, lsr #26 + umlal v21.2d, v2.2s, v19.2s + and x24, x24, #0x3ffffff + umlal v21.2d, v4.2s, v17.2s + add x22, x22, x23, lsr #25 + umlal v21.2d, v6.2s, v15.2s + bfi x24, x23, #32, #25 + umlal v21.2d, v8.2s, v13.2s + add x21, x21, x22, lsr #26 + mul v11.2s, v11.2s, v31.2s + and x22, x22, #0x3ffffff + umlal v22.2d, v0.2s, v12.2s + bic x3, x21, #0x3ffffff + umlal v22.2d, v2.2s, v10.2s + lsr x23, x3, #26 + umlal v22.2d, v4.2s, v18.2s + bfi x22, x21, #32, #26 + umlal v22.2d, v6.2s, v16.2s + add x23, x23, x3, lsr #25 + umlal v22.2d, v8.2s, v14.2s + umull x21, w25, w13 + mul v12.2s, v12.2s, v31.2s + add x23, x23, x3, lsr #22 + umull v20.2d, v1.2s, v19.2s + umull x3, w25, w12 + umlal v20.2d, v3.2s, v17.2s + umaddl x23, w25, w11, x23 + umlal v20.2d, v5.2s, v15.2s + umaddl x21, w27, w15, x21 + ushr v15.2d, v30.2d, #1 + umaddl x3, w27, w14, x3 + usra v23.2d, v22.2d, #26 + umaddl x23, w27, w13, x23 + and v22.16b, v22.16b, v30.16b + mul w27, w1, w30 + umlal v21.2d, v1.2s, v10.2s + umaddl x3, w5, w16, x3 + usra v24.2d, v23.2d, #25 + umaddl x23, w5, w15, x23 + and v23.16b, v23.16b, v15.16b + umaddl x21, w5, w17, x21 + umlal v20.2d, v7.2s, v13.2s + umaddl x3, w4, w19, x3 + usra v25.2d, v24.2d, #26 + umaddl x23, w4, w17, x23 + and v24.16b, v24.16b, v30.16b + umaddl x21, w4, w20, x21 + umlal v21.2d, v3.2s, v18.2s + umaddl x3, w1, w10, x3 + usra v26.2d, v25.2d, #25 + umaddl x23, w27, w20, x23 + and v25.16b, v25.16b, v15.16b + umaddl x21, w1, w11, x21 + umlal v20.2d, v9.2s, v11.2s + mul w25, w2, w30 + usra v27.2d, v26.2d, #26 + add x23, x23, x23 + and v26.16b, v26.16b, v30.16b + add x21, x21, x21 + umlal v21.2d, v5.2s, v16.2s + umaddl x23, w26, w12, x23 + umlal v21.2d, v7.2s, v14.2s + umaddl x3, w26, w13, x3 + umlal v21.2d, v9.2s, v12.2s + umaddl x21, w26, w14, x21 + usra v28.2d, v27.2d, #25 + umaddl x23, w28, w14, x23 + and v27.16b, v27.16b, v15.16b + umaddl x3, w28, w15, x3 + shl v20.2d, v20.2d, #1 + umaddl x21, w28, w16, x21 + usra v29.2d, v28.2d, #26 + umaddl x23, w9, w16, x23 + and v28.16b, v28.16b, v30.16b + umaddl x3, w9, w17, x3 + umlal v20.2d, v0.2s, v10.2s + umaddl x21, w9, w19, x21 + umlal v20.2d, v2.2s, v18.2s + umaddl x23, w25, w19, x23 + umlal v20.2d, v4.2s, v16.2s + umaddl x3, w25, w20, x3 + umlal v20.2d, v6.2s, v14.2s + umaddl x21, w2, w10, x21 + umlal v20.2d, v8.2s, v12.2s + umaddl x23, w0, w10, x23 + bic v19.16b, v29.16b, v15.16b + umaddl x3, w0, w11, x3 + and v29.16b, v29.16b, v15.16b + umaddl x21, w0, w12, x21 + usra v20.2d, v19.2d, #25 + add x3, x3, x23, lsr #26 + trn1 v0.4s, v0.4s, v1.4s + and x23, x23, #0x3ffffff + usra v20.2d, v19.2d, #24 + add x21, x21, x3, lsr #25 + trn1 v1.4s, v2.4s, v3.4s + bfi x23, x3, #32, #25 + usra v20.2d, v19.2d, #21 + add x8, x8, x21, lsr #26 + trn1 v2.4s, v4.4s, v5.4s + and x21, x21, #0x3ffffff + trn1 v3.4s, v6.4s, v7.4s + add x7, x7, x8, lsr #25 + usra v21.2d, v20.2d, #26 + bfi x21, x8, #32, #25 + and v20.16b, v20.16b, v30.16b + ldr x2, [sp, #96] + trn1 v4.4s, v8.4s, v9.4s + lsr x3, x2, #32 + usra v22.2d, v21.2d, #25 + add x4, sp, #104 + and v21.16b, v21.16b, v15.16b + subs w0, w2, #1 + trn1 v19.4s, v28.4s, v29.4s + asr w1, w0, #5 + usra v23.2d, v22.2d, #26 + add x6, x6, x7, lsr #26 + and v22.16b, v22.16b, v30.16b + ldr w1, [x4, w1, sxtw #2] + trn1 v11.4s, v20.4s, v21.4s + and w4, w0, #0x1f +trn1 v13.4s, v22.4s, v23.4s + and x7, x7, #0x3ffffff +trn1 v15.4s, v24.4s, v25.4s + lsr w1, w1, w4 +trn1 v17.4s, v26.4s, v27.4s + bfi x7, x6, #32, #26 +mov v10.d[0], v0.d[1] + stp w0, w1, [sp, #96] + + + eor w1, w1, w3 + + + // Make X4 and Z5 more compact + mov v12.d[0], v1.d[1] + mov v14.d[0], v2.d[1] + mov v16.d[0], v3.d[1] + mov v18.d[0], v4.d[1] + + // Z4 -> Z2 + mov v1.d[0], x23 + mov v3.d[0], x21 + mov v5.d[0], x7 + mov v7.d[0], x24 + mov v9.d[0], x22 + + + // X4 -> X2 + ldr d28, [sp, #8] + mov v0.d[0], v11.d[1] + ldr d29, [sp] + mov v2.d[0], v13.d[1] + mov v4.d[0], v15.d[1] + mov v6.d[0], v17.d[1] + mov v8.d[0], v19.d[1] + + // X4 -> X2 in v0, v2, ..., v8 + // Z4 -> Z2 in v1, v3, ..., v9 + // X5 -> X3 in v10, v12, ..., v18 + // Z5 -> Z3 in v11, v13, ..., v19 + + bpl .Lmainloop + + tst w3, #1 + + fcsel d1, d1, d11, eq + fcsel d3, d3, d13, eq + fcsel d5, d5, d15, eq + fcsel d7, d7, d17, eq + fcsel d9, d9, d19, eq + + fcsel d0, d0, d10, eq + fcsel d2, d2, d12, eq + fcsel d4, d4, d14, eq + fcsel d6, d6, d16, eq + fcsel d8, d8, d18, eq + + mov w0, v1.s[0] + mov w1, v1.s[1] + mov w2, v3.s[0] + mov w3, v3.s[1] + mov w4, v5.s[0] + mov w5, v5.s[1] + mov w6, v7.s[0] + mov w7, v7.s[1] + mov w8, v9.s[0] + mov w9, v9.s[1] + + stp w0, w1, [sp, #80] + stp w2, w3, [sp, #88] + stp w4, w5, [sp, #96] + stp w6, w7, [sp, #104] + stp w8, w9, [sp, #112] + + mov x10, v0.d[0] + mov x11, v2.d[0] + mov x12, v4.d[0] + mov x13, v6.d[0] + mov x14, v8.d[0] + + stp x10, x11, [sp] + stp x12, x13, [sp, #16] + str x14, [sp, #32] + + adr x10, invtable + str x10, [sp, #160] + +.Linvloopnext: + ldrh w11, [x10], #2 + mov v20.s[0], w11 + str x10, [sp, #160] + + and w12, w11, #0x7f + subs w30, w12, #1 // square times + bmi .Lskipsquare + + mov w23, w3 + mov w24, w4 + mov w25, w5 + mov w26, w6 + mov w27, w7 + mov w14, w8 + add w10, w0, w0 + add w11, w1, w1 + add w12, w2, w2 + +.Lsqrloop1: + umull x20, w0, w0 + add x4, x24, x23, lsr #25 + umull x21, w10, w1 + and x3, x23, #0x1ffffff + umull x22, w10, w2 + add w13, w3, w3 + umull x23, w10, w3 + add x5, x25, x4, lsr #26 + umull x24, w11, w13 + and x4, x4, #0x3ffffff + umull x28, w4, w4 + add x6, x26, x5, lsr #25 + umull x25, w12, w3 + and x5, x5, #0x1ffffff + umull x26, w13, w3 + add w15, w5, w5 + umaddl x28, w13, w15, x28 + add x7, x27, x6, lsr #26 + umull x19, w4, w15 + and x6, x6, #0x3ffffff + umull x27, w11, w6 + add x8, x14, x7, lsr #25 + umaddl x28, w12, w6, x28 + and x7, x7, #0x1ffffff + umaddl x19, w13, w6, x19 + add x9, x9, x8, lsr #26 + umaddl x27, w10, w7, x27 + add w17, w7, w7 + umaddl x28, w11, w17, x28 + and x8, x8, #0x3ffffff + umaddl x19, w10, w9, x19 + add w14, w9, w9 + umaddl x27, w12, w5, x27 + add w16, w14, w14, lsl #1 + umaddl x28, w10, w8, x28 + add w3, w15, w15, lsl #1 + umaddl x19, w12, w7, x19 + add w16, w16, w14, lsl #4 + umaddl x27, w13, w4, x27 + add w3, w3, w15, lsl #4 + umaddl x28, w16, w9, x28 + + umaddl x19, w11, w8, x19 + add w9, w6, w6, lsl #1 + umaddl x20, w3, w5, x20 + + umaddl x24, w10, w4, x24 + add w9, w9, w6, lsl #4 + umaddl x25, w10, w5, x25 + add x19, x19, x28, lsr #26 + umaddl x26, w10, w6, x26 + and x14, x28, #0x3ffffff + umaddl x22, w11, w1, x22 + add x20, x20, x19, lsr #25 + umaddl x23, w11, w2, x23 + bic x1, x19, #0x1ffffff + umaddl x26, w12, w4, x26 + add x20, x20, x1, lsr #24 + umaddl x24, w2, w2, x24 + add w0, w4, w4 + umaddl x25, w11, w4, x25 + add x20, x20, x1, lsr #21 + umaddl x26, w11, w15, x26 + add w1, w17, w17, lsl #1 + umaddl x20, w9, w0, x20 + + umaddl x21, w9, w15, x21 + add w1, w1, w17, lsl #4 + umaddl x22, w9, w6, x22 + add w10, w8, w8, lsl #1 + umaddl x20, w1, w13, x20 + and x9, x19, #0x1ffffff + umaddl x21, w1, w4, x21 + add w10, w10, w8, lsl #4 + umaddl x22, w1, w15, x22 + subs w30, w30, #1 + umaddl x20, w10, w12, x20 + + umaddl x21, w10, w13, x21 + + umaddl x22, w10, w0, x22 + + umaddl x20, w16, w11, x20 + + umaddl x21, w16, w2, x21 + + umaddl x22, w16, w13, x22 + add w11, w6, w6 + umaddl x23, w1, w6, x23 + + umaddl x24, w1, w7, x24 + add x21, x21, x20, lsr #26 + umaddl x26, w10, w8, x26 + and x0, x20, #0x3ffffff + umaddl x23, w10, w15, x23 + add x22, x22, x21, lsr #25 + umaddl x24, w10, w11, x24 + and x1, x21, #0x1ffffff + umaddl x25, w10, w17, x25 + and x2, x22, #0x3ffffff + umaddl x23, w16, w4, x23 + add w10, w0, w0 + umaddl x24, w16, w15, x24 + add w11, w1, w1 + umaddl x25, w16, w6, x25 + add w12, w2, w2 + umaddl x26, w16, w17, x26 + add x23, x23, x22, lsr #26 + umaddl x27, w16, w8, x27 + bpl .Lsqrloop1 + + mov w11, v20.s[0] + add x4, x24, x23, lsr #25 + and x3, x23, #0x1ffffff + add x5, x25, x4, lsr #26 + and x4, x4, #0x3ffffff + add x6, x26, x5, lsr #25 + and x5, x5, #0x1ffffff + add x7, x27, x6, lsr #26 + and x6, x6, #0x3ffffff + add x8, x14, x7, lsr #25 + and x7, x7, #0x1ffffff + add x9, x9, x8, lsr #26 + and x8, x8, #0x3ffffff +.Lskipsquare: + mov w12, #40 + tst w11, #1<<8 + ubfx w13, w11, #9, #2 + bne .Lskipmul + mul w20, w13, w12 + add x20, sp, x20 + + ldp w10, w11, [x20] + ldp w12, w13, [x20, #8] + ldp w14, w15, [x20, #16] + ldp w16, w17, [x20, #24] + ldp w19, w20, [x20, #32] + mov w30, #19 + + umull x21, w1, w19 + umull x22, w1, w17 + umull x23, w1, w16 + umull x24, w1, w15 + umaddl x21, w3, w16, x21 + umaddl x22, w3, w15, x22 + umaddl x23, w3, w14, x23 + umaddl x24, w3, w13, x24 + umaddl x21, w5, w14, x21 + umaddl x22, w5, w13, x22 + umaddl x23, w5, w12, x23 + umaddl x24, w5, w11, x24 + umaddl x21, w7, w12, x21 + umaddl x22, w7, w11, x22 + umaddl x23, w7, w10, x23 + mul w27, w7, w30 + mul w25, w9, w30 + mul w26, w8, w30 + mul w28, w6, w30 + umaddl x24, w27, w20, x24 + umaddl x21, w9, w10, x21 + umaddl x22, w25, w20, x22 + umaddl x23, w25, w19, x23 + umaddl x24, w25, w17, x24 + add x22, x22, x22 + umaddl x21, w0, w20, x21 + add x24, x24, x24 + umaddl x22, w0, w19, x22 + umaddl x23, w0, w17, x23 + umaddl x24, w0, w16, x24 + umaddl x21, w2, w17, x21 + umaddl x22, w2, w16, x22 + umaddl x23, w2, w15, x23 + umaddl x24, w2, w14, x24 + umaddl x21, w4, w15, x21 + umaddl x22, w4, w14, x22 + umaddl x23, w4, w13, x23 + umaddl x24, w4, w12, x24 + umaddl x21, w6, w13, x21 + umaddl x22, w6, w12, x22 + umaddl x23, w6, w11, x23 + umaddl x24, w6, w10, x24 + umaddl x21, w8, w11, x21 + umaddl x22, w8, w10, x22 + umaddl x23, w26, w20, x23 + umaddl x24, w26, w19, x24 + umull x6, w25, w16 + umull x7, w25, w15 + umull x8, w25, w14 + umaddl x6, w5, w10, x6 + mul w5, w5, w30 + umaddl x7, w27, w17, x7 + umaddl x8, w27, w16, x8 + umaddl x6, w27, w19, x6 + umaddl x7, w5, w20, x7 + umaddl x8, w5, w19, x8 + umaddl x6, w3, w12, x6 + umaddl x7, w3, w11, x7 + umaddl x8, w3, w10, x8 + umaddl x6, w1, w14, x6 + umaddl x7, w1, w13, x7 + umaddl x8, w1, w12, x8 + mul w9, w4, w30 + add x7, x7, x7 + umaddl x6, w26, w17, x6 + umaddl x7, w26, w16, x7 + umaddl x8, w26, w15, x8 + umaddl x6, w28, w20, x6 + umaddl x7, w28, w19, x7 + umaddl x8, w28, w17, x8 + umaddl x6, w4, w11, x6 + umaddl x7, w4, w10, x7 + umaddl x8, w9, w20, x8 + umaddl x6, w2, w13, x6 + umaddl x7, w2, w12, x7 + umaddl x8, w2, w11, x8 + umaddl x6, w0, w15, x6 + umaddl x7, w0, w14, x7 + umaddl x8, w0, w13, x8 + mul w4, w3, w30 + add x6, x6, x7, lsr #26 + and x7, x7, #0x3ffffff + add x24, x24, x6, lsr #25 + and x6, x6, #0x1ffffff + add x23, x23, x24, lsr #26 + and x24, x24, #0x3ffffff + add x22, x22, x23, lsr #25 + bfi x24, x23, #32, #25 + add x21, x21, x22, lsr #26 + and x22, x22, #0x3ffffff + bic x3, x21, #0x3ffffff + lsr x23, x3, #26 + bfi x22, x21, #32, #26 + add x23, x23, x3, lsr #25 + umull x21, w25, w13 + add x23, x23, x3, lsr #22 + umull x3, w25, w12 + umaddl x23, w25, w11, x23 + umaddl x21, w27, w15, x21 + umaddl x3, w27, w14, x3 + umaddl x23, w27, w13, x23 + mul w27, w1, w30 + umaddl x3, w5, w16, x3 + umaddl x23, w5, w15, x23 + umaddl x21, w5, w17, x21 + umaddl x3, w4, w19, x3 + umaddl x23, w4, w17, x23 + umaddl x21, w4, w20, x21 + umaddl x3, w1, w10, x3 + umaddl x23, w27, w20, x23 + umaddl x21, w1, w11, x21 + mul w25, w2, w30 + add x23, x23, x23 + add x21, x21, x21 + umaddl x23, w26, w12, x23 + umaddl x3, w26, w13, x3 + umaddl x21, w26, w14, x21 + umaddl x23, w28, w14, x23 + umaddl x3, w28, w15, x3 + umaddl x21, w28, w16, x21 + umaddl x23, w9, w16, x23 + umaddl x3, w9, w17, x3 + umaddl x21, w9, w19, x21 + umaddl x23, w25, w19, x23 + umaddl x3, w25, w20, x3 + umaddl x21, w2, w10, x21 + umaddl x23, w0, w10, x23 + umaddl x3, w0, w11, x3 + umaddl x21, w0, w12, x21 + add x1, x3, x23, lsr #26 + and x0, x23, #0x3ffffff + add x2, x21, x1, lsr #25 + and x1, x1, #0x1ffffff + add x3, x8, x2, lsr #26 + and x2, x2, #0x3ffffff + add x4, x7, x3, lsr #25 + and x3, x3, #0x1ffffff + add x5, x6, x4, lsr #26 + and x4, x4, #0x3ffffff + and x5, x5, #0x3ffffff + + mov w11, v20.s[0] + mov w6, w24 + lsr x7, x24, #32 + mov w8, w22 + lsr x9, x22, #32 +.Lskipmul: + ubfx w12, w11, #11, #2 + cbz w12, .Lskipstore + mov w13, #40 + mul w12, w12, w13 + add x12, sp, x12 + + stp w0, w1, [x12] + stp w2, w3, [x12, #8] + stp w4, w5, [x12, #16] + stp w6, w7, [x12, #24] + stp w8, w9, [x12, #32] +.Lskipstore: + + ldr x10, [sp, #160] + adr x11, invtable+13*2 + cmp x10, x11 + bne .Linvloopnext + + // Final reduce + // w5 and w9 are 26 bits instead of 25 + + orr x10, x0, x1, lsl #26 + orr x10, x10, x2, lsl #51 + + lsr x11, x2, #13 + orr x11, x11, x3, lsl #13 + orr x11, x11, x4, lsl #38 + + add x12, x5, x6, lsl #25 + adds x12, x12, x7, lsl #51 + + lsr x13, x7, #13 + orr x13, x13, x8, lsl #12 + orr x13, x13, x9, lsl #38 + + adcs x13, x13, xzr + adc x14, xzr, xzr + + extr x17, x14, x13, #63 + mov w19, #19 + mul w15, w17, w19 + add w15, w15, #19 + + adds x15, x10, x15 + adcs x15, x11, xzr + adcs x15, x12, xzr + adcs x15, x13, xzr + adc x16, x14, xzr + + extr x16, x16, x15, #63 + mul w16, w16, w19 + + adds x10, x10, x16 + adcs x11, x11, xzr + adcs x12, x12, xzr + adc x13, x13, xzr + and x13, x13, 0x7fffffffffffffff + + ldr x17, [sp, #176] + stp x10, x11, [x17] // todo: fix for unaligned store + stp x12, x13, [x17, #16] + + add sp, sp, 192 + + ldp x19, x20, [sp, #16] + ldp x21, x22, [sp, #32] + ldp x23, x24, [sp, #48] + ldp x25, x26, [sp, #64] + ldp x27, x28, [sp, #80] + ldp d8, d9, [sp, #96] + ldp d10, d11, [sp, #112] + ldp d12, d13, [sp, #128] + ldp d14, d15, [sp, #144] + ldp x29, x30, [sp], #160 + + ret + //.size mx25519_scalarmult_arm64, .-mx25519_scalarmult_arm64 + + //.type invtable, %object +invtable: + // square times, + // skip mul, + // mulsource, + // dest + .hword 1|(1<<8) |(1<<11) + .hword 2| (2<<9)|(2<<11) + .hword 0| (1<<9)|(1<<11) + .hword 1| (2<<9)|(2<<11) + .hword 5| (2<<9)|(2<<11) + .hword 10| (2<<9)|(3<<11) + .hword 20| (3<<9) + .hword 10| (2<<9)|(2<<11) + .hword 50| (2<<9)|(3<<11) + .hword 100| (3<<9) + .hword 50| (2<<9) + .hword 5| (1<<9) + .hword 0| (0<<9) + //.size invtable, .-invtable diff --git a/external/src/mx25519/arm64/scalarmult.h b/external/src/mx25519/arm64/scalarmult.h new file mode 100644 index 0000000..ce16c15 --- /dev/null +++ b/external/src/mx25519/arm64/scalarmult.h @@ -0,0 +1,16 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#ifndef ARM64_SCALARMULT_H +#define ARM64_SCALARMULT_H + +#include + +void mx25519_scalarmult_arm64(uint8_t* q, + const uint8_t* n, + const uint8_t* p); + +#endif diff --git a/external/src/mx25519/cpu.c b/external/src/mx25519/cpu.c new file mode 100644 index 0000000..bcf5303 --- /dev/null +++ b/external/src/mx25519/cpu.c @@ -0,0 +1,60 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#include "cpu.h" +#include "platform.h" + +#if defined(PLATFORM_X86) || defined(PLATFORM_AMD64) +#define HAVE_CPUID +#ifdef _MSC_VER +#include +#define cpuid(info, x) __cpuidex(info, x, 0) +#else +#include +static void cpuid(uint32_t info[4], uint32_t type) { + __cpuid_count(type, 0, info[0], info[1], info[2], info[3]); +} +#endif +#endif + +x25519_cpu_cap mx25519_get_cpu_cap() { + static x25519_cpu_cap cap = -1; + if (cap == -1) { + cap = 0; +#ifdef HAVE_CPUID + uint32_t info[4]; + cpuid(info, 0); + uint32_t num_ids = info[0]; + if (num_ids >= 0x00000001) { + cpuid(info, 0x00000001); + if (info[2] & (1 << 28)) { + cap |= X25519_CPU_CAP_AVX; + } + } + if (num_ids >= 0x00000007) { + cpuid(info, 0x00000007); + if (info[1] & (1 << 5)) { + cap |= X25519_CPU_CAP_AVX2; + } + if (info[1] & (1 << 8)) { + cap |= X25519_CPU_CAP_MULX; + } + if (info[1] & (1 << 19)) { + cap |= X25519_CPU_CAP_ADX; + } + } + cpuid(info, 0x80000000); + uint32_t num_ext_ids = info[0]; + if (num_ext_ids >= 0x80000001) { + cpuid(info, 0x80000001); + if (info[3] & (1 << 27)) { + cap |= X25519_CPU_CAP_RDTSCP; + } + } +#endif + } + return cap; +} diff --git a/external/src/mx25519/cpu.h b/external/src/mx25519/cpu.h new file mode 100644 index 0000000..fb77533 --- /dev/null +++ b/external/src/mx25519/cpu.h @@ -0,0 +1,20 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#ifndef CPU_H +#define CPU_H + +typedef enum x25519_cpu_cap { + X25519_CPU_CAP_RDTSCP = 1, + X25519_CPU_CAP_AVX = 2, + X25519_CPU_CAP_AVX2 = 4, + X25519_CPU_CAP_MULX = 8, + X25519_CPU_CAP_ADX = 16, +} x25519_cpu_cap; + +x25519_cpu_cap mx25519_get_cpu_cap(void); + +#endif diff --git a/external/src/mx25519/digit.h b/external/src/mx25519/digit.h new file mode 100644 index 0000000..c870b47 --- /dev/null +++ b/external/src/mx25519/digit.h @@ -0,0 +1,17 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#ifndef DIGIT_H +#define DIGIT_H + +#include + +typedef uint64_t digit; + +#define DIGIT_SIZE 8 +#define DIGIT_RADIX 64 + +#endif diff --git a/external/src/mx25519/digit_ops.h b/external/src/mx25519/digit_ops.h new file mode 100644 index 0000000..83d6438 --- /dev/null +++ b/external/src/mx25519/digit_ops.h @@ -0,0 +1,188 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#ifndef DIGIT_OPS_H +#define DIGIT_OPS_H + +#include "digit.h" +#include "platform.h" + +#include + +typedef uint8_t carry; + +#define digit_load platform_load64 +#define digit_store platform_store64 + +/* Returns x == 0 in constant time. */ +static FORCE_INLINE bool digit_eq_zero(digit x) { + return 1 ^ ((x | -x) >> (DIGIT_RADIX - 1)); +} + +/* Returns x < y in constant time. */ +static FORCE_INLINE bool digit_lt(digit x, digit y) { + return (x ^ ((x ^ y) | ((x - y) ^ y))) >> (DIGIT_RADIX - 1); +} + +/* Digit multiplication. Returns the low half of the result. The + high half is stored in *hi */ +static digit digit_mul( + digit a, digit b, digit* hi); + +/* Digit addition with carry. The result is stored in *sum_out. Returns + the carry. */ +static carry digit_addc( + carry carry_in, + digit addend1, digit addend2, + digit* sum_out); + +/* Digit subtraction with borrow. The result is stored in *diff_out. Returns + the carry. */ +static carry digit_subb( + carry borrow_in, + digit minuend, digit subtrahend, + digit* diff_out); + +/* Shifts a 2-digit quantity to the right by a number of bits specified + by shift and returns the low digit of the result */ +static digit digit_shr( + digit high_in, digit low_in, uint8_t shift); + +/* Shifts a 2-digit quantity to the left by a number of bits specified + by shift and returns the high digit of the result. */ +static digit digit_shl( + digit high_in, digit low_in, uint8_t shift); + +#if defined(PLATFORM_X64_INTRIN) + +#include +#include + +static FORCE_INLINE digit digit_mul(digit a, digit b, digit* hi) { + return _umul128(a, b, hi); +} + +static FORCE_INLINE carry digit_addc( + carry carry_in, digit addend1, digit addend2, digit* sum_out) { + return _addcarry_u64(carry_in, addend1, addend2, sum_out); +} + +static FORCE_INLINE carry digit_subb( + carry borrow_in, digit minuend, digit subtrahend, digit* diff_out) { + return _subborrow_u64(borrow_in, minuend, subtrahend, diff_out); +} + +static FORCE_INLINE digit digit_shr( + digit high_in, digit low_in, uint8_t shift) { + return __shiftright128(low_in, high_in, shift); +} + +static FORCE_INLINE digit digit_shl( + digit high_in, digit low_in, uint8_t shift) { + return __shiftleft128(low_in, high_in, shift); +} + +#elif defined(PLATFORM_UINT128) + +static FORCE_INLINE digit digit_mul(digit a, digit b, digit* hi) { + uint128_t res = (uint128_t)a * (uint128_t)b; + *hi = res >> DIGIT_RADIX; + return (digit)res; +} + +static FORCE_INLINE carry digit_addc( + carry carry_in, digit addend1, digit addend2, digit* sum_out) { + uint128_t temp = (uint128_t)addend1 + (uint128_t)addend2 + carry_in; + *sum_out = (digit)temp; + return (carry)(temp >> DIGIT_RADIX); +} + +static FORCE_INLINE carry digit_subb( + carry borrow_in, digit minuend, digit subtrahend, digit* diff_out) { + uint128_t temp = (uint128_t)minuend - (uint128_t)subtrahend - borrow_in; + *diff_out = (digit)temp; + return (carry)(temp >> (sizeof(uint128_t) * 8 - 1)); +} + +static FORCE_INLINE digit digit_shr( + digit high_in, digit low_in, uint8_t shift) { + return (low_in >> shift) | (high_in << (DIGIT_RADIX - shift)); +} + +static FORCE_INLINE digit digit_shl( + digit high_in, digit low_in, uint8_t shift) { + return (high_in << shift) | (low_in >> (DIGIT_RADIX - shift)); +} + +#else + +#define DIGIT_HBIT (DIGIT_RADIX/2) +#define MASK_LOW ((digit)(-1) >> DIGIT_HBIT) +#define MASK_HIGH ((digit)(-1) << DIGIT_HBIT) + +static FORCE_INLINE digit digit_mul(digit a, digit b, digit* hi) { + digit al, ah, bl, bh; + digit albl, albh, ahbl, ahbh; + digit tmp1, tmp2, tmp3, tmp4, carry; + + al = a & MASK_LOW; + ah = a >> DIGIT_HBIT; + bl = b & MASK_LOW; + bh = b >> DIGIT_HBIT; + + albl = al * bl; + albh = al * bh; + ahbl = ah * bl; + ahbh = ah * bh; + digit lo = albl & MASK_LOW; + + tmp1 = albl >> DIGIT_HBIT; + tmp2 = ahbl & MASK_LOW; + tmp3 = albh & MASK_LOW; + tmp4 = tmp1 + tmp2 + tmp3; + carry = tmp4 >> DIGIT_HBIT; + lo |= tmp4 << DIGIT_HBIT; + + tmp1 = ahbl >> DIGIT_HBIT; + tmp2 = albh >> DIGIT_HBIT; + tmp3 = ahbh & MASK_LOW; + tmp4 = tmp1 + tmp2 + tmp3 + carry; + *hi = tmp4 & MASK_LOW; + carry = tmp4 & MASK_HIGH; + *hi |= (ahbh & MASK_HIGH) + carry; + + return lo; +} + +static FORCE_INLINE carry digit_addc( + carry carry_in, digit addend1, digit addend2, digit* sum_out) { + digit temp = addend1 + (digit)carry_in; + *sum_out = addend2 + temp; + return digit_lt(temp, (digit)carry_in) + | digit_lt(*sum_out, temp); +} + +static FORCE_INLINE carry digit_subb( + carry borrow_in, digit minuend, digit subtrahend, digit* diff_out) { + digit temp = minuend - subtrahend; + *diff_out = temp - (digit)borrow_in; + return digit_lt(minuend, subtrahend) + | (borrow_in & digit_eq_zero(temp)); +} + +static FORCE_INLINE digit digit_shr( + digit high_in, digit low_in, uint8_t shift) { + return (low_in >> shift) | (high_in << (DIGIT_RADIX - shift)); +} + +static FORCE_INLINE digit digit_shl( + digit high_in, digit low_in, uint8_t shift) { + return (high_in << shift) | (low_in >> (DIGIT_RADIX - shift)); +} + +#endif + +#endif diff --git a/external/src/mx25519/impl.c b/external/src/mx25519/impl.c new file mode 100644 index 0000000..7f9832d --- /dev/null +++ b/external/src/mx25519/impl.c @@ -0,0 +1,56 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#include + +#include "impl.h" +#include "platform.h" +#include "portable/scalarmult.h" +#ifdef PLATFORM_ARM64 +#include "arm64/scalarmult.h" +#endif +#ifdef PLATFORM_AMD64 +#include "amd64/scalarmult.h" +#endif + +static const mx25519_impl impl_portable = { + .scmul = &mx25519_scalarmult_portable, + .type = MX25519_TYPE_PORTABLE +}; + +static const mx25519_impl impl_arm64 = { +#if defined(PLATFORM_ARM64) + .scmul = &mx25519_scalarmult_arm64, +#else + .scmul = NULL, +#endif + .type = MX25519_TYPE_ARM64 +}; + +static const mx25519_impl impl_amd64 = { +#ifdef PLATFORM_AMD64 + .scmul = &mx25519_scalarmult_amd64, +#else + .scmul = NULL, +#endif + .type = MX25519_TYPE_AMD64 +}; + +static const mx25519_impl impl_amd64x = { +#ifdef PLATFORM_AMD64 + .scmul = &mx25519_scalarmult_amd64x, +#else + .scmul = NULL, +#endif + .type = MX25519_TYPE_AMD64X +}; + +const mx25519_impl* mx25519_impls[4] = { + &impl_portable, + &impl_arm64, + &impl_amd64, + &impl_amd64x, +}; diff --git a/external/src/mx25519/impl.h b/external/src/mx25519/impl.h new file mode 100644 index 0000000..1fff143 --- /dev/null +++ b/external/src/mx25519/impl.h @@ -0,0 +1,23 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#ifndef IMPL_H +#define IMPL_H + +#include + +#include + +typedef void scmul_func(uint8_t result[32], const uint8_t key[32], const uint8_t base[32]); + +typedef struct mx25519_impl { + scmul_func* scmul; + mx25519_type type; +} mx25519_impl; + +extern const mx25519_impl* mx25519_impls[4]; + +#endif diff --git a/external/src/mx25519/mp_ops.h b/external/src/mx25519/mp_ops.h new file mode 100644 index 0000000..24f2633 --- /dev/null +++ b/external/src/mx25519/mp_ops.h @@ -0,0 +1,322 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#ifndef MP_OPS_H +#define MP_OPS_H + +#include "digit_ops.h" + +#include + +/* 256x256->512 multiplication */ +static void mp_mul256( + digit c[8], const digit a[4], const digit b[4]); + +/* 256x256->512 squaring */ +static void mp_sqr256( + digit c[8], const digit a[4]); + +/* 256x256->256 multiplication */ +static void mp_mul256_mod256( + digit c[4], const digit a[4], const digit b[4]); + +/* 512-bit addition, returns the carry */ +static carry mp_add512( + digit c[8], const digit a[8], const digit b[8]); + +/* 256-bit addition, returns the carry */ +static carry mp_add256( + digit c[4], const digit a[4], const digit b[4]); + +/* 256-bit subtraction c = a - b, returns the borrow */ +static carry mp_sub256( + digit c[4], const digit a[4], const digit b[4]); + +/* 256-bit left-shift */ +static void mp_shl(digit a[4], uint8_t count); + +/* ======================================================================= */ + +static void mp_mul256( + digit c[8], const digit a[4], const digit b[4]) +{ + /* + b3 b2 b1 b0 + a3 a2 a1 a0 + ---------------------------------------- + t001 t000 + t011 t010 + t021 t020 + t031 t030 + t101 t100 + t111 t110 + t121 t120 + t131 t130 + t201 t200 + t211 t210 + t221 t220 + t231 t230 + t301 t300 + t311 t310 + t321 t320 + t331 t330 + ---------------------------------------- + c7 c6 c5 c4 c3 c2 c1 c0 + */ + + digit cr = 0; + + //c0 = t000 + digit t000, t001; + t000 = digit_mul(a[0], b[0], &t001); + c[0] = t000; + + //c1 = t001 + t010 + t100 + c[1] = t001; + digit t010, t011; + t010 = digit_mul(a[0], b[1], &t011); + digit t100, t101; + t100 = digit_mul(a[1], b[0], &t101); + cr += digit_addc(0, c[1], t010, &c[1]); + cr += digit_addc(0, c[1], t100, &c[1]); + + //c2 = t011 + t020 + t101 + t110 + t200 + cr = digit_addc(0, cr, t011, &c[2]); + digit t020, t021; + t020 = digit_mul(a[0], b[2], &t021); + digit t110, t111; + t110 = digit_mul(a[1], b[1], &t111); + digit t201, t200; + t200 = digit_mul(a[2], b[0], &t201); + cr += digit_addc(0, c[2], t101, &c[2]); + cr += digit_addc(0, c[2], t020, &c[2]); + cr += digit_addc(0, c[2], t110, &c[2]); + cr += digit_addc(0, c[2], t200, &c[2]); + + //c3 = t021 + t030 + t111 + t120 + t201 + t210 + t300 + cr = digit_addc(0, cr, t021, &c[3]); + digit t030, t031; + t030 = digit_mul(a[0], b[3], &t031); + digit t120, t121; + t120 = digit_mul(a[1], b[2], &t121); + digit t210, t211; + t210 = digit_mul(a[2], b[1], &t211); + digit t300, t301; + t300 = digit_mul(a[3], b[0], &t301); + cr += digit_addc(0, c[3], t111, &c[3]); + cr += digit_addc(0, c[3], t030, &c[3]); + cr += digit_addc(0, c[3], t201, &c[3]); + cr += digit_addc(0, c[3], t120, &c[3]); + cr += digit_addc(0, c[3], t210, &c[3]); + cr += digit_addc(0, c[3], t300, &c[3]); + + //c4 = t031 + t121 + t130 + t211 + t220 + t301 + t310 + cr = digit_addc(0, cr, t031, &c[4]); + digit t130, t131; + t130 = digit_mul(a[1], b[3], &t131); + digit t220, t221; + t220 = digit_mul(a[2], b[2], &t221); + digit t310, t311; + t310 = digit_mul(a[3], b[1], &t311); + cr += digit_addc(0, c[4], t121, &c[4]); + cr += digit_addc(0, c[4], t211, &c[4]); + cr += digit_addc(0, c[4], t301, &c[4]); + cr += digit_addc(0, c[4], t130, &c[4]); + cr += digit_addc(0, c[4], t220, &c[4]); + cr += digit_addc(0, c[4], t310, &c[4]); + + //c5 = t131 + t221 + t230 + t311 + t320 + cr = digit_addc(0, cr, t131, &c[5]); + digit t230, t231; + t230 = digit_mul(a[2], b[3], &t231); + digit t320, t321; + t320 = digit_mul(a[3], b[2], &t321); + cr += digit_addc(0, c[5], t221, &c[5]); + cr += digit_addc(0, c[5], t311, &c[5]); + cr += digit_addc(0, c[5], t230, &c[5]); + cr += digit_addc(0, c[5], t320, &c[5]); + + //c6 = t231 + t321 + t330 + cr = digit_addc(0, cr, t231, &c[6]); + digit t330, t331; + t330 = digit_mul(a[3], b[3], &t331); + cr += digit_addc(0, c[6], t321, &c[6]); + cr += digit_addc(0, c[6], t330, &c[6]); + + //c7 = t331 + c[7] = t331 + cr; +} + +static void mp_sqr256( + digit c[8], const digit a[4]) +{ + digit cr = 0; + + //c0 = t000 + digit t000, t001; + t000 = digit_mul(a[0], a[0], &t001); + c[0] = t000; + + //c1 = t001 + t010 + t100 + c[1] = t001; + digit t010, t011; + t010 = digit_mul(a[0], a[1], &t011); + cr += digit_addc(0, c[1], t010, &c[1]); + cr += digit_addc(0, c[1], t010, &c[1]); + + //c2 = t011 + t020 + t101 + t110 + t200 + cr = digit_addc(0, cr, t011, &c[2]); + digit t020, t021; + t020 = digit_mul(a[0], a[2], &t021); + digit t110, t111; + t110 = digit_mul(a[1], a[1], &t111); + cr += digit_addc(0, c[2], t011, &c[2]); + cr += digit_addc(0, c[2], t020, &c[2]); + cr += digit_addc(0, c[2], t110, &c[2]); + cr += digit_addc(0, c[2], t020, &c[2]); + + //c3 = t021 + t030 + t111 + t120 + t201 + t210 + t300 + cr = digit_addc(0, cr, t021, &c[3]); + digit t030, t031; + t030 = digit_mul(a[0], a[3], &t031); + digit t120, t121; + t120 = digit_mul(a[1], a[2], &t121); + cr += digit_addc(0, c[3], t111, &c[3]); + cr += digit_addc(0, c[3], t030, &c[3]); + cr += digit_addc(0, c[3], t021, &c[3]); + cr += digit_addc(0, c[3], t120, &c[3]); + cr += digit_addc(0, c[3], t120, &c[3]); + cr += digit_addc(0, c[3], t030, &c[3]); + + //c4 = t031 + t121 + t130 + t211 + t220 + t301 + t310 + cr = digit_addc(0, cr, t031, &c[4]); + digit t130, t131; + t130 = digit_mul(a[1], a[3], &t131); + digit t220, t221; + t220 = digit_mul(a[2], a[2], &t221); + cr += digit_addc(0, c[4], t121, &c[4]); + cr += digit_addc(0, c[4], t121, &c[4]); + cr += digit_addc(0, c[4], t031, &c[4]); + cr += digit_addc(0, c[4], t130, &c[4]); + cr += digit_addc(0, c[4], t220, &c[4]); + cr += digit_addc(0, c[4], t130, &c[4]); + + //c5 = t131 + t221 + t230 + t311 + t320 + cr = digit_addc(0, cr, t131, &c[5]); + digit t230, t231; + t230 = digit_mul(a[2], a[3], &t231); + cr += digit_addc(0, c[5], t221, &c[5]); + cr += digit_addc(0, c[5], t131, &c[5]); + cr += digit_addc(0, c[5], t230, &c[5]); + cr += digit_addc(0, c[5], t230, &c[5]); + + //c6 = t231 + t321 + t330 + cr = digit_addc(0, cr, t231, &c[6]); + digit t330, t331; + t330 = digit_mul(a[3], a[3], &t331); + cr += digit_addc(0, c[6], t231, &c[6]); + cr += digit_addc(0, c[6], t330, &c[6]); + + //c7 = t331 + c[7] = t331 + cr; +} + +static void mp_mul256_mod256( + digit c[4], const digit a[4], const digit b[4]) +{ + digit cr = 0; + + //c0 = t000 + digit t000, t001; + t000 = digit_mul(a[0], b[0], &t001); + c[0] = t000; + + //c1 = t001 + t010 + t100 + c[1] = t001; + digit t010, t011; + t010 = digit_mul(a[0], b[1], &t011); + digit t100, t101; + t100 = digit_mul(a[1], b[0], &t101); + cr += digit_addc(0, c[1], t010, &c[1]); + cr += digit_addc(0, c[1], t100, &c[1]); + + //c2 = t011 + t020 + t101 + t110 + t200 + cr = digit_addc(0, cr, t011, &c[2]); + digit t020, t021; + t020 = digit_mul(a[0], b[2], &t021); + digit t110, t111; + t110 = digit_mul(a[1], b[1], &t111); + digit t201, t200; + t200 = digit_mul(a[2], b[0], &t201); + cr += digit_addc(0, c[2], t101, &c[2]); + cr += digit_addc(0, c[2], t020, &c[2]); + cr += digit_addc(0, c[2], t110, &c[2]); + cr += digit_addc(0, c[2], t200, &c[2]); + + //c3 = t021 + t030 + t111 + t120 + t201 + t210 + t300 + c[3] = t021 + cr; + digit t030; + t030 = a[0] * b[3]; + digit t120; + t120 = a[1] * b[2]; + digit t210; + t210 = a[2] * b[1]; + digit t300; + t300 = a[3] * b[0]; + c[3] += t111; + c[3] += t030; + c[3] += t201; + c[3] += t120; + c[3] += t210; + c[3] += t300; +} + +static carry mp_add512( + digit c[8], const digit a[8], const digit b[8]) +{ + carry cr = 0; + cr = digit_addc(cr, a[0], b[0], &c[0]); + cr = digit_addc(cr, a[1], b[1], &c[1]); + cr = digit_addc(cr, a[2], b[2], &c[2]); + cr = digit_addc(cr, a[3], b[3], &c[3]); + cr = digit_addc(cr, a[4], b[4], &c[4]); + cr = digit_addc(cr, a[5], b[5], &c[5]); + cr = digit_addc(cr, a[6], b[6], &c[6]); + cr = digit_addc(cr, a[7], b[7], &c[7]); + return cr; +} + +static carry mp_add256( + digit c[4], const digit a[4], const digit b[4]) +{ + carry cr = 0; + cr = digit_addc(cr, a[0], b[0], &c[0]); + cr = digit_addc(cr, a[1], b[1], &c[1]); + cr = digit_addc(cr, a[2], b[2], &c[2]); + cr = digit_addc(cr, a[3], b[3], &c[3]); + return cr; +} + +static carry mp_sub256( + digit c[4], const digit a[4], const digit b[4]) +{ + carry br = 0; + br = digit_subb(br, a[0], b[0], &c[0]); + br = digit_subb(br, a[1], b[1], &c[1]); + br = digit_subb(br, a[2], b[2], &c[2]); + br = digit_subb(br, a[3], b[3], &c[3]); + return br; +} + +static void mp_shl(digit a[4], uint8_t count) +{ + a[3] = digit_shl(a[3], a[2], count); + a[2] = digit_shl(a[2], a[1], count); + a[1] = digit_shl(a[1], a[0], count); + a[0] = a[0] << count; +} + +#endif diff --git a/external/src/mx25519/mx25519.c b/external/src/mx25519/mx25519.c new file mode 100644 index 0000000..d1b282f --- /dev/null +++ b/external/src/mx25519/mx25519.c @@ -0,0 +1,137 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#include + +#include "impl.h" +#include "cpu.h" +#include "scalar.h" +#include "platform.h" + +#include +#include +#include + +static const mx25519_pubkey x25519_base = { + .data = { 9 } +}; + +static bool impl_supported(mx25519_type impl) { + if (impl == MX25519_TYPE_PORTABLE) { + return true; + } + if (impl == MX25519_TYPE_ARM64) { +#if defined(PLATFORM_ARM64) + return true; +#else + return false; +#endif + } + if (impl == MX25519_TYPE_AMD64) { +#if defined(PLATFORM_AMD64) + return true; +#else + return false; +#endif + } + if (impl == MX25519_TYPE_AMD64X) { +#if defined(PLATFORM_AMD64) + x25519_cpu_cap cap = mx25519_get_cpu_cap(); + return (cap & X25519_CPU_CAP_MULX) != 0 + && (cap & X25519_CPU_CAP_ADX) != 0; +#else + return false; +#endif + } + return false; +} + +static mx25519_type select_best_impl(void) { +#if defined(PLATFORM_AMD64) + if (impl_supported(MX25519_TYPE_AMD64X)) { + return MX25519_TYPE_AMD64X; + } + return MX25519_TYPE_AMD64; +#elif defined(PLATFORM_ARM64) + return MX25519_TYPE_ARM64; +#else + return MX25519_TYPE_PORTABLE; +#endif +} + +const mx25519_impl* mx25519_select_impl(mx25519_type type) +{ + if (type == MX25519_TYPE_AUTO) { + type = select_best_impl(); + } + else if (!impl_supported(type)) { + return NULL; + } + assert(type >= 0 && type < 4); + return mx25519_impls[type]; +} + +mx25519_type mx25519_impl_type(const mx25519_impl* impl) +{ + assert(impl != NULL); + return impl->type; +} + +void mx25519_scmul_base(const mx25519_impl* impl, mx25519_pubkey* result, + const mx25519_privkey* key) +{ + assert(impl != NULL); + assert(key != NULL); + assert(result != NULL); + impl->scmul(result->data, key->data, x25519_base.data); +} + +void mx25519_scmul_key(const mx25519_impl* impl, mx25519_pubkey* result, + const mx25519_privkey* key, const mx25519_pubkey* pt) +{ + assert(impl != NULL); + assert(pt != NULL); + assert(key != NULL); + assert(result != NULL); + impl->scmul(result->data, key->data, pt->data); +} + +int mx25519_invkey(mx25519_privkey* invkey, const mx25519_privkey keys[], + size_t num_keys) +{ + assert(invkey != NULL); + assert(keys != NULL || num_keys == 0); + + /* calculate 8*key[0]*key[1]*... in Montgomery form */ + x25519_scalar_mont prod_mont = mx25519_sc8_mont; + + for (size_t i = 0; i < num_keys; ++i) { + x25519_scalar key_sc; + x25519_scalar_mont key_mont; + mx25519_scalar_unpack(&key_sc, keys[i].data); + key_sc.v[0] &= 0xfffffffffffffff8; + key_sc.v[3] &= 0x7fffffffffffffff; + mx25519_scalar_to_mont(&key_mont, &key_sc); + mx25519_scalar_mul(&prod_mont, &prod_mont, &key_mont); + } + + /* invert in Montgomery form */ + mx25519_scalar_inv(&prod_mont, &prod_mont); + + /* convert back from Montgomery form */ + x25519_scalar res; + mx25519_scalar_from_mont(&res, &prod_mont); + + if (res.v[3] >= 0x1000000000000000) { + return 1; /* inverse is larger than or equal to 2^252 */ + } + + /* shift left by 3 bits */ + mx25519_scalar_lsh3(&res); + + mx25519_scalar_pack(invkey->data, &res); + return 0; +} diff --git a/external/src/mx25519/mx25519.h b/external/src/mx25519/mx25519.h new file mode 100644 index 0000000..05b9602 --- /dev/null +++ b/external/src/mx25519/mx25519.h @@ -0,0 +1,138 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#ifndef MX25519_H +#define MX25519_H + +#include +#include + +/* + * X25519 scalar (private key). + * All private keys are implicitly multiples of 8 as the library only uses + * bits 3-254. Bits 0-2 and 255 are internally set to 0. + * Note that the key clamping procedure of this library differs from RFC 7748 + * by not setting the value of bit 254 to 1. This is done to support inverted + * keys, which might have a zero bit in that position. + */ +typedef struct mx25519_privkey { + uint8_t data[32]; +} mx25519_privkey; + +/* + * X25519 X-coordinate (public key). + */ +typedef struct mx25519_pubkey { + uint8_t data[32]; +} mx25519_pubkey; + +/* + * Opaque struct holding a scalar multiplication implementation. + */ +typedef struct mx25519_impl mx25519_impl; + +/* + * Implementation types. + */ +typedef enum mx25519_type { + MX25519_TYPE_AUTO = -1, /* select automatically */ + MX25519_TYPE_PORTABLE, /* portable C implementation */ + MX25519_TYPE_ARM64, /* ARM64 assembly */ + MX25519_TYPE_AMD64, /* AMD64 assembly */ + MX25519_TYPE_AMD64X, /* AMD64 assembly with MULX+ADX */ +} mx25519_type; + +#if defined(_WIN32) || defined(__CYGWIN__) +#define MX25519_WIN +#endif + +/* Shared/static library definitions */ +#ifdef MX25519_WIN + #ifdef MX25519_SHARED + #define MX25519_API __declspec(dllexport) + #elif !defined(MX25519_STATIC) + #define MX25519_API __declspec(dllimport) + #else + #define MX25519_API + #endif + #define MX25519_PRIVATE +#else + #ifdef MX25519_SHARED + #define MX25519_API __attribute__ ((visibility ("default"))) + #else + #define MX25519_API __attribute__ ((visibility ("hidden"))) + #endif + #define MX25519_PRIVATE __attribute__ ((visibility ("hidden"))) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Selects an implementation. + * + * @param type is the requested implementation type. If MX25519_TYPE_AUTO + * is specified, the best implementation for the current machine + * will be selected. + * + * @return pointer to an internal implementation structure. Returns NULL + * if the requested implementation is not supported. + */ +MX25519_API const mx25519_impl* mx25519_select_impl(mx25519_type type); + +/* + * @param impl is a pointer to an implementation. Must not be NULL. + * + * @return the type of the implementation. + */ +MX25519_API mx25519_type mx25519_impl_type(const mx25519_impl* impl); + +/* + * Calculates x(key*G), where G is the generator point of Curve25519. + * + * @param impl is a pointer to an implementation. Must not be NULL. + * @param result is the pointer where the resulting public key will be stored. + * Must not be NULL. + * @param key is a pointer to the private key. Must not be NULL. + */ +MX25519_API void mx25519_scmul_base(const mx25519_impl* impl, + mx25519_pubkey* result, const mx25519_privkey* key); + +/* + * Calculates x(key*P), where P is a given public key. + * + * @param impl is a pointer to an implementation. Must not be NULL. + * @param result is the pointer where the resulting public key will be stored. + * Must not be NULL. + * @param key is a pointer to the private key. Must not be NULL. + * @param p is a pointer to the base point P. Must not be NULL. + */ +MX25519_API void mx25519_scmul_key(const mx25519_impl* impl, mx25519_pubkey* result, + const mx25519_privkey* key, const mx25519_pubkey* p); + +/* + * Calculates invkey = 1/(key[0]*key[1]*...). This private key can be used + * to remove the respective private key components from a public key. + * (This only works for public keys that lie on Curve25519 and not on + * its quadratic twist.) + * + * @param invkey is the pointer where the resulting private key will be stored. + * Must not be NULL. + * @param key is an array of private keys to invert. Must not be NULL. + * @param num_keys is the number of private keys in the array. + * + * @return zero on success, a non-zero value in case of a failure. A failure + * can occur with a probability of approx. 2^(-124). + */ +MX25519_API int mx25519_invkey(mx25519_privkey* invkey, + const mx25519_privkey keys[], size_t num_keys); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/external/src/mx25519/platform.c b/external/src/mx25519/platform.c new file mode 100644 index 0000000..a5ebb24 --- /dev/null +++ b/external/src/mx25519/platform.c @@ -0,0 +1,65 @@ +/* Copyright (c) 2021-2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#include "platform.h" +#include "cpu.h" +#include + +#if defined(PLATFORM_WIN) +#include +#else +#include +#endif + +#if defined(_MSC_VER) +#include +#endif + +uint64_t mx25519_cpu_cycles() { +#if defined(PLATFORM_X86) || defined(PLATFORM_AMD64) + x25519_cpu_cap cpu_cap = mx25519_get_cpu_cap(); + if (cpu_cap & X25519_CPU_CAP_RDTSCP) { +#if defined(_MSC_VER) + uint32_t aux; + return __rdtscp(&aux); +#else + uint32_t lo, hi; + __asm__ volatile("rdtscp" : "=a"(lo), "=d"(hi) : : "%ecx"); + return ((uint64_t)hi << 32) | lo; +#endif + } +#endif +#if defined(PLATFORM_ARM64) + uint64_t vct; + __asm__ volatile("mrs %0, cntvct_el0" : "=r"(vct)); + return vct; +#endif + return clock(); /* fallback */ +} + +double mx25519_wall_clock() { +#ifdef PLATFORM_WIN + static double freq = 0; + if (freq == 0) { + LARGE_INTEGER freq_long; + if (!QueryPerformanceFrequency(&freq_long)) { + return 0; + } + freq = freq_long.QuadPart; + } + LARGE_INTEGER time; + if (!QueryPerformanceCounter(&time)) { + return 0; + } + return time.QuadPart / freq; +#else + struct timeval time; + if (gettimeofday(&time, NULL) != 0) { + return 0; + } + return (double)time.tv_sec + (double)time.tv_usec * 1.0e-6; +#endif +} diff --git a/external/src/mx25519/platform.h b/external/src/mx25519/platform.h new file mode 100644 index 0000000..3b70c15 --- /dev/null +++ b/external/src/mx25519/platform.h @@ -0,0 +1,113 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#ifndef PLATFORM_H +#define PLATFORM_H + +#include +#include + +#if defined(_WIN32) || defined(__CYGWIN__) +#define PLATFORM_WIN +#endif + +#if defined(_M_IX86) || defined(__i386) +#define PLATFORM_X86 +#elif defined(_M_X64) || defined(__x86_64__) +#define PLATFORM_AMD64 +#elif defined(_M_ARM64) || defined(__aarch64__) +#define PLATFORM_ARM64 +#else + +#endif + +#ifdef _MSC_VER +#pragma warning(error: 4013) /* calls to undefined functions */ +#pragma warning(error: 4090) /* different const qualifiers */ +#pragma warning(error: 4133) /* incompatible pointer types */ +#pragma warning(disable: 4146) /* unary minus applied to unsigned type */ +#endif + +#if defined(_M_X64) +#define PLATFORM_X64_INTRIN /* 64-bit intrinsics */ +#endif + +#if defined(__SIZEOF_INT128__) +typedef unsigned __int128 uint128_t; +#define PLATFORM_UINT128 /* compiler support for 128-bit integers */ +#endif + +/* force inline */ +#if defined(_MSC_VER) +#define FORCE_INLINE __forceinline +#elif defined(__GNUC__) || defined(__clang__) +#define FORCE_INLINE __attribute__((always_inline)) __inline__ +#else +#define FORCE_INLINE INLINE +#endif + +/* detect native little-endian platforms */ +#if (defined(__BYTE_ORDER__) && \ + (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) || \ + defined(__LITTLE_ENDIAN__) || defined(__ARMEL__) || defined(__MIPSEL__) || \ + defined(__AARCH64EL__) || defined(__amd64__) || defined(__i386__) || \ + defined(_M_IX86) || defined(_M_X64) || defined(_M_AMD64) || \ + defined(_M_ARM) +#define PLATFORM_LE +#endif +/* platforms not listed above will use endian-agnostic code */ + +/* load in little endian format */ +static FORCE_INLINE uint64_t platform_load64(const void* src) { +#if defined(PLATFORM_LE) + uint64_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t* p = (const uint8_t*)src; + uint64_t w = *p++; + w |= (uint64_t)(*p++) << 8; + w |= (uint64_t)(*p++) << 16; + w |= (uint64_t)(*p++) << 24; + w |= (uint64_t)(*p++) << 32; + w |= (uint64_t)(*p++) << 40; + w |= (uint64_t)(*p++) << 48; + w |= (uint64_t)(*p++) << 56; + return w; +#endif +} + +/* store in little endian format */ +static FORCE_INLINE void platform_store64(void* dst, uint64_t w) { +#if defined(PLATFORM_LE) + memcpy(dst, &w, sizeof w); +#else + uint8_t* p = (uint8_t*)dst; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; + w >>= 8; + *p++ = (uint8_t)w; +#endif +} + +/* current value of a hardware timer */ +uint64_t mx25519_cpu_cycles(void); + +/* time in seconds from a fixed point in the past */ +double mx25519_wall_clock(void); + +#endif diff --git a/external/src/mx25519/portable/fe.h b/external/src/mx25519/portable/fe.h new file mode 100644 index 0000000..7d53608 --- /dev/null +++ b/external/src/mx25519/portable/fe.h @@ -0,0 +1,960 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#ifndef FE_H +#define FE_H + +#ifdef _MSC_VER +#pragma warning(disable: 4146) /* unary minus applied to unsigned type */ +#endif + +#include + +typedef int32_t fe[10]; + +/* +fe means field element. +Here the field is \Z/(2^255-19). +An element t, entries t[0]...t[9], represents the integer +t[0]+2^26 t[1]+2^51 t[2]+2^77 t[3]+2^102 t[4]+...+2^230 t[9]. +Bounds on each t[i] vary depending on context. +*/ + +static void fe_frombytes(fe,const uint8_t *); +static void fe_tobytes(uint8_t *,fe); + +static void fe_copy(fe,fe); +static void fe_0(fe); +static void fe_1(fe); +static void fe_cswap(fe,fe,unsigned int); + +static void fe_add(fe,fe,fe); +static void fe_sub(fe,fe,fe); +static void fe_mul(fe,fe,fe); +static void fe_sq(fe,fe); +static void fe_mul121666(fe,fe); +static void fe_invert(fe,fe); + +static uint64_t load_3(const uint8_t *in) +{ + uint64_t result; + result = (uint64_t) in[0]; + result |= ((uint64_t) in[1]) << 8; + result |= ((uint64_t) in[2]) << 16; + return result; +} + +static uint64_t load_4(const uint8_t *in) +{ + uint64_t result; + result = (uint64_t) in[0]; + result |= ((uint64_t) in[1]) << 8; + result |= ((uint64_t) in[2]) << 16; + result |= ((uint64_t) in[3]) << 24; + return result; +} + +static void fe_frombytes(fe h,const uint8_t *s) +{ + int64_t h0 = load_4(s); + int64_t h1 = load_3(s + 4) << 6; + int64_t h2 = load_3(s + 7) << 5; + int64_t h3 = load_3(s + 10) << 3; + int64_t h4 = load_3(s + 13) << 2; + int64_t h5 = load_4(s + 16); + int64_t h6 = load_3(s + 20) << 7; + int64_t h7 = load_3(s + 23) << 5; + int64_t h8 = load_3(s + 26) << 4; + int64_t h9 = (load_3(s + 29) & 8388607) << 2; + int64_t carry0; + int64_t carry1; + int64_t carry2; + int64_t carry3; + int64_t carry4; + int64_t carry5; + int64_t carry6; + int64_t carry7; + int64_t carry8; + int64_t carry9; + + carry9 = (h9 + (int64_t) (1<<24)) >> 25; h0 += carry9 * 19; h9 -= carry9 << 25; + carry1 = (h1 + (int64_t) (1<<24)) >> 25; h2 += carry1; h1 -= carry1 << 25; + carry3 = (h3 + (int64_t) (1<<24)) >> 25; h4 += carry3; h3 -= carry3 << 25; + carry5 = (h5 + (int64_t) (1<<24)) >> 25; h6 += carry5; h5 -= carry5 << 25; + carry7 = (h7 + (int64_t) (1<<24)) >> 25; h8 += carry7; h7 -= carry7 << 25; + + carry0 = (h0 + (int64_t) (1<<25)) >> 26; h1 += carry0; h0 -= carry0 << 26; + carry2 = (h2 + (int64_t) (1<<25)) >> 26; h3 += carry2; h2 -= carry2 << 26; + carry4 = (h4 + (int64_t) (1<<25)) >> 26; h5 += carry4; h4 -= carry4 << 26; + carry6 = (h6 + (int64_t) (1<<25)) >> 26; h7 += carry6; h6 -= carry6 << 26; + carry8 = (h8 + (int64_t) (1<<25)) >> 26; h9 += carry8; h8 -= carry8 << 26; + + h[0] = h0; + h[1] = h1; + h[2] = h2; + h[3] = h3; + h[4] = h4; + h[5] = h5; + h[6] = h6; + h[7] = h7; + h[8] = h8; + h[9] = h9; +} + +/* +Preconditions: + |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc. + +Write p=2^255-19; q=floor(h/p). +Basic claim: q = floor(2^(-255)(h + 19 2^(-25)h9 + 2^(-1))). + +Proof: + Have |h|<=p so |q|<=1 so |19^2 2^(-255) q|<1/4. + Also have |h-2^230 h9|<2^230 so |19 2^(-255)(h-2^230 h9)|<1/4. + + Write y=2^(-1)-19^2 2^(-255)q-19 2^(-255)(h-2^230 h9). + Then 0> 25; + q = (h0 + q) >> 26; + q = (h1 + q) >> 25; + q = (h2 + q) >> 26; + q = (h3 + q) >> 25; + q = (h4 + q) >> 26; + q = (h5 + q) >> 25; + q = (h6 + q) >> 26; + q = (h7 + q) >> 25; + q = (h8 + q) >> 26; + q = (h9 + q) >> 25; + + /* Goal: Output h-(2^255-19)q, which is between 0 and 2^255-20. */ + h0 += 19 * q; + /* Goal: Output h-2^255 q, which is between 0 and 2^255-20. */ + + carry0 = h0 >> 26; h1 += carry0; h0 -= carry0 << 26; + carry1 = h1 >> 25; h2 += carry1; h1 -= carry1 << 25; + carry2 = h2 >> 26; h3 += carry2; h2 -= carry2 << 26; + carry3 = h3 >> 25; h4 += carry3; h3 -= carry3 << 25; + carry4 = h4 >> 26; h5 += carry4; h4 -= carry4 << 26; + carry5 = h5 >> 25; h6 += carry5; h5 -= carry5 << 25; + carry6 = h6 >> 26; h7 += carry6; h6 -= carry6 << 26; + carry7 = h7 >> 25; h8 += carry7; h7 -= carry7 << 25; + carry8 = h8 >> 26; h9 += carry8; h8 -= carry8 << 26; + carry9 = h9 >> 25; h9 -= carry9 << 25; + /* h10 = carry9 */ + + /* + Goal: Output h0+...+2^255 h10-2^255 q, which is between 0 and 2^255-20. + Have h0+...+2^230 h9 between 0 and 2^255-1; + evidently 2^255 h10-2^255 q = 0. + Goal: Output h0+...+2^230 h9. + */ + + s[0] = h0 >> 0; + s[1] = h0 >> 8; + s[2] = h0 >> 16; + s[3] = (h0 >> 24) | (h1 << 2); + s[4] = h1 >> 6; + s[5] = h1 >> 14; + s[6] = (h1 >> 22) | (h2 << 3); + s[7] = h2 >> 5; + s[8] = h2 >> 13; + s[9] = (h2 >> 21) | (h3 << 5); + s[10] = h3 >> 3; + s[11] = h3 >> 11; + s[12] = (h3 >> 19) | (h4 << 6); + s[13] = h4 >> 2; + s[14] = h4 >> 10; + s[15] = h4 >> 18; + s[16] = h5 >> 0; + s[17] = h5 >> 8; + s[18] = h5 >> 16; + s[19] = (h5 >> 24) | (h6 << 1); + s[20] = h6 >> 7; + s[21] = h6 >> 15; + s[22] = (h6 >> 23) | (h7 << 3); + s[23] = h7 >> 5; + s[24] = h7 >> 13; + s[25] = (h7 >> 21) | (h8 << 4); + s[26] = h8 >> 4; + s[27] = h8 >> 12; + s[28] = (h8 >> 20) | (h9 << 6); + s[29] = h9 >> 2; + s[30] = h9 >> 10; + s[31] = h9 >> 18; +} + +static void fe_copy(fe h, fe f) +{ + int32_t f0 = f[0]; + int32_t f1 = f[1]; + int32_t f2 = f[2]; + int32_t f3 = f[3]; + int32_t f4 = f[4]; + int32_t f5 = f[5]; + int32_t f6 = f[6]; + int32_t f7 = f[7]; + int32_t f8 = f[8]; + int32_t f9 = f[9]; + h[0] = f0; + h[1] = f1; + h[2] = f2; + h[3] = f3; + h[4] = f4; + h[5] = f5; + h[6] = f6; + h[7] = f7; + h[8] = f8; + h[9] = f9; +} + +static void fe_0(fe h) +{ + h[0] = 0; + h[1] = 0; + h[2] = 0; + h[3] = 0; + h[4] = 0; + h[5] = 0; + h[6] = 0; + h[7] = 0; + h[8] = 0; + h[9] = 0; +} + +static void fe_1(fe h) +{ + h[0] = 1; + h[1] = 0; + h[2] = 0; + h[3] = 0; + h[4] = 0; + h[5] = 0; + h[6] = 0; + h[7] = 0; + h[8] = 0; + h[9] = 0; +} + +/* +Replace (f,g) with (g,f) if b == 1; +replace (f,g) with (f,g) if b == 0. + +Preconditions: b in {0,1}. +*/ + +static void fe_cswap(fe f, fe g, unsigned int b) +{ + int32_t f0 = f[0]; + int32_t f1 = f[1]; + int32_t f2 = f[2]; + int32_t f3 = f[3]; + int32_t f4 = f[4]; + int32_t f5 = f[5]; + int32_t f6 = f[6]; + int32_t f7 = f[7]; + int32_t f8 = f[8]; + int32_t f9 = f[9]; + int32_t g0 = g[0]; + int32_t g1 = g[1]; + int32_t g2 = g[2]; + int32_t g3 = g[3]; + int32_t g4 = g[4]; + int32_t g5 = g[5]; + int32_t g6 = g[6]; + int32_t g7 = g[7]; + int32_t g8 = g[8]; + int32_t g9 = g[9]; + int32_t x0 = f0 ^ g0; + int32_t x1 = f1 ^ g1; + int32_t x2 = f2 ^ g2; + int32_t x3 = f3 ^ g3; + int32_t x4 = f4 ^ g4; + int32_t x5 = f5 ^ g5; + int32_t x6 = f6 ^ g6; + int32_t x7 = f7 ^ g7; + int32_t x8 = f8 ^ g8; + int32_t x9 = f9 ^ g9; + b = -b; + x0 &= b; + x1 &= b; + x2 &= b; + x3 &= b; + x4 &= b; + x5 &= b; + x6 &= b; + x7 &= b; + x8 &= b; + x9 &= b; + f[0] = f0 ^ x0; + f[1] = f1 ^ x1; + f[2] = f2 ^ x2; + f[3] = f3 ^ x3; + f[4] = f4 ^ x4; + f[5] = f5 ^ x5; + f[6] = f6 ^ x6; + f[7] = f7 ^ x7; + f[8] = f8 ^ x8; + f[9] = f9 ^ x9; + g[0] = g0 ^ x0; + g[1] = g1 ^ x1; + g[2] = g2 ^ x2; + g[3] = g3 ^ x3; + g[4] = g4 ^ x4; + g[5] = g5 ^ x5; + g[6] = g6 ^ x6; + g[7] = g7 ^ x7; + g[8] = g8 ^ x8; + g[9] = g9 ^ x9; +} + +/* +h = f + g +Can overlap h with f or g. + +Preconditions: + |f| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc. + |g| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc. + +Postconditions: + |h| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc. +*/ + +static void fe_add(fe h, fe f, fe g) +{ + int32_t f0 = f[0]; + int32_t f1 = f[1]; + int32_t f2 = f[2]; + int32_t f3 = f[3]; + int32_t f4 = f[4]; + int32_t f5 = f[5]; + int32_t f6 = f[6]; + int32_t f7 = f[7]; + int32_t f8 = f[8]; + int32_t f9 = f[9]; + int32_t g0 = g[0]; + int32_t g1 = g[1]; + int32_t g2 = g[2]; + int32_t g3 = g[3]; + int32_t g4 = g[4]; + int32_t g5 = g[5]; + int32_t g6 = g[6]; + int32_t g7 = g[7]; + int32_t g8 = g[8]; + int32_t g9 = g[9]; + int32_t h0 = f0 + g0; + int32_t h1 = f1 + g1; + int32_t h2 = f2 + g2; + int32_t h3 = f3 + g3; + int32_t h4 = f4 + g4; + int32_t h5 = f5 + g5; + int32_t h6 = f6 + g6; + int32_t h7 = f7 + g7; + int32_t h8 = f8 + g8; + int32_t h9 = f9 + g9; + h[0] = h0; + h[1] = h1; + h[2] = h2; + h[3] = h3; + h[4] = h4; + h[5] = h5; + h[6] = h6; + h[7] = h7; + h[8] = h8; + h[9] = h9; +} + +/* +h = f - g +Can overlap h with f or g. + +Preconditions: + |f| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc. + |g| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc. + +Postconditions: + |h| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc. +*/ + +static void fe_sub(fe h, fe f, fe g) +{ + int32_t f0 = f[0]; + int32_t f1 = f[1]; + int32_t f2 = f[2]; + int32_t f3 = f[3]; + int32_t f4 = f[4]; + int32_t f5 = f[5]; + int32_t f6 = f[6]; + int32_t f7 = f[7]; + int32_t f8 = f[8]; + int32_t f9 = f[9]; + int32_t g0 = g[0]; + int32_t g1 = g[1]; + int32_t g2 = g[2]; + int32_t g3 = g[3]; + int32_t g4 = g[4]; + int32_t g5 = g[5]; + int32_t g6 = g[6]; + int32_t g7 = g[7]; + int32_t g8 = g[8]; + int32_t g9 = g[9]; + int32_t h0 = f0 - g0; + int32_t h1 = f1 - g1; + int32_t h2 = f2 - g2; + int32_t h3 = f3 - g3; + int32_t h4 = f4 - g4; + int32_t h5 = f5 - g5; + int32_t h6 = f6 - g6; + int32_t h7 = f7 - g7; + int32_t h8 = f8 - g8; + int32_t h9 = f9 - g9; + h[0] = h0; + h[1] = h1; + h[2] = h2; + h[3] = h3; + h[4] = h4; + h[5] = h5; + h[6] = h6; + h[7] = h7; + h[8] = h8; + h[9] = h9; +} + +/* +h = f * g +Can overlap h with f or g. + +Preconditions: + |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc. + |g| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc. + +Postconditions: + |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc. +*/ + +/* +Notes on implementation strategy: + +Using schoolbook multiplication. +Karatsuba would save a little in some cost models. + +Most multiplications by 2 and 19 are 32-bit precomputations; +cheaper than 64-bit postcomputations. + +There is one remaining multiplication by 19 in the carry chain; +one *19 precomputation can be merged into this, +but the resulting data flow is considerably less clean. + +There are 12 carries below. +10 of them are 2-way parallelizable and vectorizable. +Can get away with 11 carries, but then data flow is much deeper. + +With tighter constraints on inputs can squeeze carries into int32. +*/ + +static void fe_mul(fe h, fe f, fe g) +{ + int32_t f0 = f[0]; + int32_t f1 = f[1]; + int32_t f2 = f[2]; + int32_t f3 = f[3]; + int32_t f4 = f[4]; + int32_t f5 = f[5]; + int32_t f6 = f[6]; + int32_t f7 = f[7]; + int32_t f8 = f[8]; + int32_t f9 = f[9]; + int32_t g0 = g[0]; + int32_t g1 = g[1]; + int32_t g2 = g[2]; + int32_t g3 = g[3]; + int32_t g4 = g[4]; + int32_t g5 = g[5]; + int32_t g6 = g[6]; + int32_t g7 = g[7]; + int32_t g8 = g[8]; + int32_t g9 = g[9]; + int32_t g1_19 = 19 * g1; /* 1.4*2^29 */ + int32_t g2_19 = 19 * g2; /* 1.4*2^30; still ok */ + int32_t g3_19 = 19 * g3; + int32_t g4_19 = 19 * g4; + int32_t g5_19 = 19 * g5; + int32_t g6_19 = 19 * g6; + int32_t g7_19 = 19 * g7; + int32_t g8_19 = 19 * g8; + int32_t g9_19 = 19 * g9; + int32_t f1_2 = 2 * f1; + int32_t f3_2 = 2 * f3; + int32_t f5_2 = 2 * f5; + int32_t f7_2 = 2 * f7; + int32_t f9_2 = 2 * f9; + int64_t f0g0 = f0 * (int64_t)g0; + int64_t f0g1 = f0 * (int64_t)g1; + int64_t f0g2 = f0 * (int64_t)g2; + int64_t f0g3 = f0 * (int64_t)g3; + int64_t f0g4 = f0 * (int64_t)g4; + int64_t f0g5 = f0 * (int64_t)g5; + int64_t f0g6 = f0 * (int64_t)g6; + int64_t f0g7 = f0 * (int64_t)g7; + int64_t f0g8 = f0 * (int64_t)g8; + int64_t f0g9 = f0 * (int64_t)g9; + int64_t f1g0 = f1 * (int64_t)g0; + int64_t f1g1_2 = f1_2 * (int64_t)g1; + int64_t f1g2 = f1 * (int64_t)g2; + int64_t f1g3_2 = f1_2 * (int64_t)g3; + int64_t f1g4 = f1 * (int64_t)g4; + int64_t f1g5_2 = f1_2 * (int64_t)g5; + int64_t f1g6 = f1 * (int64_t)g6; + int64_t f1g7_2 = f1_2 * (int64_t)g7; + int64_t f1g8 = f1 * (int64_t)g8; + int64_t f1g9_38 = f1_2 * (int64_t)g9_19; + int64_t f2g0 = f2 * (int64_t)g0; + int64_t f2g1 = f2 * (int64_t)g1; + int64_t f2g2 = f2 * (int64_t)g2; + int64_t f2g3 = f2 * (int64_t)g3; + int64_t f2g4 = f2 * (int64_t)g4; + int64_t f2g5 = f2 * (int64_t)g5; + int64_t f2g6 = f2 * (int64_t)g6; + int64_t f2g7 = f2 * (int64_t)g7; + int64_t f2g8_19 = f2 * (int64_t)g8_19; + int64_t f2g9_19 = f2 * (int64_t)g9_19; + int64_t f3g0 = f3 * (int64_t)g0; + int64_t f3g1_2 = f3_2 * (int64_t)g1; + int64_t f3g2 = f3 * (int64_t)g2; + int64_t f3g3_2 = f3_2 * (int64_t)g3; + int64_t f3g4 = f3 * (int64_t)g4; + int64_t f3g5_2 = f3_2 * (int64_t)g5; + int64_t f3g6 = f3 * (int64_t)g6; + int64_t f3g7_38 = f3_2 * (int64_t)g7_19; + int64_t f3g8_19 = f3 * (int64_t)g8_19; + int64_t f3g9_38 = f3_2 * (int64_t)g9_19; + int64_t f4g0 = f4 * (int64_t)g0; + int64_t f4g1 = f4 * (int64_t)g1; + int64_t f4g2 = f4 * (int64_t)g2; + int64_t f4g3 = f4 * (int64_t)g3; + int64_t f4g4 = f4 * (int64_t)g4; + int64_t f4g5 = f4 * (int64_t)g5; + int64_t f4g6_19 = f4 * (int64_t)g6_19; + int64_t f4g7_19 = f4 * (int64_t)g7_19; + int64_t f4g8_19 = f4 * (int64_t)g8_19; + int64_t f4g9_19 = f4 * (int64_t)g9_19; + int64_t f5g0 = f5 * (int64_t)g0; + int64_t f5g1_2 = f5_2 * (int64_t)g1; + int64_t f5g2 = f5 * (int64_t)g2; + int64_t f5g3_2 = f5_2 * (int64_t)g3; + int64_t f5g4 = f5 * (int64_t)g4; + int64_t f5g5_38 = f5_2 * (int64_t)g5_19; + int64_t f5g6_19 = f5 * (int64_t)g6_19; + int64_t f5g7_38 = f5_2 * (int64_t)g7_19; + int64_t f5g8_19 = f5 * (int64_t)g8_19; + int64_t f5g9_38 = f5_2 * (int64_t)g9_19; + int64_t f6g0 = f6 * (int64_t)g0; + int64_t f6g1 = f6 * (int64_t)g1; + int64_t f6g2 = f6 * (int64_t)g2; + int64_t f6g3 = f6 * (int64_t)g3; + int64_t f6g4_19 = f6 * (int64_t)g4_19; + int64_t f6g5_19 = f6 * (int64_t)g5_19; + int64_t f6g6_19 = f6 * (int64_t)g6_19; + int64_t f6g7_19 = f6 * (int64_t)g7_19; + int64_t f6g8_19 = f6 * (int64_t)g8_19; + int64_t f6g9_19 = f6 * (int64_t)g9_19; + int64_t f7g0 = f7 * (int64_t)g0; + int64_t f7g1_2 = f7_2 * (int64_t)g1; + int64_t f7g2 = f7 * (int64_t)g2; + int64_t f7g3_38 = f7_2 * (int64_t)g3_19; + int64_t f7g4_19 = f7 * (int64_t)g4_19; + int64_t f7g5_38 = f7_2 * (int64_t)g5_19; + int64_t f7g6_19 = f7 * (int64_t)g6_19; + int64_t f7g7_38 = f7_2 * (int64_t)g7_19; + int64_t f7g8_19 = f7 * (int64_t)g8_19; + int64_t f7g9_38 = f7_2 * (int64_t)g9_19; + int64_t f8g0 = f8 * (int64_t)g0; + int64_t f8g1 = f8 * (int64_t)g1; + int64_t f8g2_19 = f8 * (int64_t)g2_19; + int64_t f8g3_19 = f8 * (int64_t)g3_19; + int64_t f8g4_19 = f8 * (int64_t)g4_19; + int64_t f8g5_19 = f8 * (int64_t)g5_19; + int64_t f8g6_19 = f8 * (int64_t)g6_19; + int64_t f8g7_19 = f8 * (int64_t)g7_19; + int64_t f8g8_19 = f8 * (int64_t)g8_19; + int64_t f8g9_19 = f8 * (int64_t)g9_19; + int64_t f9g0 = f9 * (int64_t)g0; + int64_t f9g1_38 = f9_2 * (int64_t)g1_19; + int64_t f9g2_19 = f9 * (int64_t)g2_19; + int64_t f9g3_38 = f9_2 * (int64_t)g3_19; + int64_t f9g4_19 = f9 * (int64_t)g4_19; + int64_t f9g5_38 = f9_2 * (int64_t)g5_19; + int64_t f9g6_19 = f9 * (int64_t)g6_19; + int64_t f9g7_38 = f9_2 * (int64_t)g7_19; + int64_t f9g8_19 = f9 * (int64_t)g8_19; + int64_t f9g9_38 = f9_2 * (int64_t)g9_19; + int64_t h0 = f0g0 + f1g9_38 + f2g8_19 + f3g7_38 + f4g6_19 + f5g5_38 + f6g4_19 + f7g3_38 + f8g2_19 + f9g1_38; + int64_t h1 = f0g1 + f1g0 + f2g9_19 + f3g8_19 + f4g7_19 + f5g6_19 + f6g5_19 + f7g4_19 + f8g3_19 + f9g2_19; + int64_t h2 = f0g2 + f1g1_2 + f2g0 + f3g9_38 + f4g8_19 + f5g7_38 + f6g6_19 + f7g5_38 + f8g4_19 + f9g3_38; + int64_t h3 = f0g3 + f1g2 + f2g1 + f3g0 + f4g9_19 + f5g8_19 + f6g7_19 + f7g6_19 + f8g5_19 + f9g4_19; + int64_t h4 = f0g4 + f1g3_2 + f2g2 + f3g1_2 + f4g0 + f5g9_38 + f6g8_19 + f7g7_38 + f8g6_19 + f9g5_38; + int64_t h5 = f0g5 + f1g4 + f2g3 + f3g2 + f4g1 + f5g0 + f6g9_19 + f7g8_19 + f8g7_19 + f9g6_19; + int64_t h6 = f0g6 + f1g5_2 + f2g4 + f3g3_2 + f4g2 + f5g1_2 + f6g0 + f7g9_38 + f8g8_19 + f9g7_38; + int64_t h7 = f0g7 + f1g6 + f2g5 + f3g4 + f4g3 + f5g2 + f6g1 + f7g0 + f8g9_19 + f9g8_19; + int64_t h8 = f0g8 + f1g7_2 + f2g6 + f3g5_2 + f4g4 + f5g3_2 + f6g2 + f7g1_2 + f8g0 + f9g9_38; + int64_t h9 = f0g9 + f1g8 + f2g7 + f3g6 + f4g5 + f5g4 + f6g3 + f7g2 + f8g1 + f9g0; + int64_t carry0; + int64_t carry1; + int64_t carry2; + int64_t carry3; + int64_t carry4; + int64_t carry5; + int64_t carry6; + int64_t carry7; + int64_t carry8; + int64_t carry9; + + /* + |h0| <= (1.1*1.1*2^52*(1+19+19+19+19)+1.1*1.1*2^50*(38+38+38+38+38)) + i.e. |h0| <= 1.2*2^59; narrower ranges for h2, h4, h6, h8 + |h1| <= (1.1*1.1*2^51*(1+1+19+19+19+19+19+19+19+19)) + i.e. |h1| <= 1.5*2^58; narrower ranges for h3, h5, h7, h9 + */ + + carry0 = (h0 + (int64_t)(1 << 25)) >> 26; h1 += carry0; h0 -= carry0 << 26; + carry4 = (h4 + (int64_t)(1 << 25)) >> 26; h5 += carry4; h4 -= carry4 << 26; + /* |h0| <= 2^25 */ + /* |h4| <= 2^25 */ + /* |h1| <= 1.51*2^58 */ + /* |h5| <= 1.51*2^58 */ + + carry1 = (h1 + (int64_t)(1 << 24)) >> 25; h2 += carry1; h1 -= carry1 << 25; + carry5 = (h5 + (int64_t)(1 << 24)) >> 25; h6 += carry5; h5 -= carry5 << 25; + /* |h1| <= 2^24; from now on fits into int32 */ + /* |h5| <= 2^24; from now on fits into int32 */ + /* |h2| <= 1.21*2^59 */ + /* |h6| <= 1.21*2^59 */ + + carry2 = (h2 + (int64_t)(1 << 25)) >> 26; h3 += carry2; h2 -= carry2 << 26; + carry6 = (h6 + (int64_t)(1 << 25)) >> 26; h7 += carry6; h6 -= carry6 << 26; + /* |h2| <= 2^25; from now on fits into int32 unchanged */ + /* |h6| <= 2^25; from now on fits into int32 unchanged */ + /* |h3| <= 1.51*2^58 */ + /* |h7| <= 1.51*2^58 */ + + carry3 = (h3 + (int64_t)(1 << 24)) >> 25; h4 += carry3; h3 -= carry3 << 25; + carry7 = (h7 + (int64_t)(1 << 24)) >> 25; h8 += carry7; h7 -= carry7 << 25; + /* |h3| <= 2^24; from now on fits into int32 unchanged */ + /* |h7| <= 2^24; from now on fits into int32 unchanged */ + /* |h4| <= 1.52*2^33 */ + /* |h8| <= 1.52*2^33 */ + + carry4 = (h4 + (int64_t)(1 << 25)) >> 26; h5 += carry4; h4 -= carry4 << 26; + carry8 = (h8 + (int64_t)(1 << 25)) >> 26; h9 += carry8; h8 -= carry8 << 26; + /* |h4| <= 2^25; from now on fits into int32 unchanged */ + /* |h8| <= 2^25; from now on fits into int32 unchanged */ + /* |h5| <= 1.01*2^24 */ + /* |h9| <= 1.51*2^58 */ + + carry9 = (h9 + (int64_t)(1 << 24)) >> 25; h0 += carry9 * 19; h9 -= carry9 << 25; + /* |h9| <= 2^24; from now on fits into int32 unchanged */ + /* |h0| <= 1.8*2^37 */ + + carry0 = (h0 + (int64_t)(1 << 25)) >> 26; h1 += carry0; h0 -= carry0 << 26; + /* |h0| <= 2^25; from now on fits into int32 unchanged */ + /* |h1| <= 1.01*2^24 */ + + h[0] = h0; + h[1] = h1; + h[2] = h2; + h[3] = h3; + h[4] = h4; + h[5] = h5; + h[6] = h6; + h[7] = h7; + h[8] = h8; + h[9] = h9; +} + +/* +h = f * f +Can overlap h with f. + +Preconditions: + |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc. + +Postconditions: + |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc. +*/ + +/* +See fe_mul for discussion of implementation strategy. +*/ + +static void fe_sq(fe h, fe f) +{ + int32_t f0 = f[0]; + int32_t f1 = f[1]; + int32_t f2 = f[2]; + int32_t f3 = f[3]; + int32_t f4 = f[4]; + int32_t f5 = f[5]; + int32_t f6 = f[6]; + int32_t f7 = f[7]; + int32_t f8 = f[8]; + int32_t f9 = f[9]; + int32_t f0_2 = 2 * f0; + int32_t f1_2 = 2 * f1; + int32_t f2_2 = 2 * f2; + int32_t f3_2 = 2 * f3; + int32_t f4_2 = 2 * f4; + int32_t f5_2 = 2 * f5; + int32_t f6_2 = 2 * f6; + int32_t f7_2 = 2 * f7; + int32_t f5_38 = 38 * f5; /* 1.31*2^30 */ + int32_t f6_19 = 19 * f6; /* 1.31*2^30 */ + int32_t f7_38 = 38 * f7; /* 1.31*2^30 */ + int32_t f8_19 = 19 * f8; /* 1.31*2^30 */ + int32_t f9_38 = 38 * f9; /* 1.31*2^30 */ + int64_t f0f0 = f0 * (int64_t)f0; + int64_t f0f1_2 = f0_2 * (int64_t)f1; + int64_t f0f2_2 = f0_2 * (int64_t)f2; + int64_t f0f3_2 = f0_2 * (int64_t)f3; + int64_t f0f4_2 = f0_2 * (int64_t)f4; + int64_t f0f5_2 = f0_2 * (int64_t)f5; + int64_t f0f6_2 = f0_2 * (int64_t)f6; + int64_t f0f7_2 = f0_2 * (int64_t)f7; + int64_t f0f8_2 = f0_2 * (int64_t)f8; + int64_t f0f9_2 = f0_2 * (int64_t)f9; + int64_t f1f1_2 = f1_2 * (int64_t)f1; + int64_t f1f2_2 = f1_2 * (int64_t)f2; + int64_t f1f3_4 = f1_2 * (int64_t)f3_2; + int64_t f1f4_2 = f1_2 * (int64_t)f4; + int64_t f1f5_4 = f1_2 * (int64_t)f5_2; + int64_t f1f6_2 = f1_2 * (int64_t)f6; + int64_t f1f7_4 = f1_2 * (int64_t)f7_2; + int64_t f1f8_2 = f1_2 * (int64_t)f8; + int64_t f1f9_76 = f1_2 * (int64_t)f9_38; + int64_t f2f2 = f2 * (int64_t)f2; + int64_t f2f3_2 = f2_2 * (int64_t)f3; + int64_t f2f4_2 = f2_2 * (int64_t)f4; + int64_t f2f5_2 = f2_2 * (int64_t)f5; + int64_t f2f6_2 = f2_2 * (int64_t)f6; + int64_t f2f7_2 = f2_2 * (int64_t)f7; + int64_t f2f8_38 = f2_2 * (int64_t)f8_19; + int64_t f2f9_38 = f2 * (int64_t)f9_38; + int64_t f3f3_2 = f3_2 * (int64_t)f3; + int64_t f3f4_2 = f3_2 * (int64_t)f4; + int64_t f3f5_4 = f3_2 * (int64_t)f5_2; + int64_t f3f6_2 = f3_2 * (int64_t)f6; + int64_t f3f7_76 = f3_2 * (int64_t)f7_38; + int64_t f3f8_38 = f3_2 * (int64_t)f8_19; + int64_t f3f9_76 = f3_2 * (int64_t)f9_38; + int64_t f4f4 = f4 * (int64_t)f4; + int64_t f4f5_2 = f4_2 * (int64_t)f5; + int64_t f4f6_38 = f4_2 * (int64_t)f6_19; + int64_t f4f7_38 = f4 * (int64_t)f7_38; + int64_t f4f8_38 = f4_2 * (int64_t)f8_19; + int64_t f4f9_38 = f4 * (int64_t)f9_38; + int64_t f5f5_38 = f5 * (int64_t)f5_38; + int64_t f5f6_38 = f5_2 * (int64_t)f6_19; + int64_t f5f7_76 = f5_2 * (int64_t)f7_38; + int64_t f5f8_38 = f5_2 * (int64_t)f8_19; + int64_t f5f9_76 = f5_2 * (int64_t)f9_38; + int64_t f6f6_19 = f6 * (int64_t)f6_19; + int64_t f6f7_38 = f6 * (int64_t)f7_38; + int64_t f6f8_38 = f6_2 * (int64_t)f8_19; + int64_t f6f9_38 = f6 * (int64_t)f9_38; + int64_t f7f7_38 = f7 * (int64_t)f7_38; + int64_t f7f8_38 = f7_2 * (int64_t)f8_19; + int64_t f7f9_76 = f7_2 * (int64_t)f9_38; + int64_t f8f8_19 = f8 * (int64_t)f8_19; + int64_t f8f9_38 = f8 * (int64_t)f9_38; + int64_t f9f9_38 = f9 * (int64_t)f9_38; + int64_t h0 = f0f0 + f1f9_76 + f2f8_38 + f3f7_76 + f4f6_38 + f5f5_38; + int64_t h1 = f0f1_2 + f2f9_38 + f3f8_38 + f4f7_38 + f5f6_38; + int64_t h2 = f0f2_2 + f1f1_2 + f3f9_76 + f4f8_38 + f5f7_76 + f6f6_19; + int64_t h3 = f0f3_2 + f1f2_2 + f4f9_38 + f5f8_38 + f6f7_38; + int64_t h4 = f0f4_2 + f1f3_4 + f2f2 + f5f9_76 + f6f8_38 + f7f7_38; + int64_t h5 = f0f5_2 + f1f4_2 + f2f3_2 + f6f9_38 + f7f8_38; + int64_t h6 = f0f6_2 + f1f5_4 + f2f4_2 + f3f3_2 + f7f9_76 + f8f8_19; + int64_t h7 = f0f7_2 + f1f6_2 + f2f5_2 + f3f4_2 + f8f9_38; + int64_t h8 = f0f8_2 + f1f7_4 + f2f6_2 + f3f5_4 + f4f4 + f9f9_38; + int64_t h9 = f0f9_2 + f1f8_2 + f2f7_2 + f3f6_2 + f4f5_2; + int64_t carry0; + int64_t carry1; + int64_t carry2; + int64_t carry3; + int64_t carry4; + int64_t carry5; + int64_t carry6; + int64_t carry7; + int64_t carry8; + int64_t carry9; + + carry0 = (h0 + (int64_t)(1 << 25)) >> 26; h1 += carry0; h0 -= carry0 << 26; + carry4 = (h4 + (int64_t)(1 << 25)) >> 26; h5 += carry4; h4 -= carry4 << 26; + + carry1 = (h1 + (int64_t)(1 << 24)) >> 25; h2 += carry1; h1 -= carry1 << 25; + carry5 = (h5 + (int64_t)(1 << 24)) >> 25; h6 += carry5; h5 -= carry5 << 25; + + carry2 = (h2 + (int64_t)(1 << 25)) >> 26; h3 += carry2; h2 -= carry2 << 26; + carry6 = (h6 + (int64_t)(1 << 25)) >> 26; h7 += carry6; h6 -= carry6 << 26; + + carry3 = (h3 + (int64_t)(1 << 24)) >> 25; h4 += carry3; h3 -= carry3 << 25; + carry7 = (h7 + (int64_t)(1 << 24)) >> 25; h8 += carry7; h7 -= carry7 << 25; + + carry4 = (h4 + (int64_t)(1 << 25)) >> 26; h5 += carry4; h4 -= carry4 << 26; + carry8 = (h8 + (int64_t)(1 << 25)) >> 26; h9 += carry8; h8 -= carry8 << 26; + + carry9 = (h9 + (int64_t)(1 << 24)) >> 25; h0 += carry9 * 19; h9 -= carry9 << 25; + + carry0 = (h0 + (int64_t)(1 << 25)) >> 26; h1 += carry0; h0 -= carry0 << 26; + + h[0] = h0; + h[1] = h1; + h[2] = h2; + h[3] = h3; + h[4] = h4; + h[5] = h5; + h[6] = h6; + h[7] = h7; + h[8] = h8; + h[9] = h9; +} + +/* +h = f * 121666 +Can overlap h with f. + +Preconditions: + |f| bounded by 1.1*2^26,1.1*2^25,1.1*2^26,1.1*2^25,etc. + +Postconditions: + |h| bounded by 1.1*2^25,1.1*2^24,1.1*2^25,1.1*2^24,etc. +*/ + +static void fe_mul121666(fe h, fe f) +{ + int32_t f0 = f[0]; + int32_t f1 = f[1]; + int32_t f2 = f[2]; + int32_t f3 = f[3]; + int32_t f4 = f[4]; + int32_t f5 = f[5]; + int32_t f6 = f[6]; + int32_t f7 = f[7]; + int32_t f8 = f[8]; + int32_t f9 = f[9]; + int64_t h0 = f0 * (int64_t)121666; + int64_t h1 = f1 * (int64_t)121666; + int64_t h2 = f2 * (int64_t)121666; + int64_t h3 = f3 * (int64_t)121666; + int64_t h4 = f4 * (int64_t)121666; + int64_t h5 = f5 * (int64_t)121666; + int64_t h6 = f6 * (int64_t)121666; + int64_t h7 = f7 * (int64_t)121666; + int64_t h8 = f8 * (int64_t)121666; + int64_t h9 = f9 * (int64_t)121666; + int64_t carry0; + int64_t carry1; + int64_t carry2; + int64_t carry3; + int64_t carry4; + int64_t carry5; + int64_t carry6; + int64_t carry7; + int64_t carry8; + int64_t carry9; + + carry9 = (h9 + (int64_t)(1 << 24)) >> 25; h0 += carry9 * 19; h9 -= carry9 << 25; + carry1 = (h1 + (int64_t)(1 << 24)) >> 25; h2 += carry1; h1 -= carry1 << 25; + carry3 = (h3 + (int64_t)(1 << 24)) >> 25; h4 += carry3; h3 -= carry3 << 25; + carry5 = (h5 + (int64_t)(1 << 24)) >> 25; h6 += carry5; h5 -= carry5 << 25; + carry7 = (h7 + (int64_t)(1 << 24)) >> 25; h8 += carry7; h7 -= carry7 << 25; + + carry0 = (h0 + (int64_t)(1 << 25)) >> 26; h1 += carry0; h0 -= carry0 << 26; + carry2 = (h2 + (int64_t)(1 << 25)) >> 26; h3 += carry2; h2 -= carry2 << 26; + carry4 = (h4 + (int64_t)(1 << 25)) >> 26; h5 += carry4; h4 -= carry4 << 26; + carry6 = (h6 + (int64_t)(1 << 25)) >> 26; h7 += carry6; h6 -= carry6 << 26; + carry8 = (h8 + (int64_t)(1 << 25)) >> 26; h9 += carry8; h8 -= carry8 << 26; + + h[0] = h0; + h[1] = h1; + h[2] = h2; + h[3] = h3; + h[4] = h4; + h[5] = h5; + h[6] = h6; + h[7] = h7; + h[8] = h8; + h[9] = h9; +} + +static void fe_invert(fe out, fe z) +{ + fe t0; + fe t1; + fe t2; + fe t3; + int i; + + /* calculate z^(2^255-21) */ + fe_sq(t0, z); for (i = 1; i < 1; ++i) fe_sq(t0, t0); + fe_sq(t1, t0); for (i = 1; i < 2; ++i) fe_sq(t1, t1); + fe_mul(t1, z, t1); + fe_mul(t0, t0, t1); + fe_sq(t2, t0); for (i = 1; i < 1; ++i) fe_sq(t2, t2); + fe_mul(t1, t1, t2); + fe_sq(t2, t1); for (i = 1; i < 5; ++i) fe_sq(t2, t2); + fe_mul(t1, t2, t1); + fe_sq(t2, t1); for (i = 1; i < 10; ++i) fe_sq(t2, t2); + fe_mul(t2, t2, t1); + fe_sq(t3, t2); for (i = 1; i < 20; ++i) fe_sq(t3, t3); + fe_mul(t2, t3, t2); + fe_sq(t2, t2); for (i = 1; i < 10; ++i) fe_sq(t2, t2); + fe_mul(t1, t2, t1); + fe_sq(t2, t1); for (i = 1; i < 50; ++i) fe_sq(t2, t2); + fe_mul(t2, t2, t1); + fe_sq(t3, t2); for (i = 1; i < 100; ++i) fe_sq(t3, t3); + fe_mul(t2, t3, t2); + fe_sq(t2, t2); for (i = 1; i < 50; ++i) fe_sq(t2, t2); + fe_mul(t1, t2, t1); + fe_sq(t1, t1); for (i = 1; i < 5; ++i) fe_sq(t1, t1); + fe_mul(out, t1, t0); +} + +#endif diff --git a/external/src/mx25519/portable/scalarmult.c b/external/src/mx25519/portable/scalarmult.c new file mode 100644 index 0000000..b73076d --- /dev/null +++ b/external/src/mx25519/portable/scalarmult.c @@ -0,0 +1,72 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#include "scalarmult.h" +#include "fe.h" + +void mx25519_scalarmult_portable(uint8_t* q, + const uint8_t* n, + const uint8_t* p) +{ + uint8_t e[32]; + unsigned int i; + fe x1; + fe x2; + fe z2; + fe x3; + fe z3; + fe tmp0; + fe tmp1; + int pos; + unsigned int swap; + unsigned int b; + + for (i = 0; i < 32; ++i) e[i] = n[i]; + //e[0] &= 248; + e[31] &= 127; + //e[31] |= 64; do not set bit 254 + fe_frombytes(x1, p); + fe_1(x2); + fe_0(z2); + fe_copy(x3, x1); + fe_1(z3); + + swap = 0; + for (pos = 254; pos >= 0; --pos) { + b = e[pos / 8] >> (pos & 7); + b &= 1; + swap ^= b; + fe_cswap(x2, x3, swap); + fe_cswap(z2, z3, swap); + swap = b; + fe_sub(tmp0, x3, z3); + + fe_sub(tmp1, x2, z2); + fe_add(x2, x2, z2); + fe_add(z2, x3, z3); + + fe_mul(z3, tmp0, x2); + fe_mul(z2, z2, tmp1); + fe_sq(tmp0, tmp1); + fe_sq(tmp1, x2); + fe_add(x3, z3, z2); + fe_sub(z2, z3, z2); + fe_mul(x2, tmp1, tmp0); + fe_sub(tmp1, tmp1, tmp0); + fe_sq(z2, z2); + fe_mul121666(z3, tmp1); + fe_sq(x3, x3); + fe_add(tmp0, tmp0, z3); + fe_mul(z3, x1, z2); + fe_mul(z2, tmp1, tmp0); + } + fe_cswap(x2, x3, swap); + fe_cswap(z2, z3, swap); + + fe_invert(z2, z2); + fe_mul(x2, x2, z2); + fe_tobytes(q, x2); +} diff --git a/external/src/mx25519/portable/scalarmult.h b/external/src/mx25519/portable/scalarmult.h new file mode 100644 index 0000000..9b92483 --- /dev/null +++ b/external/src/mx25519/portable/scalarmult.h @@ -0,0 +1,16 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#ifndef PORTABLE_SCALARMULT_H +#define PORTABLE_SCALARMULT_H + +#include + +void mx25519_scalarmult_portable(uint8_t* q, + const uint8_t* n, + const uint8_t* p); + +#endif diff --git a/external/src/mx25519/scalar.c b/external/src/mx25519/scalar.c new file mode 100644 index 0000000..bd18170 --- /dev/null +++ b/external/src/mx25519/scalar.c @@ -0,0 +1,186 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#include "scalar.h" +#include "platform.h" + +#include "mp_ops.h" + +#define U256_CONST(d3, d2, d1, d0) {{(d0), (d1), (d2), (d3)}} + +/* l = 2^252 + 27742317777372353535851937790883648493 */ +static const x25519_scalar group_order = + U256_CONST( + 0x1000000000000000, 0x0000000000000000, + 0x14DEF9DEA2F79CD6, 0x5812631A5CF5D3ED); + +/* the Montgomery form of 8 */ +const x25519_scalar_mont mx25519_sc8_mont = + U256_CONST( + 0x0FFFFFFFFFFFFFFF, 0xFFFFFFFFFFFFFFF5, + 0xA5620A8D272931AA, 0x4EE0D5EBE20BDD6D); + +/* 2^512 mod l */ +static const x25519_scalar_mont mont_modulus = + U256_CONST( + 0x0399411B7C309A3D, 0xCEEC73D217F5BE65, + 0xD00E1BA768859347, 0xA40611E3449C0F01); + +/* -l^(-1) mod 2^256 */ +static const x25519_scalar_mont mont_rprime = + U256_CONST( + 0x9DB6C6F26FE91836, 0x14E75438FFA36BEA, + 0xB1A206F2FDBA84FF, 0xD2B51DA312547E1B); + +/* Mongomery reduction of a 512-bit product */ +static void scalar_reduce_mont(digit res[4], const digit prod[8]) { + digit mask; + digit quot[4]; + digit temp[8]; + carry cout = 0, bout = 0; + + mp_mul256_mod256(quot, prod, mont_rprime.v); // quot = prod * r' mod 2^256 + mp_mul256(temp, quot, group_order.v); // temp = quot * l + cout = mp_add512(temp, temp, prod); // temp = temp + prod + + //res = temp / 2^256 + res[0] = temp[4]; + res[1] = temp[5]; + res[2] = temp[6]; + res[3] = temp[7]; + + //constant-time subtraction of l + bout = mp_sub256(res, res, group_order.v); + mask = (digit)cout - (digit)bout; //mask = 0xFF..FF if res < l + + temp[0] = group_order.v[0] & mask; + temp[1] = group_order.v[1] & mask; + temp[2] = group_order.v[2] & mask; + temp[3] = group_order.v[3] & mask; + + mp_add256(res, res, temp); +} + +void mx25519_scalar_unpack(x25519_scalar* sc, const uint8_t key[32]) +{ + sc->v[0] = digit_load(key + 0); + sc->v[1] = digit_load(key + 8); + sc->v[2] = digit_load(key + 16); + sc->v[3] = digit_load(key + 24); +} + +void mx25519_scalar_pack(uint8_t key[32], const x25519_scalar* sc) +{ + digit_store(key + 0, sc->v[0]); + digit_store(key + 8, sc->v[1]); + digit_store(key + 16, sc->v[2]); + digit_store(key + 24, sc->v[3]); +} + +/* Converts a scalar to the Mongtgomery representation */ +void mx25519_scalar_to_mont(x25519_scalar_mont* sc_mont, const x25519_scalar* sc) +{ + digit prod[8]; + mp_mul256(prod, sc->v, mont_modulus.v); + scalar_reduce_mont(sc_mont->v, prod); +} + +/* Converts a scalar from the Montgomery representation */ +void mx25519_scalar_from_mont(x25519_scalar* sc, const x25519_scalar_mont* sc_mont) +{ + digit prod[8]; + prod[0] = sc_mont->v[0]; + prod[1] = sc_mont->v[1]; + prod[2] = sc_mont->v[2]; + prod[3] = sc_mont->v[3]; + prod[4] = 0; + prod[5] = 0; + prod[6] = 0; + prod[7] = 0; + scalar_reduce_mont(sc->v, prod); +} + +void mx25519_scalar_lsh3(x25519_scalar* sc) +{ + mp_shl(sc->v, 3); +} + +void mx25519_scalar_mul(x25519_scalar_mont* c, const x25519_scalar_mont* a, + const x25519_scalar_mont* b) +{ + digit prod[8]; + mp_mul256(prod, a->v, b->v); + scalar_reduce_mont(c->v, prod); +} + +#define scalar_mul mx25519_scalar_mul + +static void scalar_sqr(x25519_scalar_mont* c, const x25519_scalar_mont* a) +{ + digit prod[8]; + mp_sqr256(prod, a->v); + scalar_reduce_mont(c->v, prod); +} + +#define scalar_nsqr_mul(res, n, mul) do { \ + for (int i = 0; i < n; ++i) scalar_sqr(res, res); \ + scalar_mul(res, res, mul); \ +} while (0) + +/* +* Scalar inversion mod l +* https://briansmith.org/ecc-inversion-addition-chains-01#curve25519_scalar_inversion +*/ +void mx25519_scalar_inv(x25519_scalar_mont* inv, const x25519_scalar_mont* sc) +{ + x25519_scalar_mont _1 = *sc; + x25519_scalar_mont _10; + x25519_scalar_mont _100; + x25519_scalar_mont _11; + x25519_scalar_mont _101; + x25519_scalar_mont _111; + x25519_scalar_mont _1001; + x25519_scalar_mont _1011; + x25519_scalar_mont _1111; + scalar_sqr(&_10, &_1); + scalar_sqr(&_100, &_10); + scalar_mul(&_11, &_10, &_1); + scalar_mul(&_101, &_10, &_11); + scalar_mul(&_111, &_10, &_101); + scalar_mul(&_1001, &_10, &_111); + scalar_mul(&_1011, &_10, &_1001); + scalar_mul(&_1111, &_100, &_1011); + + scalar_mul(inv, &_1, &_1111); //inv = _10000 + + scalar_nsqr_mul(inv, 123 + 3, &_101); + scalar_nsqr_mul(inv, 2 + 2, &_11); + scalar_nsqr_mul(inv, 1 + 4, &_1111); + scalar_nsqr_mul(inv, 1 + 4, &_1111); + scalar_nsqr_mul(inv, 4, &_1001); + scalar_nsqr_mul(inv, 2, &_11); + scalar_nsqr_mul(inv, 1 + 4, &_1111); + scalar_nsqr_mul(inv, 1 + 3, &_101); + scalar_nsqr_mul(inv, 3 + 3, &_101); + scalar_nsqr_mul(inv, 3, &_111); + scalar_nsqr_mul(inv, 1 + 4, &_1111); + scalar_nsqr_mul(inv, 2 + 3, &_111); + scalar_nsqr_mul(inv, 2 + 2, &_11); + scalar_nsqr_mul(inv, 1 + 4, &_1011); + scalar_nsqr_mul(inv, 2 + 4, &_1011); + scalar_nsqr_mul(inv, 6 + 4, &_1001); + scalar_nsqr_mul(inv, 2 + 2, &_11); + scalar_nsqr_mul(inv, 3 + 2, &_11); + scalar_nsqr_mul(inv, 3 + 2, &_11); + scalar_nsqr_mul(inv, 1 + 4, &_1001); + scalar_nsqr_mul(inv, 1 + 3, &_111); + scalar_nsqr_mul(inv, 2 + 4, &_1111); + scalar_nsqr_mul(inv, 1 + 4, &_1011); + scalar_nsqr_mul(inv, 3, &_101); + scalar_nsqr_mul(inv, 2 + 4, &_1111); + scalar_nsqr_mul(inv, 3, &_101); + scalar_nsqr_mul(inv, 1 + 2, &_11); +} diff --git a/external/src/mx25519/scalar.h b/external/src/mx25519/scalar.h new file mode 100644 index 0000000..52e601b --- /dev/null +++ b/external/src/mx25519/scalar.h @@ -0,0 +1,36 @@ +/* Copyright (c) 2022 tevador + * + * This file is part of mx25519, which is released under LGPLv3. + * See LICENSE for full license details. +*/ + +#ifndef SCALAR_H +#define SCALAR_H + +#include "digit.h" + +#include + +typedef struct x25519_scalar { + digit v[4]; +} x25519_scalar; + +typedef struct x25519_scalar_mont { + digit v[4]; +} x25519_scalar_mont; + +extern const x25519_scalar_mont mx25519_sc8_mont; + +void mx25519_scalar_unpack(x25519_scalar* sc, const uint8_t key[32]); +void mx25519_scalar_pack(uint8_t key[32], const x25519_scalar* sc); + +void mx25519_scalar_to_mont(x25519_scalar_mont* sc_mont, const x25519_scalar* sc); +void mx25519_scalar_from_mont(x25519_scalar* sc, const x25519_scalar_mont* sc_mont); + +void mx25519_scalar_mul(x25519_scalar_mont* c, const x25519_scalar_mont* a, const x25519_scalar_mont* b); + +void mx25519_scalar_inv(x25519_scalar_mont* inv, const x25519_scalar_mont* sc); + +void mx25519_scalar_lsh3(x25519_scalar* sc); + +#endif diff --git a/src/block_template.cpp b/src/block_template.cpp index b14bc89..9f33703 100644 --- a/src/block_template.cpp +++ b/src/block_template.cpp @@ -21,12 +21,14 @@ #include "common.h" #include "block_template.h" #include "wallet.h" +#include "carrot_crypto.h" #include "crypto.h" #include "keccak.h" #include "mempool.h" #include "p2pool.h" #include "side_chain.h" #include "pool_block.h" +#include "protocol_tx_hash.h" #include "merkle.h" #include #include @@ -242,6 +244,7 @@ void BlockTemplate::update(const MinerData& data, const Mempool& mempool, const }; m_height = data.height; + m_majorVersion = data.major_version; m_difficulty = data.difficulty; m_seedHash = data.seed_hash; @@ -355,8 +358,11 @@ void BlockTemplate::update(const MinerData& data, const Mempool& mempool, const uint64_t base_reward = get_base_reward(data.already_generated_coins); + // Save the FULL reward before the split + uint64_t full_block_reward = base_reward; // ADD THIS LINE + // Salvium: 20% goes to staking, miners get 80% - base_reward = (base_reward * 4) / 5; // 80% of total reward + base_reward = base_reward - (base_reward / 5); // 80% of total reward uint64_t total_tx_fees = 0; uint64_t total_tx_weight = 0; @@ -387,7 +393,7 @@ void BlockTemplate::update(const MinerData& data, const Mempool& mempool, const }; uint64_t max_reward_amounts_weight = get_reward_amounts_weight(); - if (create_miner_tx(data, m_shares, max_reward_amounts_weight, true) < 0) { + if (create_miner_tx(data, m_shares, max_reward_amounts_weight, true, full_block_reward) < 0) { use_old_template(); return; } @@ -540,14 +546,16 @@ void BlockTemplate::update(const MinerData& data, const Mempool& mempool, const #endif } + // Salvium: 1/5 of block reward is burnt, only 4/5 goes to miners if (!SideChain::split_reward(final_reward, m_shares, m_rewards)) { use_old_template(); return; } m_finalReward = final_reward; + m_fullBlockReward = full_block_reward; - const int create_miner_tx_result = create_miner_tx(data, m_shares, max_reward_amounts_weight, false); + const int create_miner_tx_result = create_miner_tx(data, m_shares, max_reward_amounts_weight, false, full_block_reward); if (create_miner_tx_result < 0) { if (create_miner_tx_result == -3) { // Too many extra bytes were added, refine max_reward_amounts_weight and miner_tx_weight @@ -567,7 +575,7 @@ void BlockTemplate::update(const MinerData& data, const Mempool& mempool, const max_reward_amounts_weight = get_reward_amounts_weight(); - if (create_miner_tx(data, m_shares, max_reward_amounts_weight, true) < 0) { + if (create_miner_tx(data, m_shares, max_reward_amounts_weight, true, full_block_reward) < 0) { use_old_template(); return; } @@ -583,7 +591,7 @@ void BlockTemplate::update(const MinerData& data, const Mempool& mempool, const return; } - if (create_miner_tx(data, m_shares, max_reward_amounts_weight, false) < 0) { + if (create_miner_tx(data, m_shares, max_reward_amounts_weight, false, full_block_reward) < 0) { use_old_template(); return; } @@ -602,47 +610,71 @@ void BlockTemplate::update(const MinerData& data, const Mempool& mempool, const return; } - m_blockTemplateBlob = m_blockHeader; - m_extraNonceOffsetInTemplate += m_blockHeader.size(); - m_minerTxOffsetInTemplate = m_blockHeader.size(); - m_minerTxSize = m_minerTx.size(); - m_blockTemplateBlob.insert(m_blockTemplateBlob.end(), m_minerTx.begin(), m_minerTx.end()); - - // Add protocol_tx for Salvium Carrot v1+ - if (data.major_version >= 10) { - writeVarint(4, m_blockTemplateBlob); // version = TRANSACTION_VERSION_CARROT - writeVarint(60, m_blockTemplateBlob); // unlock_time = 60 - - // vin (1 txin_gen) - writeVarint(1, m_blockTemplateBlob); // vin.size() = 1 - m_blockTemplateBlob.push_back(TXIN_GEN); - writeVarint(data.height, m_blockTemplateBlob); - - // vout (empty) - writeVarint(0, m_blockTemplateBlob); // vout.size() = 0 - - // extra (2 bytes: 0x02 0x00) - writeVarint(2, m_blockTemplateBlob); // extra.size() = 2 - m_blockTemplateBlob.push_back(0x02); - m_blockTemplateBlob.push_back(0x00); - - // type = PROTOCOL - writeVarint(2, m_blockTemplateBlob); // transaction_type::PROTOCOL = 2 - - // rct_signatures (null) - m_blockTemplateBlob.push_back(0); // RCTTypeNull + m_blockTemplateBlob = m_blockHeader; + if (m_extraNonceOffsetInTemplate > 0) { + m_extraNonceOffsetInTemplate += m_blockHeader.size(); } + m_minerTxOffsetInTemplate = m_blockHeader.size(); + m_minerTxSize = m_minerTx.size(); + m_blockTemplateBlob.insert(m_blockTemplateBlob.end(), m_minerTx.begin(), m_minerTx.end()); + + // DEBUG: Show ALL of m_minerTx + LOGINFO(6, "DEBUG: FULL m_minerTx (" << m_minerTx.size() << " bytes):"); + std::string full_tx_hex; + full_tx_hex.reserve(m_minerTx.size() * 2); + for (size_t i = 0; i < m_minerTx.size(); ++i) { + char buf[3]; + snprintf(buf, sizeof(buf), "%02x", m_minerTx[i]); + full_tx_hex.append(buf); + } + LOGINFO(6, full_tx_hex); + + LOGINFO(6, "DEBUG: major_version = " << data.major_version << ", checking if >= 10"); + // Protocol tx for Salvium Carrot v1+ + + // First, calculate and store the miner TX hash + hash miner_tx_hash = calc_miner_tx_hash(0); + + // Clear and rebuild m_transactionHashes with both hashes + m_transactionHashes.clear(); + m_transactionHashes.reserve(HASH_SIZE * 2); + m_transactionHashes.insert(m_transactionHashes.end(), miner_tx_hash.h, miner_tx_hash.h + HASH_SIZE); + + LOGINFO(3, "Stored miner TX hash at position 0: " << miner_tx_hash); + + // Write protocol tx bytes to blob + writeVarint(4, m_blockTemplateBlob); // version + writeVarint(60, m_blockTemplateBlob); // unlock_time + writeVarint(1, m_blockTemplateBlob); // vin count + m_blockTemplateBlob.push_back(0xff); // TXIN_GEN + writeVarint(data.height, m_blockTemplateBlob); // height + writeVarint(0, m_blockTemplateBlob); // vout count + writeVarint(2, m_blockTemplateBlob); // extra size + m_blockTemplateBlob.push_back(0x02); // extra[0] + m_blockTemplateBlob.push_back(0x00); // extra[1] + writeVarint(2, m_blockTemplateBlob); // type PROTOCOL + m_blockTemplateBlob.push_back(0); // RCT type + + // Calculate protocol tx hash and store in member variable + calculate_protocol_tx_hash(data.height, m_protocolTxHash); + LOGINFO(3, "Protocol TX hash: " << m_protocolTxHash); + + // Add to transaction list after miner tx + m_transactionHashes.insert(m_transactionHashes.end(), m_protocolTxHash.h, m_protocolTxHash.h + HASH_SIZE); + + // Now write tx_hashes section + // For HF10+, blob tx_count excludes protocol tx (it's implicit like miner tx) + const uint64_t blob_tx_count = (data.major_version >= 10) ? (m_numTransactionHashes >= 2 ? m_numTransactionHashes - 2 : 0) : m_numTransactionHashes; + writeVarint(blob_tx_count, m_blockTemplateBlob); - writeVarint(m_numTransactionHashes, m_blockTemplateBlob); + // Miner tx hash is skipped here because it's not a part of block template + m_blockTemplateBlob.insert(m_blockTemplateBlob.end(), m_transactionHashes.begin() + HASH_SIZE * 2, m_transactionHashes.end()); - // Miner tx hash is skipped here because it's not a part of block template - m_blockTemplateBlob.insert(m_blockTemplateBlob.end(), m_transactionHashes.begin() + HASH_SIZE, m_transactionHashes.end()); - - m_poolBlockTemplate->m_transactions.clear(); + m_poolBlockTemplate->m_transactions.clear(); m_poolBlockTemplate->m_transactions.resize(1); m_poolBlockTemplate->m_transactions.reserve(m_mempoolTxsOrder.size() + 1); for (size_t i = 0, n = m_mempoolTxsOrder.size(); i < n; ++i) { - m_poolBlockTemplate->m_transactions.push_back(m_mempoolTxs[m_mempoolTxsOrder[i]].id); + m_poolBlockTemplate->m_transactions.push_back(m_mempoolTxs[m_mempoolTxsOrder[i]].id); } m_poolBlockTemplate->m_minerWallet = params->m_miningWallet; @@ -672,7 +704,7 @@ void BlockTemplate::update(const MinerData& data, const Mempool& mempool, const std::vector v; v.reserve(HASH_SIZE + 16); - v.assign(c.data.h, c.data.h + HASH_SIZE); + v.assign(c.data.h, c.data.h + HASH_SIZE * 2); writeVarint(c.difficulty.lo, v); writeVarint(c.difficulty.hi, v); @@ -733,6 +765,9 @@ void BlockTemplate::update(const MinerData& data, const Mempool& mempool, const } if (pool_block_debug()) { + + LOGINFO(3, "DEBUG: pool_block_debug() is TRUE - executing debug block"); + const size_t merkle_root_offset = m_extraNonceOffsetInTemplate + m_poolBlockTemplate->m_extraNonceSize + 2 + m_poolBlockTemplate->m_merkleTreeDataSize; memcpy(m_blockTemplateBlob.data() + merkle_root_offset, m_poolBlockTemplate->m_merkleRoot.h, HASH_SIZE); @@ -948,119 +983,194 @@ void BlockTemplate::select_mempool_transactions(const Mempool& mempool) LOGINFO(4, "mempool has " << total_mempool_transactions << " transactions, taking " << m_mempoolTxs.size() << " transactions from it"); } -int BlockTemplate::create_miner_tx(const MinerData& data, const std::vector& shares, uint64_t max_reward_amounts_weight, bool dry_run) +int BlockTemplate::create_miner_tx(const MinerData& data, const std::vector& shares, uint64_t max_reward_amounts_weight, bool dry_run, uint64_t full_block_reward) { - // Miner transaction (coinbase) - m_minerTx.clear(); + m_minerTx.clear(); const size_t num_outputs = shares.size(); m_minerTx.reserve(num_outputs * 39 + 55); - // tx version - m_minerTx.push_back(TX_VERSION); + // For Carrot v1 (HF10+), use version 4 + m_minerTx.push_back(4); // TRANSACTION_VERSION_CARROT - // Unlock time - writeVarint(MINER_REWARD_UNLOCK_TIME, m_minerTx); + writeVarint(MINER_REWARD_UNLOCK_TIME, m_minerTx); + m_minerTx.push_back(1); // Number of inputs + m_minerTx.push_back(TXIN_GEN); + writeVarint(data.height, m_minerTx); + m_poolBlockTemplate->m_txinGenHeight = data.height; - // Number of inputs - m_minerTx.push_back(1); + writeVarint(num_outputs, m_minerTx); - // Input type (txin_gen) - m_minerTx.push_back(TXIN_GEN); + m_poolBlockTemplate->m_ephPublicKeys.clear(); + m_poolBlockTemplate->m_outputAmounts.clear(); + m_poolBlockTemplate->m_ephPublicKeys.reserve(num_outputs); + m_poolBlockTemplate->m_outputAmounts.reserve(num_outputs); - // txin_gen height - writeVarint(data.height, m_minerTx); - m_poolBlockTemplate->m_txinGenHeight = data.height; + uint64_t reward_amounts_weight = 0; - // Number of outputs (1 output per miner) - writeVarint(num_outputs, m_minerTx); + // Carrot v1 outputs - prepare shared data for all outputs + uint8_t input_context[33]; + carrot::make_input_context_coinbase(data.height, input_context); - m_poolBlockTemplate->m_ephPublicKeys.clear(); - m_poolBlockTemplate->m_outputAmounts.clear(); + uint8_t null_payment_id[8] = {0}; - m_poolBlockTemplate->m_ephPublicKeys.reserve(num_outputs); - m_poolBlockTemplate->m_outputAmounts.reserve(num_outputs); + for (size_t i = 0; i < num_outputs; ++i) { + // Amount (not encrypted for coinbase) + writeVarint(m_rewards[i], [this, &reward_amounts_weight](uint8_t b) { + m_minerTx.push_back(b); + ++reward_amounts_weight; + }); - uint64_t reward_amounts_weight = 0; - for (size_t i = 0; i < num_outputs; ++i) { - writeVarint(m_rewards[i], [this, &reward_amounts_weight](uint8_t b) - { - m_minerTx.push_back(b); - ++reward_amounts_weight; - }); - m_minerTx.push_back(TXOUT_TO_TAGGED_KEY); + // txout_to_carrot_v1 structure + m_minerTx.push_back(TXOUT_TO_CARROT_V1); // variant tag = 4 - uint8_t view_tag = 0; + // K_o - onetime address (32 bytes) + hash onetime_address; + hash ephemeral_pubkey; + uint8_t view_tag[3] = {0}; + uint8_t encrypted_anchor[16] = {0}; + + if (!dry_run) { + // Generate janus anchor (randomness for this output) + uint8_t anchor[16]; + carrot::generate_janus_anchor(anchor); + + // Generate ephemeral private key + hash ephemeral_privkey; + carrot::make_ephemeral_privkey( + anchor, + input_context, + shares[i].m_wallet->spend_public_key(), + null_payment_id, + ephemeral_privkey); + + // Generate ephemeral public key D_e + carrot::make_ephemeral_pubkey_mainaddress(ephemeral_privkey, ephemeral_pubkey); + + // Generate shared secret (sender-side ECDH) + hash shared_secret_unctx; + if (!carrot::make_shared_secret_sender( + ephemeral_privkey, + shares[i].m_wallet->view_public_key(), + shared_secret_unctx)) { + LOGERR(1, "Failed to generate shared secret for output " << i); + return -4; + } + + // Generate contextualized sender-receiver secret + hash sender_receiver_secret; + carrot::make_sender_receiver_secret( + shared_secret_unctx, + ephemeral_pubkey, + input_context, + sender_receiver_secret); + + // Generate onetime address K_o + carrot::make_onetime_address_coinbase( + shares[i].m_wallet->spend_public_key(), + sender_receiver_secret, + m_rewards[i], + onetime_address); + + // Generate 3-byte view tag + carrot::make_view_tag(shared_secret_unctx, input_context, onetime_address, view_tag); + + // Encrypt janus anchor + carrot::encrypt_anchor(anchor, sender_receiver_secret, onetime_address, encrypted_anchor); + + // Save for pool block template + m_poolBlockTemplate->m_ephPublicKeys.emplace_back(ephemeral_pubkey); + m_poolBlockTemplate->m_outputAmounts.emplace_back(m_rewards[i], view_tag[0]); - if (dry_run) { - m_minerTx.insert(m_minerTx.end(), HASH_SIZE, 0); - } - else { - hash eph_public_key; - if (!shares[i].m_wallet->get_eph_public_key(m_poolBlockTemplate->m_txkeySec, i, eph_public_key, view_tag)) { - LOGERR(1, "get_eph_public_key failed at index " << i); - } - m_minerTx.insert(m_minerTx.end(), eph_public_key.h, eph_public_key.h + HASH_SIZE); - m_poolBlockTemplate->m_ephPublicKeys.emplace_back(eph_public_key); - m_poolBlockTemplate->m_outputAmounts.emplace_back(m_rewards[i], view_tag); - } + // Save view_tag and encrypted_anchor for later + std::vector vt(view_tag, view_tag + 3); + std::vector ea(encrypted_anchor, encrypted_anchor + 16); + m_poolBlockTemplate->m_viewTags.push_back(vt); + m_poolBlockTemplate->m_encryptedAnchors.push_back(ea); + } - m_minerTx.emplace_back(view_tag); - } + // Write output data (zeros for dry_run, real values otherwise) + m_minerTx.insert(m_minerTx.end(), onetime_address.h, onetime_address.h + HASH_SIZE); - if (dry_run) { - if (reward_amounts_weight != max_reward_amounts_weight) { - LOGERR(1, "create_miner_tx: incorrect miner rewards during the dry run (" << reward_amounts_weight << " != " << max_reward_amounts_weight << ")"); - return -1; - } - } - else if (reward_amounts_weight > max_reward_amounts_weight) { - LOGERR(1, "create_miner_tx: incorrect miner rewards during the real run (" << reward_amounts_weight << " > " << max_reward_amounts_weight << ")"); - return -2; - } + // asset_type - string "SAL1" + m_minerTx.push_back(4); // string length + m_minerTx.push_back('S'); + m_minerTx.push_back('A'); + m_minerTx.push_back('L'); + m_minerTx.push_back('1'); - // TX_EXTRA begin - m_minerTxExtra.clear(); + // view_tag and encrypted_anchor + m_minerTx.insert(m_minerTx.end(), view_tag, view_tag + 3); + m_minerTx.insert(m_minerTx.end(), encrypted_anchor, encrypted_anchor + 16); + } - m_minerTxExtra.push_back(TX_EXTRA_TAG_PUBKEY); - m_minerTxExtra.insert(m_minerTxExtra.end(), m_poolBlockTemplate->m_txkeyPub.h, m_poolBlockTemplate->m_txkeyPub.h + HASH_SIZE); + if (dry_run) { + if (reward_amounts_weight != max_reward_amounts_weight) { + LOGERR(1, "create_miner_tx: incorrect miner rewards during dry run"); + return -1; + } + } else if (reward_amounts_weight > max_reward_amounts_weight) { + LOGERR(1, "create_miner_tx: incorrect miner rewards during real run"); + return -2; + } - m_minerTxExtra.push_back(TX_EXTRA_NONCE); + // TX_EXTRA + LOGINFO(3, "DEBUG: Carrot extra - major_version=" << data.major_version << ", static_cast(dry_run)=" << static_cast(dry_run) << ", num_eph_keys=" << m_poolBlockTemplate->m_ephPublicKeys.size()); + m_minerTxExtra.clear(); - const uint64_t corrected_extra_nonce_size = EXTRA_NONCE_SIZE + max_reward_amounts_weight - reward_amounts_weight; - if (corrected_extra_nonce_size > EXTRA_NONCE_SIZE) { - if (corrected_extra_nonce_size > EXTRA_NONCE_MAX_SIZE) { - LOGWARN(5, "create_miner_tx: corrected_extra_nonce_size (" << corrected_extra_nonce_size << ") is too large"); - return -3; - } - LOGINFO(4, "increased EXTRA_NONCE from " << EXTRA_NONCE_SIZE << " to " << corrected_extra_nonce_size << " bytes to maintain miner tx weight"); - } - writeVarint(corrected_extra_nonce_size, m_minerTxExtra); - - uint64_t extraNonceOffsetInMinerTx = m_minerTxExtra.size(); - m_minerTxExtra.insert(m_minerTxExtra.end(), corrected_extra_nonce_size, 0); + // Carrot v1: TX_EXTRA contains ephemeral pubkey, extra_nonce, and merge mining tag + // Ephemeral pubkey + m_minerTxExtra.push_back(TX_EXTRA_TAG_PUBKEY); + if (dry_run) { + m_minerTxExtra.insert(m_minerTxExtra.end(), HASH_SIZE, 0); + } else { + m_minerTxExtra.insert(m_minerTxExtra.end(), + m_poolBlockTemplate->m_ephPublicKeys[0].h, + m_poolBlockTemplate->m_ephPublicKeys[0].h + HASH_SIZE); + } - m_poolBlockTemplate->m_extraNonceSize = corrected_extra_nonce_size; + // Extra nonce + m_minerTxExtra.push_back(TX_EXTRA_NONCE); + const uint64_t corrected_extra_nonce_size = EXTRA_NONCE_SIZE + max_reward_amounts_weight - reward_amounts_weight; + if (corrected_extra_nonce_size > EXTRA_NONCE_MAX_SIZE) { + LOGWARN(5, "create_miner_tx: corrected_extra_nonce_size too large"); + return -3; + } + writeVarint(corrected_extra_nonce_size, m_minerTxExtra); + uint64_t extraNonceOffsetInMinerTx = m_minerTxExtra.size(); + m_minerTxExtra.insert(m_minerTxExtra.end(), corrected_extra_nonce_size, 0); + m_poolBlockTemplate->m_extraNonceSize = corrected_extra_nonce_size; - m_minerTxExtra.push_back(TX_EXTRA_MERGE_MINING_TAG); + // Merge mining tag + m_minerTxExtra.push_back(TX_EXTRA_MERGE_MINING_TAG); + m_minerTxExtra.push_back(static_cast(m_poolBlockTemplate->m_merkleTreeDataSize + HASH_SIZE)); + writeVarint(m_poolBlockTemplate->m_merkleTreeData, m_minerTxExtra); + m_minerTxExtra.insert(m_minerTxExtra.end(), HASH_SIZE, 0); - m_minerTxExtra.push_back(static_cast(m_poolBlockTemplate->m_merkleTreeDataSize + HASH_SIZE)); - writeVarint(m_poolBlockTemplate->m_merkleTreeData, m_minerTxExtra); - m_minerTxExtra.insert(m_minerTxExtra.end(), HASH_SIZE, 0); - // TX_EXTRA end + // Write TX_EXTRA to miner tx + writeVarint(m_minerTxExtra.size(), m_minerTx); + extraNonceOffsetInMinerTx += m_minerTx.size(); + m_extraNonceOffsetInTemplate = extraNonceOffsetInMinerTx; + m_minerTx.insert(m_minerTx.end(), m_minerTxExtra.begin(), m_minerTxExtra.end()); - writeVarint(m_minerTxExtra.size(), m_minerTx); - extraNonceOffsetInMinerTx += m_minerTx.size(); - m_extraNonceOffsetInTemplate = extraNonceOffsetInMinerTx; - m_minerTx.insert(m_minerTx.end(), m_minerTxExtra.begin(), m_minerTxExtra.end()); + m_minerTxExtra.clear(); - m_minerTxExtra.clear(); + // type = MINER (1) + writeVarint(1, m_minerTx); - // vin_rct_type - // Not a part of transaction hash data - m_minerTx.push_back(0); + // amount_burnt = 20% of total block reward = 25% of miner outputs + uint64_t miner_total = 0; + for (size_t i = 0; i < num_outputs; ++i) { + miner_total += m_rewards[i]; + } + uint64_t stake_amount = full_block_reward / 5; + writeVarint(stake_amount, m_minerTx); - return 1; + // Save prefix size - everything up to here is the transaction prefix + m_minerTxPrefixSize = static_cast(m_minerTx.size()); + + m_minerTx.push_back(0); // RCT type + return 1; } hash BlockTemplate::calc_sidechain_hash(uint32_t sidechain_extra_nonce) const @@ -1115,107 +1225,148 @@ hash BlockTemplate::calc_sidechain_hash(uint32_t sidechain_extra_nonce) const hash BlockTemplate::calc_miner_tx_hash(uint32_t extra_nonce) const { - // Calculate 3 partial hashes - uint8_t hashes[HASH_SIZE * 3]; + uint8_t hashes[HASH_SIZE * 3]; + const uint8_t* data = m_blockTemplateBlob.data() + m_minerTxOffsetInTemplate; + const size_t prefix_size = m_minerTxPrefixSize; + const size_t base_rct_size = m_minerTxSize - prefix_size; + + LOGINFO(3, "DEBUG: minerTxOffsetInTemplate=" << m_minerTxOffsetInTemplate << ", m_minerTxSize=" << m_minerTxSize); + LOGINFO(3, "DEBUG: First 20 bytes of miner tx in template:"); + char hex_buf[128] = {0}; + for (size_t i = 0; i < 20 && i < m_minerTxSize; ++i) { + snprintf(hex_buf + i*2, 3, "%02x", data[i]); + } + LOGINFO(3, static_cast(hex_buf)); - const uint8_t* data = m_blockTemplateBlob.data() + m_minerTxOffsetInTemplate; + // Pre-Carrot: original logic with patching + const size_t extra_nonce_offset = m_extraNonceOffsetInTemplate - m_minerTxOffsetInTemplate; + const uint8_t extra_nonce_buf[EXTRA_NONCE_SIZE] = { + static_cast(extra_nonce >> 0), + static_cast(extra_nonce >> 8), + static_cast(extra_nonce >> 16), + static_cast(extra_nonce >> 24) + }; - const size_t extra_nonce_offset = m_extraNonceOffsetInTemplate - m_minerTxOffsetInTemplate; - const uint8_t extra_nonce_buf[EXTRA_NONCE_SIZE] = { - static_cast(extra_nonce >> 0), - static_cast(extra_nonce >> 8), - static_cast(extra_nonce >> 16), - static_cast(extra_nonce >> 24) - }; + hash merge_mining_root; + { + const hash sidechain_id = calc_sidechain_hash(extra_nonce); + const uint32_t n_aux_chains = static_cast(m_poolBlockTemplate->m_auxChains.size() + 1); + const uint32_t aux_slot = get_aux_slot(m_sidechain->consensus_hash(), m_poolBlockTemplate->m_auxNonce, n_aux_chains); + merge_mining_root = get_root_from_proof(sidechain_id, m_poolBlockTemplate->m_merkleProof, aux_slot, n_aux_chains); + } - // Calculate sidechain id and merge mining root hash with this extra_nonce - hash merge_mining_root; - { - const hash sidechain_id = calc_sidechain_hash(extra_nonce); - const uint32_t n_aux_chains = static_cast(m_poolBlockTemplate->m_auxChains.size() + 1); - const uint32_t aux_slot = get_aux_slot(m_sidechain->consensus_hash(), m_poolBlockTemplate->m_auxNonce, n_aux_chains); - merge_mining_root = get_root_from_proof(sidechain_id, m_poolBlockTemplate->m_merkleProof, aux_slot, n_aux_chains); - } + const size_t merkle_root_offset = extra_nonce_offset + m_poolBlockTemplate->m_extraNonceSize + 2 + m_poolBlockTemplate->m_merkleTreeDataSize; - const size_t merkle_root_offset = extra_nonce_offset + m_poolBlockTemplate->m_extraNonceSize + 2 + m_poolBlockTemplate->m_merkleTreeDataSize; + // 1. Hash prefix with extra_nonce and merge_mining_root applied + hash full_hash; + uint8_t tx_buf[288]; - // 1. Prefix (everything except vin_rct_type byte in the end) - // Apply extra_nonce in-place because we can't write to the block template here - const size_t tx_size = m_minerTxSize - 1; + const size_t N = m_minerTxKeccakStateInputLength; + const bool b = N && (N <= extra_nonce_offset) && (N < prefix_size) && (prefix_size - N <= sizeof(tx_buf)); - hash full_hash; - uint8_t tx_buf[288]; + LOGINFO(6, "DEBUG: extra_nonce=" << extra_nonce << ", extra_nonce_offset=" << extra_nonce_offset << ", merkle_root_offset=" << merkle_root_offset); - const size_t N = m_minerTxKeccakStateInputLength; - const bool b = N && (N <= extra_nonce_offset) && (N < tx_size) && (tx_size - N <= sizeof(tx_buf)); + // DEBUG: Log what we're actually hashing + std::vector debug_prefix(prefix_size); + for (size_t i = 0; i < prefix_size; ++i) { + uint32_t k = static_cast(i - extra_nonce_offset); + if (k < EXTRA_NONCE_SIZE) { + debug_prefix[i] = extra_nonce_buf[k]; + } else { + k = static_cast(i - merkle_root_offset); + if (k < HASH_SIZE) { + debug_prefix[i] = merge_mining_root.h[k]; + } else { + debug_prefix[i] = data[i]; + } + } + } + LOGINFO(6, "DEBUG: Hashing prefix (" << prefix_size << " bytes): " << log::hex_buf(debug_prefix.data(), std::min(size_t(120), prefix_size))); - // Slow path: O(N) - if (!b || pool_block_debug()) - { - keccak_custom([data, extra_nonce_offset, &extra_nonce_buf, merkle_root_offset, &merge_mining_root](int offset) { - uint32_t k = static_cast(offset - static_cast(extra_nonce_offset)); - if (k < EXTRA_NONCE_SIZE) { - return extra_nonce_buf[k]; - } + // Slow path: O(N) + if (!b || pool_block_debug()) + { + keccak_custom([data, extra_nonce_offset, &extra_nonce_buf, merkle_root_offset, &merge_mining_root](int offset) { + uint32_t k = static_cast(offset - static_cast(extra_nonce_offset)); + if (k < EXTRA_NONCE_SIZE) { + return extra_nonce_buf[k]; + } - k = static_cast(offset - static_cast(merkle_root_offset)); - if (k < HASH_SIZE) { - return merge_mining_root.h[k]; - } + k = static_cast(offset - static_cast(merkle_root_offset)); + if (k < HASH_SIZE) { + return merge_mining_root.h[k]; + } - return data[offset]; - }, static_cast(tx_size), full_hash.h, HASH_SIZE); - memcpy(hashes, full_hash.h, HASH_SIZE); - } + return data[offset]; + }, static_cast(prefix_size), full_hash.h, HASH_SIZE); + memcpy(hashes, full_hash.h, HASH_SIZE); + } - // Fast path: O(1) - if (b) { - const int inlen = static_cast(tx_size - N); + // Fast path: O(1) + if (b) { + const int inlen = static_cast(prefix_size - N); - memcpy(tx_buf, data + N, inlen); - memcpy(tx_buf + extra_nonce_offset - N, extra_nonce_buf, EXTRA_NONCE_SIZE); - memcpy(tx_buf + merkle_root_offset - N, merge_mining_root.h, HASH_SIZE); + memcpy(tx_buf, data + N, inlen); + memcpy(tx_buf + extra_nonce_offset - N, extra_nonce_buf, EXTRA_NONCE_SIZE); + memcpy(tx_buf + merkle_root_offset - N, merge_mining_root.h, HASH_SIZE); - std::array st = m_minerTxKeccakState; - keccak_finish(tx_buf, inlen, st); + std::array st = m_minerTxKeccakState; + keccak_finish(tx_buf, inlen, st); - if (pool_block_debug() && (memcmp(st.data(), full_hash.h, HASH_SIZE) != 0)) { - LOGERR(1, "calc_miner_tx_hash fast path is broken. Fix the code!"); - } + if (pool_block_debug() && (memcmp(st.data(), full_hash.h, HASH_SIZE) != 0)) { + LOGERR(1, "calc_miner_tx_hash fast path is broken. Fix the code!"); + } - memcpy(hashes, st.data(), HASH_SIZE); - } + memcpy(hashes, st.data(), HASH_SIZE); + } - // 2. Base RCT, single 0 byte in miner tx - static constexpr uint8_t known_second_hash[HASH_SIZE] = { - 188,54,120,158,122,30,40,20,54,70,66,41,130,143,129,125,102,18,247,180,119,214,101,145,255,150,169,224,100,188,201,138 - }; - memcpy(hashes + HASH_SIZE, known_second_hash, HASH_SIZE); + // 2. Hash base RCT (type + amount_burnt bytes) + uint8_t base_rct_hash[HASH_SIZE]; + keccak(data + prefix_size, static_cast(base_rct_size), base_rct_hash); + memcpy(hashes + HASH_SIZE, base_rct_hash, HASH_SIZE); - // 3. Prunable RCT, empty in miner tx - memset(hashes + HASH_SIZE * 2, 0, HASH_SIZE); + // 3. Prunable RCT is null for coinbase + memset(hashes + HASH_SIZE * 2, 0, HASH_SIZE); - // Calculate miner transaction hash - hash result; - keccak(hashes, sizeof(hashes), result.h); + // Calculate miner transaction hash (hash of the 3 hashes) + hash result; + keccak(hashes, sizeof(hashes), result.h); - return result; + // Debug: log the component hashes + char prefix_hash_hex[65] = {0}; + char base_rct_hash_hex[65] = {0}; + char prunable_hash_hex[65] = {0}; + char final_hash_hex[65] = {0}; + for (int i = 0; i < 32; ++i) { + snprintf(prefix_hash_hex + i*2, 3, "%02x", hashes[i]); + snprintf(base_rct_hash_hex + i*2, 3, "%02x", hashes[32 + i]); + snprintf(prunable_hash_hex + i*2, 3, "%02x", hashes[64 + i]); + snprintf(final_hash_hex + i*2, 3, "%02x", result.h[i]); + } + LOGINFO(3, "Miner TX hash components:"); + LOGINFO(3, " Prefix hash: " << static_cast(prefix_hash_hex)); + LOGINFO(3, " Base RCT hash: " << static_cast(base_rct_hash_hex)); + LOGINFO(3, " Prunable hash: " << static_cast(prunable_hash_hex)); + LOGINFO(3, " Final TX hash: " << static_cast(final_hash_hex)); + LOGINFO(3, " Prefix size: " << prefix_size << ", Base RCT size: " << base_rct_size << ", Total TX size: " << m_minerTxSize); + + return result; } void BlockTemplate::calc_merkle_tree_main_branch() { - m_merkleTreeMainBranch.clear(); - - const uint64_t count = m_numTransactionHashes + 1; - if (count == 1) { - return; - } - - const uint8_t* h = m_transactionHashes.data(); - - if (count == 2) { - m_merkleTreeMainBranch.insert(m_merkleTreeMainBranch.end(), h + HASH_SIZE, h + HASH_SIZE * 2); - } + m_merkleTreeMainBranch.clear(); + const uint64_t count = m_numTransactionHashes + (m_majorVersion >= 10 ? 2 : 1); + if (count == 1) { + return; + } + const uint8_t* h = m_transactionHashes.data(); + if (count == 2) { + hash protocol_hash; + memcpy(protocol_hash.h, h + HASH_SIZE, HASH_SIZE); + LOGINFO(3, "Merkle branch protocol tx hash: " << protocol_hash); + m_merkleTreeMainBranch.insert(m_merkleTreeMainBranch.end(), h + HASH_SIZE, h + HASH_SIZE * 2); + } else { size_t i, j, cnt; @@ -1249,6 +1400,18 @@ void BlockTemplate::calc_merkle_tree_main_branch() m_merkleTreeMainBranch.insert(m_merkleTreeMainBranch.end(), ints.data() + HASH_SIZE, ints.data() + HASH_SIZE * 2); } + // DEBUG: Log the calculated merkle root + if (m_majorVersion >= 10) { + hash merkle_root; + // The merkle root is the hash of (miner_hash + last_branch_element) + uint8_t buf[HASH_SIZE * 2]; + memcpy(buf, m_transactionHashes.data(), HASH_SIZE); + if (!m_merkleTreeMainBranch.empty()) { + memcpy(buf + HASH_SIZE, m_merkleTreeMainBranch.data() + m_merkleTreeMainBranch.size() - HASH_SIZE, HASH_SIZE); + keccak(buf, sizeof(buf), merkle_root.h); + LOGINFO(6, "Calculated merkle root: " << merkle_root); + } + } } bool BlockTemplate::get_difficulties(const uint32_t template_id, uint64_t& height, uint64_t& sidechain_height, difficulty_type& mainchain_difficulty, difficulty_type& aux_diff, difficulty_type& sidechain_difficulty) const @@ -1315,31 +1478,62 @@ uint32_t BlockTemplate::get_hashing_blob(uint32_t extra_nonce, uint8_t (&blob)[1 uint32_t BlockTemplate::get_hashing_blob_nolock(uint32_t extra_nonce, uint8_t* blob) const { - uint8_t* p = blob; + uint8_t* p = blob; + // Block header + memcpy(p, m_blockTemplateBlob.data(), m_blockHeaderSize); + p += m_blockHeaderSize; + + // Merkle tree hash + hash root_hash = calc_miner_tx_hash(extra_nonce); + + // For Carrot v1 (HF10+) with protocol TX, simple 2-transaction merkle tree + if (m_majorVersion >= 10) { + // Just hash miner TX hash + protocol TX hash + uint8_t merkle_data[HASH_SIZE * 2]; + memcpy(merkle_data, root_hash.h, HASH_SIZE); + // Protocol TX hash is the first (and only) entry in transaction hashes after miner TX + memcpy(merkle_data + HASH_SIZE, m_protocolTxHash.h, HASH_SIZE); + + // DEBUG: Show what we're actually hashing + LOGINFO(6, "DEBUG: About to hash merkle data (raw bytes):"); + LOGINFO(6, " Full 64 bytes: " << log::hex_buf(merkle_data, HASH_SIZE * 2)); - // Block header - memcpy(p, m_blockTemplateBlob.data(), m_blockHeaderSize); - p += m_blockHeaderSize; + LOGINFO(6, " First 32 bytes (miner): " << log::hex_buf(merkle_data, HASH_SIZE)); + LOGINFO(6, " Second 32 bytes (protocol): " << log::hex_buf(merkle_data + HASH_SIZE, HASH_SIZE)); + LOGINFO(6, " m_transactionHashes size: " << m_transactionHashes.size()); + + keccak(merkle_data, HASH_SIZE * 2, root_hash.h); - // Merkle tree hash - hash root_hash = calc_miner_tx_hash(extra_nonce); + // DEBUG: Verify the result + LOGINFO(6, " Result merkle root: " << log::hex_buf(root_hash.h, HASH_SIZE)); + + // DEBUG: Manually verify by re-hashing + hash verify_hash; + keccak(merkle_data, HASH_SIZE * 2, verify_hash.h); + LOGINFO(6, " Verify merkle root: " << log::hex_buf(verify_hash.h, HASH_SIZE)); - for (size_t i = 0; i < m_merkleTreeMainBranch.size(); i += HASH_SIZE) { - uint8_t h[HASH_SIZE * 2]; + } else { + // Pre-Carrot: use merkle branch logic + for (size_t i = 0; i < m_merkleTreeMainBranch.size(); i += HASH_SIZE) { + uint8_t h[HASH_SIZE * 2]; + memcpy(h, root_hash.h, HASH_SIZE); + memcpy(h + HASH_SIZE, m_merkleTreeMainBranch.data() + i, HASH_SIZE); + keccak(h, HASH_SIZE * 2, root_hash.h); + } + } + + memcpy(p, root_hash.h, HASH_SIZE); + p += HASH_SIZE; + + // Total number of transactions in this block (including the miner tx) + // FOR HF10+, include both miner tx and protocol tx + const uint64_t tx_count_in_header = m_numTransactionHashes + (m_majorVersion >= 10 ? 2 : 1); + writeVarint(tx_count_in_header, [&p](uint8_t b) { *(p++) = b; }); - memcpy(h, root_hash.h, HASH_SIZE); - memcpy(h + HASH_SIZE, m_merkleTreeMainBranch.data() + i, HASH_SIZE); - - keccak(h, HASH_SIZE * 2, root_hash.h); - } - - memcpy(p, root_hash.h, HASH_SIZE); - p += HASH_SIZE; - - // Total number of transactions in this block (including the miner tx) - writeVarint(m_numTransactionHashes + 1, [&p](uint8_t b) { *(p++) = b; }); - - return static_cast(p - blob); + // DEBUG: Show what hashing blob we're creating + LOGINFO(6, "DEBUG get_hashing_blob result (" << static_cast(p - blob) << " bytes): " << log::hex_buf(blob, std::min(size_t(76), static_cast(p - blob)))); + + return static_cast(p - blob); } uint32_t BlockTemplate::get_hashing_blobs(uint32_t extra_nonce_start, uint32_t count, std::vector& blobs, uint64_t& height, difficulty_type& difficulty, difficulty_type& aux_diff, difficulty_type& sidechain_difficulty, hash& seed_hash, size_t& nonce_offset, uint32_t& template_id) const @@ -1478,34 +1672,35 @@ bool BlockTemplate::get_aux_proof(const uint32_t template_id, uint32_t extra_non std::vector BlockTemplate::get_block_template_blob(uint32_t template_id, uint32_t sidechain_extra_nonce, size_t& nonce_offset, size_t& extra_nonce_offset, size_t& merkle_root_offset, hash& merge_mining_root, const BlockTemplate** pThis) const { - ReadLock lock(m_lock); + ReadLock lock(m_lock); + if (template_id != m_templateId) { + const BlockTemplate* old = m_oldTemplates[template_id % array_size(&BlockTemplate::m_oldTemplates)]; + if (old && (template_id == old->m_templateId)) { + return old->get_block_template_blob(template_id, sidechain_extra_nonce, nonce_offset, extra_nonce_offset, merkle_root_offset, merge_mining_root, pThis); + } + nonce_offset = 0; + extra_nonce_offset = 0; + merkle_root_offset = 0; + merge_mining_root = {}; + return std::vector(); + } - if (template_id != m_templateId) { - const BlockTemplate* old = m_oldTemplates[template_id % array_size(&BlockTemplate::m_oldTemplates)]; - if (old && (template_id == old->m_templateId)) { - return old->get_block_template_blob(template_id, sidechain_extra_nonce, nonce_offset, extra_nonce_offset, merkle_root_offset, merge_mining_root, pThis); - } - - nonce_offset = 0; - extra_nonce_offset = 0; - merkle_root_offset = 0; - merge_mining_root = {}; - return std::vector(); - } - - nonce_offset = m_nonceOffset; - extra_nonce_offset = m_extraNonceOffsetInTemplate; - - const hash sidechain_id = calc_sidechain_hash(sidechain_extra_nonce); - const uint32_t n_aux_chains = static_cast(m_poolBlockTemplate->m_auxChains.size() + 1); - const uint32_t aux_slot = get_aux_slot(m_sidechain->consensus_hash(), m_poolBlockTemplate->m_auxNonce, n_aux_chains); - merge_mining_root = get_root_from_proof(sidechain_id, m_poolBlockTemplate->m_merkleProof, aux_slot, n_aux_chains); - - merkle_root_offset = m_extraNonceOffsetInTemplate + m_poolBlockTemplate->m_extraNonceSize + 2 + m_poolBlockTemplate->m_merkleTreeDataSize; - - *pThis = this; - - return m_blockTemplateBlob; + nonce_offset = m_nonceOffset; + extra_nonce_offset = m_extraNonceOffsetInTemplate; + + const hash sidechain_id = calc_sidechain_hash(sidechain_extra_nonce); + const uint32_t n_aux_chains = static_cast(m_poolBlockTemplate->m_auxChains.size() + 1); + const uint32_t aux_slot = get_aux_slot(m_sidechain->consensus_hash(), m_poolBlockTemplate->m_auxNonce, n_aux_chains); + merge_mining_root = get_root_from_proof(sidechain_id, m_poolBlockTemplate->m_merkleProof, aux_slot, n_aux_chains); + + if (m_extraNonceOffsetInTemplate > 0) { + merkle_root_offset = m_extraNonceOffsetInTemplate + m_poolBlockTemplate->m_extraNonceSize + 2 + m_poolBlockTemplate->m_merkleTreeDataSize; + } else { + merkle_root_offset = 0; + } + + *pThis = this; + return m_blockTemplateBlob; } bool BlockTemplate::submit_sidechain_block(uint32_t template_id, uint32_t nonce, uint32_t extra_nonce) @@ -1578,6 +1773,20 @@ bool BlockTemplate::submit_sidechain_block(uint32_t template_id, uint32_t nonce, return false; } +hash BlockTemplate::calc_tx_merkle_root(uint32_t extra_nonce) const +{ + const hash miner_hash = calc_miner_tx_hash(extra_nonce); + const uint8_t* protocol_hash_ptr = m_transactionHashes.data() + HASH_SIZE; + + uint8_t combined[HASH_SIZE * 2]; + memcpy(combined, miner_hash.h, HASH_SIZE); + memcpy(combined + HASH_SIZE, protocol_hash_ptr, HASH_SIZE); + + hash result; + keccak(combined, HASH_SIZE * 2, result.h); + return result; +} + void BlockTemplate::init_merge_mining_merkle_proof() { const uint32_t n_aux_chains = static_cast(m_poolBlockTemplate->m_auxChains.size() + 1); diff --git a/src/block_template.h b/src/block_template.h index 5a5b873..a5d2392 100644 --- a/src/block_template.h +++ b/src/block_template.h @@ -75,10 +75,12 @@ private: private: void select_mempool_transactions(const Mempool& mempool); - int create_miner_tx(const MinerData& data, const std::vector& shares, uint64_t max_reward_amounts_weight, bool dry_run); + int create_miner_tx(const MinerData& data, const std::vector& shares, uint64_t max_reward_amounts_weight, bool dry_run, uint64_t full_block_reward); hash calc_sidechain_hash(uint32_t sidechain_extra_nonce) const; hash calc_miner_tx_hash(uint32_t extra_nonce) const; + hash calc_tx_merkle_root(uint32_t extra_nonce) const; void calc_merkle_tree_main_branch(); + hash m_protocolTxHash; uint32_t get_hashing_blob_nolock(uint32_t extra_nonce, uint8_t* blob) const; @@ -100,6 +102,7 @@ private: size_t m_numTransactionHashes; hash m_prevId; std::atomic m_height; + uint8_t m_majorVersion; difficulty_type m_difficulty; difficulty_type m_auxDifficulty; hash m_seedHash; @@ -115,10 +118,12 @@ private: #endif std::atomic m_finalReward; + std::atomic m_fullBlockReward; // Temp vectors, will be cleaned up after use and skipped in copy constructor/assignment operators std::vector m_minerTx; std::array m_minerTxKeccakState; + uint32_t m_minerTxPrefixSize; size_t m_minerTxKeccakStateInputLength; std::vector m_sidechainHashBlob; diff --git a/src/carrot_crypto.cpp b/src/carrot_crypto.cpp new file mode 100644 index 0000000..633141e --- /dev/null +++ b/src/carrot_crypto.cpp @@ -0,0 +1,423 @@ +#include "common.h" +#include "carrot_crypto.h" +#include "crypto.h" + +extern "C" { +#include "crypto-ops.h" +#include "mx25519.h" +#include "blake2/blake2.h" +} + +#include +#include +#include "log.h" + +namespace p2pool { +namespace carrot { + +// Domain separators (must match Salvium's carrot_core/config.h) +static constexpr const char DOMAIN_SEP_EPHEMERAL_PRIVKEY[] = "Carrot sending key normal"; +static constexpr const char DOMAIN_SEP_SENDER_RECEIVER_SECRET[] = "Carrot sender-receiver secret"; +static constexpr const char DOMAIN_SEP_VIEW_TAG[] = "Carrot view tag"; +static constexpr const char DOMAIN_SEP_ENCRYPTION_MASK_ANCHOR[] = "Carrot encryption mask anchor"; +static constexpr const char DOMAIN_SEP_ONETIME_EXTENSION_G[] = "Carrot key extension G"; +static constexpr const char DOMAIN_SEP_ONETIME_EXTENSION_T[] = "Carrot key extension T"; + +// T generator for Carrot (from Salvium generators.cpp) +static const uint8_t generator_T[32] = { + 0x96, 0x6f, 0xc6, 0x6b, 0x82, 0xcd, 0x56, 0xcf, + 0x85, 0xea, 0xec, 0x80, 0x1c, 0x42, 0x84, 0x5f, + 0x5f, 0x40, 0x88, 0x78, 0xd1, 0x56, 0x1e, 0x00, + 0xd3, 0xd7, 0xde, 0xd2, 0x79, 0x4d, 0x09, 0x4f +}; + +// H generator for Pedersen commitments (from RingCT) +static const uint8_t generator_H[32] = { + 0x8b, 0x65, 0x59, 0x70, 0x15, 0x37, 0x99, 0xaf, + 0x2a, 0xea, 0xdc, 0x9f, 0xf1, 0xad, 0xd0, 0xea, + 0x6c, 0x72, 0x51, 0xd5, 0x41, 0x54, 0xcf, 0xa9, + 0x2c, 0x17, 0x3a, 0x0d, 0xd3, 0x9c, 0x1f, 0x94 +}; + +// Curve order L for Ed25519 +static const uint8_t curve_order[32] = { + 0xed, 0xd3, 0xf5, 0x5c, 0x1a, 0x63, 0x12, 0x58, + 0xd6, 0x9c, 0xf7, 0xa2, 0xde, 0xf9, 0xde, 0x14, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10 +}; + +static const uint8_t identity_point[32] = { + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +}; + +// Get mx25519 implementation (cached) +static const mx25519_impl* get_mx25519_impl() +{ + static std::once_flag of; + static const mx25519_impl *impl; + std::call_once(of, [&](){ impl = mx25519_select_impl(MX25519_TYPE_AUTO); }); + return impl; +} + +// Build transcript: [1-byte length][domain_sep][args...] +// Returns total size written to buffer +template +static size_t build_transcript(uint8_t* buf, const char (&domain_sep)[N]) +{ + // N includes null terminator, actual length is N-1 + constexpr size_t len = N - 1; + static_assert(len <= 255, "Domain separator too long"); + buf[0] = static_cast(len); + memcpy(buf + 1, domain_sep, len); + return 1 + len; +} + +// Debug helper to print hex +static void debug_hex(const char* label, const uint8_t* data, size_t len) +{ + static constexpr char log_category_prefix[] = "CarrotCrypto "; + char hex[130] = {0}; + for (size_t i = 0; i < len && i < 64; ++i) { + sprintf(hex + i*2, "%02x", data[i]); + } + LOGINFO(0, label << ": " << static_cast(hex)); +} + +// BLAKE2b keyed hash wrapper +// key can be nullptr for unkeyed hash +static void blake2b_hash(void* out, size_t outlen, + const void* data, size_t datalen, + const void* key, size_t keylen) +{ + blake2b(out, outlen, data, datalen, key, keylen); +} + +static void derive_scalar(const void* transcript, size_t transcript_len, + const void* key, // 32 bytes or nullptr + uint8_t* scalar_out) +{ + uint8_t temp[64]; + blake2b_hash(temp, 64, transcript, transcript_len, + key, key ? 32 : 0); + sc_reduce(temp); // Reduce 64 bytes mod l + memcpy(scalar_out, temp, 32); +} + +// derive_bytes_32: H_32 (32-byte output, no reduction) +static void derive_bytes_32(const void* transcript, size_t transcript_len, + const void* key, // 32 bytes + uint8_t* out) +{ + blake2b_hash(out, 32, transcript, transcript_len, key, 32); +} + +// derive_bytes_3: H_3 (3-byte output) +static void derive_bytes_3(const void* transcript, size_t transcript_len, + const void* key, // 32 bytes + uint8_t* out) +{ + blake2b_hash(out, 3, transcript, transcript_len, key, 32); +} + +// derive_bytes_16: H_16 (16-byte output) +static void derive_bytes_16(const void* transcript, size_t transcript_len, + const void* key, // 32 bytes + uint8_t* out) +{ + blake2b_hash(out, 16, transcript, transcript_len, key, 32); +} + +void generate_janus_anchor(uint8_t (&anchor)[16]) +{ + hash tmp; + generate_keys(tmp, tmp); // Use existing random generation + memcpy(anchor, tmp.h, 16); +} + +void make_input_context_coinbase(uint64_t block_index, uint8_t (&input_context)[33]) +{ + // input_context = 'C' || block_index as little-endian 256-bit (32 bytes) + input_context[0] = 'C'; + memset(input_context + 1, 0, 32); + // Little-endian 64-bit at start of the 32-byte field + memcpy(input_context + 1, &block_index, sizeof(block_index)); + + debug_hex("input_context", input_context, 33); +} + +void make_ephemeral_privkey( + const uint8_t (&anchor)[16], + const uint8_t (&input_context)[33], + const hash& address_spend_pubkey, + const uint8_t (&payment_id)[8], + hash& ephemeral_privkey_out) +{ + // d_e = H_n("Carrot sending key normal" || anchor || input_context || K^j_s || pid) + // Transcript: [len][domain_sep][anchor:16][input_context:33][K_s:32][pid:8] + constexpr size_t domain_len = sizeof(DOMAIN_SEP_EPHEMERAL_PRIVKEY) - 1; // 25 + constexpr size_t transcript_size = 1 + domain_len + 16 + 33 + 32 + 8; // 115 + + uint8_t transcript[transcript_size]; + size_t offset = build_transcript(transcript, DOMAIN_SEP_EPHEMERAL_PRIVKEY); + memcpy(transcript + offset, anchor, 16); + offset += 16; + memcpy(transcript + offset, input_context, 33); + offset += 33; + memcpy(transcript + offset, address_spend_pubkey.h, 32); + offset += 32; + memcpy(transcript + offset, payment_id, 8); + + // Unkeyed hash (key = nullptr) + derive_scalar(transcript, transcript_size, nullptr, ephemeral_privkey_out.h); + + debug_hex("eph_priv anchor", anchor, 16); + debug_hex("eph_priv K_s", address_spend_pubkey.h, 32); + debug_hex("eph_priv d_e", ephemeral_privkey_out.h, 32); +} + +void make_ephemeral_pubkey_mainaddress( + const hash& ephemeral_privkey, + hash& ephemeral_pubkey_out) +{ + // D_e = d_e * B (X25519 scalar mult with base point) + mx25519_scmul_base(get_mx25519_impl(), + reinterpret_cast(ephemeral_pubkey_out.h), + reinterpret_cast(ephemeral_privkey.h)); + + debug_hex("eph_pub D_e", ephemeral_pubkey_out.h, 32); +} + +bool make_shared_secret_sender( + const hash& ephemeral_privkey, + const hash& address_view_pubkey, + hash& shared_secret_out) +{ + // First verify the view pubkey is valid and convert to X25519 + ge_p3 view_point; + if (ge_frombytes_vartime(&view_point, address_view_pubkey.h) != 0) { + return false; + } + + // Check if point is in prime order subgroup: l*P == identity + ge_p2 check_point; + ge_scalarmult(&check_point, curve_order, &view_point); + uint8_t check_bytes[32]; + ge_tobytes(check_bytes, &check_point); + if (memcmp(check_bytes, identity_point, 32) != 0) { + return false; // Point not in prime order subgroup + } + + // D^j_v = ConvertPointE(K^j_v) - convert Ed25519 pubkey to X25519 + mx25519_pubkey address_view_pubkey_x25519; + ge_p3_to_x25519(address_view_pubkey_x25519.data, &view_point); + + // s_sr = d_e * D^j_v (native X25519 scalar multiplication) + mx25519_scmul_key(get_mx25519_impl(), + reinterpret_cast(shared_secret_out.h), + reinterpret_cast(ephemeral_privkey.h), + &address_view_pubkey_x25519); + + debug_hex("shared K_v_ed25519", address_view_pubkey.h, 32); + debug_hex("shared K_v_x25519", address_view_pubkey_x25519.data, 32); + debug_hex("shared s_sr_unctx", shared_secret_out.h, 32); + return true; +} + +void make_sender_receiver_secret( + const hash& shared_secret_unctx, + const hash& ephemeral_pubkey, + const uint8_t (&input_context)[33], + hash& sender_receiver_out) +{ + // s^ctx_sr = H_32[s_sr]("Carrot sender-receiver secret" || D_e || input_context) + // Transcript: [len][domain_sep][D_e:32][input_context:33] + constexpr size_t domain_len = sizeof(DOMAIN_SEP_SENDER_RECEIVER_SECRET) - 1; // 29 + constexpr size_t transcript_size = 1 + domain_len + 32 + 33; // 95 + + uint8_t transcript[transcript_size]; + size_t offset = build_transcript(transcript, DOMAIN_SEP_SENDER_RECEIVER_SECRET); + memcpy(transcript + offset, ephemeral_pubkey.h, 32); + offset += 32; + memcpy(transcript + offset, input_context, 33); + + // Keyed with shared_secret_unctx + derive_bytes_32(transcript, transcript_size, shared_secret_unctx.h, sender_receiver_out.h); + + debug_hex("s_sr_ctx result", sender_receiver_out.h, 32); +} + +void make_onetime_address_coinbase( + const hash& address_spend_pubkey, + const hash& sender_receiver_secret, + uint64_t amount, + hash& onetime_address_out) +{ + // For coinbase: K_o = K^j_s + k^o_g * G + k^o_t * T + // k^o_g = H_n[s^ctx_sr]("Carrot key extension G" || C_a) + // k^o_t = H_n[s^ctx_sr]("Carrot key extension T" || C_a) + // C_a = 1*G + amount*H (coinbase uses k_a = 1) + + // 1. Compute C_a = 1*G + amount*H + // First: 1*G + ge_p3 one_G; + { + uint8_t one_scalar[32] = {0}; + one_scalar[0] = 1; + ge_scalarmult_base(&one_G, one_scalar); + } + + // Second: amount*H + ge_p3 H_point; + if (ge_frombytes_vartime(&H_point, generator_H) != 0) { + memcpy(onetime_address_out.h, address_spend_pubkey.h, 32); + return; + } + + ge_p2 amount_H_p2; + uint8_t amount_le[32] = {0}; + memcpy(amount_le, &amount, sizeof(amount)); // little-endian + ge_scalarmult(&amount_H_p2, amount_le, &H_point); + + // Convert amount_H to p3 + uint8_t amount_H_bytes[32]; + ge_tobytes(amount_H_bytes, &amount_H_p2); + ge_p3 amount_H; + ge_frombytes_vartime(&amount_H, amount_H_bytes); + + // C_a = 1*G + amount*H + ge_cached one_G_cached; + ge_p3_to_cached(&one_G_cached, &one_G); + ge_p1p1 C_a_p1p1; + ge_add(&C_a_p1p1, &amount_H, &one_G_cached); + ge_p3 C_a_p3; + ge_p1p1_to_p3(&C_a_p3, &C_a_p1p1); + uint8_t C_a[32]; + ge_p3_tobytes(C_a, &C_a_p3); + + debug_hex("K_o C_a commitment", C_a, 32); + + // 2. k^o_g = H_n[s^ctx_sr]("Carrot key extension G" || C_a) + constexpr size_t domain_len_g = sizeof(DOMAIN_SEP_ONETIME_EXTENSION_G) - 1; + constexpr size_t transcript_size_g = 1 + domain_len_g + 32; + uint8_t transcript_g[transcript_size_g]; + size_t offset_g = build_transcript(transcript_g, DOMAIN_SEP_ONETIME_EXTENSION_G); + memcpy(transcript_g + offset_g, C_a, 32); + + uint8_t k_o_g[32]; + derive_scalar(transcript_g, transcript_size_g, sender_receiver_secret.h, k_o_g); + debug_hex("K_o k^o_g scalar", k_o_g, 32); + + // 3. k^o_t = H_n[s^ctx_sr]("Carrot key extension T" || C_a) + constexpr size_t domain_len_t = sizeof(DOMAIN_SEP_ONETIME_EXTENSION_T) - 1; + constexpr size_t transcript_size_t = 1 + domain_len_t + 32; + uint8_t transcript_t[transcript_size_t]; + size_t offset_t = build_transcript(transcript_t, DOMAIN_SEP_ONETIME_EXTENSION_T); + memcpy(transcript_t + offset_t, C_a, 32); + + uint8_t k_o_t[32]; + derive_scalar(transcript_t, transcript_size_t, sender_receiver_secret.h, k_o_t); + debug_hex("K_o k^o_t scalar", k_o_t, 32); + + // 4. K^o_ext = k^o_g * G + k^o_t * T + // First: k^o_g * G + ge_p3 k_o_g_G; + ge_scalarmult_base(&k_o_g_G, k_o_g); + + // Second: k^o_t * T + ge_p3 T_point; + if (ge_frombytes_vartime(&T_point, generator_T) != 0) { + memcpy(onetime_address_out.h, address_spend_pubkey.h, 32); + return; + } + ge_p2 k_o_t_T_p2; + ge_scalarmult(&k_o_t_T_p2, k_o_t, &T_point); + uint8_t k_o_t_T_bytes[32]; + ge_tobytes(k_o_t_T_bytes, &k_o_t_T_p2); + ge_p3 k_o_t_T; + ge_frombytes_vartime(&k_o_t_T, k_o_t_T_bytes); + + // K^o_ext = k^o_g*G + k^o_t*T + ge_cached k_o_g_G_cached; + ge_p3_to_cached(&k_o_g_G_cached, &k_o_g_G); + ge_p1p1 ext_p1p1; + ge_add(&ext_p1p1, &k_o_t_T, &k_o_g_G_cached); + ge_p3 extension_point; + ge_p1p1_to_p3(&extension_point, &ext_p1p1); + + // 5. K_o = K^j_s + K^o_ext + ge_p3 spend_point; + if (ge_frombytes_vartime(&spend_point, address_spend_pubkey.h) != 0) { + memcpy(onetime_address_out.h, address_spend_pubkey.h, 32); + return; + } + + ge_cached extension_cached; + ge_p3_to_cached(&extension_cached, &extension_point); + + ge_p1p1 result_p1p1; + ge_add(&result_p1p1, &spend_point, &extension_cached); + + ge_p3 result; + ge_p1p1_to_p3(&result, &result_p1p1); + ge_p3_tobytes(onetime_address_out.h, &result); + + debug_hex("K_o result", onetime_address_out.h, 32); +} + +void make_view_tag( + const hash& shared_secret_unctx, + const uint8_t (&input_context)[33], + const hash& onetime_address, + uint8_t (&view_tag)[3]) +{ + // vt = H_3[s_sr]("Carrot view tag" || input_context || K_o) + // Transcript: [len][domain_sep][input_context:33][K_o:32] + constexpr size_t domain_len = sizeof(DOMAIN_SEP_VIEW_TAG) - 1; // 15 + constexpr size_t transcript_size = 1 + domain_len + 33 + 32; // 81 + + uint8_t transcript[transcript_size]; + size_t offset = build_transcript(transcript, DOMAIN_SEP_VIEW_TAG); + memcpy(transcript + offset, input_context, 33); + offset += 33; + memcpy(transcript + offset, onetime_address.h, 32); + + // Keyed with shared_secret_unctx (NOT contextualized s_sr) + derive_bytes_3(transcript, transcript_size, shared_secret_unctx.h, view_tag); + + debug_hex("view_tag result", view_tag, 3); +} + +void encrypt_anchor( + const uint8_t (&anchor)[16], + const hash& sender_receiver_secret, + const hash& onetime_address, + uint8_t (&encrypted_anchor)[16]) +{ + // anchor_enc = anchor XOR H_16[s^ctx_sr]("Carrot encryption mask anchor" || K_o) + // Transcript: [len][domain_sep][K_o:32] + constexpr size_t domain_len = sizeof(DOMAIN_SEP_ENCRYPTION_MASK_ANCHOR) - 1; // 29 + constexpr size_t transcript_size = 1 + domain_len + 32; // 62 + + uint8_t transcript[transcript_size]; + size_t offset = build_transcript(transcript, DOMAIN_SEP_ENCRYPTION_MASK_ANCHOR); + memcpy(transcript + offset, onetime_address.h, 32); + + // Keyed with sender_receiver_secret (contextualized) + uint8_t mask[16]; + derive_bytes_16(transcript, transcript_size, sender_receiver_secret.h, mask); + + for (size_t i = 0; i < 16; ++i) { + encrypted_anchor[i] = anchor[i] ^ mask[i]; + } + + debug_hex("anchor mask", mask, 16); + debug_hex("anchor encrypted", encrypted_anchor, 16); +} + +} // namespace carrot +} // namespace p2pool + diff --git a/src/carrot_crypto.h b/src/carrot_crypto.h new file mode 100644 index 0000000..fc53374 --- /dev/null +++ b/src/carrot_crypto.h @@ -0,0 +1,66 @@ + + +#pragma once + +#include "common.h" + +namespace p2pool { +namespace carrot { + +// Generate random 16-byte janus anchor +void generate_janus_anchor(uint8_t (&anchor)[16]); + +// Create coinbase input context: "C" || block_index (as 256-bit little-endian) +void make_input_context_coinbase(uint64_t block_index, uint8_t (&input_context)[33]); + +// Generate ephemeral private key: d_e = H_n(anchor, input_context, K^j_s, pid) +void make_ephemeral_privkey( + const uint8_t (&anchor)[16], + const uint8_t (&input_context)[33], + const hash& address_spend_pubkey, + const uint8_t (&payment_id)[8], + hash& ephemeral_privkey_out); + +// Generate ephemeral public key: D_e = d_e * G (main address) +void make_ephemeral_pubkey_mainaddress( + const hash& ephemeral_privkey, + hash& ephemeral_pubkey_out); + +// Generate uncontextualized shared secret (sender side): s_sr = d_e * ConvertPointE(K^j_v) +bool make_shared_secret_sender( + const hash& ephemeral_privkey, + const hash& address_view_pubkey, + hash& shared_secret_out); + +// Generate contextualized sender-receiver secret: s^ctx_sr = H_32(s_sr, D_e, input_context) +void make_sender_receiver_secret( + const hash& shared_secret_unctx, + const hash& ephemeral_pubkey, + const uint8_t (&input_context)[33], + hash& sender_receiver_out); + +// Generate onetime address: K_o = K^j_s + (k^o_g * G + k^o_t * T) +// For coinbase with dummy commitment, T point is handled specially +void make_onetime_address_coinbase( + const hash& address_spend_pubkey, + const hash& sender_receiver_secret, + uint64_t amount, + hash& onetime_address_out); + +// Generate 3-byte view tag: vt = H_3(s_sr, input_context, K_o) +void make_view_tag( + const hash& shared_secret_unctx, + const uint8_t (&input_context)[33], + const hash& onetime_address, + uint8_t (&view_tag)[3]); + +// Encrypt janus anchor: anchor_enc = anchor XOR H_16(s^ctx_sr, K_o) +void encrypt_anchor( + const uint8_t (&anchor)[16], + const hash& sender_receiver_secret, + const hash& onetime_address, + uint8_t (&encrypted_anchor)[16]); + +} // namespace carrot +} // namespace p2pool + diff --git a/src/common.h b/src/common.h index a18b88f..14f8a57 100644 --- a/src/common.h +++ b/src/common.h @@ -124,9 +124,10 @@ constexpr uint8_t EXTRA_NONCE_SIZE = 4; constexpr uint8_t EXTRA_NONCE_MAX_SIZE = EXTRA_NONCE_SIZE + 10; constexpr uint8_t TX_VERSION = 4; constexpr uint8_t TXIN_GEN = 0xFF; -constexpr uint8_t TXOUT_TO_TAGGED_KEY = 3; +constexpr uint8_t TXOUT_TO_TAGGED_KEY = 4; // Changed from 3 to 4 for Carrot v1 constexpr uint8_t TXOUT_TO_CARROT_V1 = 4; constexpr uint8_t TX_EXTRA_TAG_PUBKEY = 1; +constexpr uint8_t TX_EXTRA_TAG_ADDITIONAL_PUBKEYS = 4; constexpr uint8_t TX_EXTRA_NONCE = 2; constexpr uint8_t TX_EXTRA_MERGE_MINING_TAG = 3; @@ -552,7 +553,7 @@ struct MinerData std::vector aux_chains; uint32_t aux_nonce; - + std::vector protocol_tx_identifier; std::chrono::high_resolution_clock::time_point time_received; }; diff --git a/src/p2pool.cpp b/src/p2pool.cpp index 7ed3903..4764de4 100644 --- a/src/p2pool.cpp +++ b/src/p2pool.cpp @@ -1148,23 +1148,30 @@ void p2pool::submit_block() const request = "{\"jsonrpc\":\"2.0\",\"id\":\"0\",\"method\":\"submit_block\",\"params\":[\""; - const uint32_t template_id = submit_data.template_id; - const uint32_t nonce = submit_data.nonce; - const uint32_t extra_nonce = submit_data.extra_nonce; + const uint32_t template_id = submit_data.template_id; + const uint32_t nonce = submit_data.nonce; + const uint32_t extra_nonce = submit_data.extra_nonce; - std::vector blob; - blob.reserve(submit_data.blob.size()); + std::vector blob; + blob.reserve(submit_data.blob.size()); - for (size_t i = 0; i < submit_data.blob.size(); ++i) { - uint8_t b; - if (nonce_offset && nonce_offset <= i && i < nonce_offset + sizeof(submit_data.nonce)) { - b = submit_data.nonce & 255; - submit_data.nonce >>= 8; - } - else if (extra_nonce_offset && extra_nonce_offset <= i && i < extra_nonce_offset + sizeof(submit_data.extra_nonce)) { - b = submit_data.extra_nonce & 255; - submit_data.extra_nonce >>= 8; - } + LOGINFO(0, "DEBUG submit_block offsets: nonce_offset=" << nonce_offset + << ", extra_nonce_offset=" << extra_nonce_offset + << ", merkle_root_offset=" << merkle_root_offset + << ", blob_size=" << submit_data.blob.size()); + + for (size_t i = 0; i < submit_data.blob.size(); ++i) { + uint8_t b; + if (nonce_offset && nonce_offset <= i && i < nonce_offset + sizeof(nonce)) { + b = (nonce >> ((i - nonce_offset) * 8)) & 255; + } + else if (extra_nonce_offset && extra_nonce_offset <= i && i < extra_nonce_offset + sizeof(extra_nonce)) { + b = (extra_nonce >> ((i - extra_nonce_offset) * 8)) & 255; + if (i == extra_nonce_offset) { + LOGINFO(0, "DEBUG: Patching extra_nonce at offset " << i << ", value=" << extra_nonce); + } + + } else if (merkle_root_offset && merkle_root_offset <= i && i < merkle_root_offset + HASH_SIZE) { b = merge_mining_root.h[i - merkle_root_offset]; } @@ -1179,6 +1186,10 @@ void p2pool::submit_block() const } request.append("\"]}"); + // DEBUG: Show what we're actually sending + std::string sent_blob_hex = request.substr(request.find("\":[\"")+4, 400); + LOGINFO(0, "DEBUG: First 200 bytes of hex being sent: " << sent_blob_hex.substr(0, 400)); + hash digest; sha256(blob.data(), static_cast(blob.size()), digest.h); diff --git a/src/p2pool.h b/src/p2pool.h index 113d487..d83c923 100644 --- a/src/p2pool.h +++ b/src/p2pool.h @@ -193,6 +193,8 @@ private: void get_miner_data(bool retry = true); void parse_get_miner_data_rpc(const char* data, size_t size); + void fetch_block_template(MinerData& data); + void parse_block_template_rpc(const char* data, size_t size, MinerData& miner_data); bool parse_block_header(const char* data, size_t size, ChainMain& c); uint32_t parse_block_headers_range(const char* data, size_t size); diff --git a/src/pool_block.cpp b/src/pool_block.cpp index 9d37b06..36efc82 100644 --- a/src/pool_block.cpp +++ b/src/pool_block.cpp @@ -20,6 +20,7 @@ #include "keccak.h" #include "side_chain.h" #include "pow_hash.h" +#include "protocol_tx_hash.h" #include "crypto.h" #include "merkle.h" @@ -219,12 +220,31 @@ std::vector PoolBlock::serialize_mainchain_data(size_t* header_size, si memcpy(p, m_merkleRoot.h, HASH_SIZE); p += HASH_SIZE; - writeVarint(static_cast(p - tx_extra), data); - data.insert(data.end(), tx_extra, p); + writeVarint(static_cast(p - tx_extra), data); + data.insert(data.end(), tx_extra, p); - data.push_back(0); + // For Carrot v1+ (major_version >= 10), add type and amount_burnt instead of vin_rct_type + if (m_majorVersion >= 10) { + // type = MINER + writeVarint(1, data); + + // amount_burnt = 20% of total reward + uint64_t miner_total = 0; + for (const TxOutput& output : m_outputAmounts) { + miner_total += output.m_reward; + } + uint64_t stake_amount = miner_total / 4; + writeVarint(stake_amount, data); - if (miner_tx_size) { + data.push_back(0); + + } + else { + // vin_rct_type (only for legacy transactions) + data.push_back(0); + } + + if (miner_tx_size) { *miner_tx_size = data.size() - header_size0; } @@ -248,12 +268,14 @@ std::vector PoolBlock::serialize_mainchain_data(size_t* header_size, si // type = PROTOCOL writeVarint(2, data); // transaction_type::PROTOCOL = 2 + data.push_back(0); // RCT - // rct_signatures (null) - data.push_back(0); // RCTTypeNull + // Calculate protocol tx hash using salviumd-compatible serialization + hash protocol_tx_hash; + calculate_protocol_tx_hash(m_txinGenHeight, protocol_tx_hash); + LOGINFO(3, "Sidechain protocol TX hash: " << protocol_tx_hash); } - - writeVarint(m_transactions.size() - 1, data); + writeVarint(m_transactions.size() - 1, data); #ifdef WITH_INDEXED_HASHES for (size_t i = 1, n = m_transactions.size(); i < n; ++i) { diff --git a/src/pool_block.h b/src/pool_block.h index 7497e0c..5e6c255 100644 --- a/src/pool_block.h +++ b/src/pool_block.h @@ -116,6 +116,8 @@ struct PoolBlock std::vector m_ephPublicKeys; std::vector m_outputAmounts; + std::vector> m_viewTags; + std::vector> m_encryptedAnchors; hash m_txkeyPub; uint64_t m_extraNonceSize; diff --git a/src/protocol_tx_hash.cpp b/src/protocol_tx_hash.cpp new file mode 100644 index 0000000..d10ff5b --- /dev/null +++ b/src/protocol_tx_hash.cpp @@ -0,0 +1,50 @@ +#include "protocol_tx_hash.h" +#include "keccak.h" +#include +#include + +namespace p2pool { + +// Minimal varint encoder matching salviumd's format +static void write_varint(uint64_t value, std::vector& dest) { + while (value >= 0x80) { + dest.push_back(static_cast((value & 0x7f) | 0x80)); + value >>= 7; + } + dest.push_back(static_cast(value)); +} + +void calculate_protocol_tx_hash(uint64_t height, hash& result) { + std::vector prefix_serialized; + prefix_serialized.reserve(32); + + // Serialize transaction_prefix (everything except RCT type byte) + write_varint(4, prefix_serialized); // version + write_varint(60, prefix_serialized); // unlock_time + write_varint(1, prefix_serialized); // vin.size() + prefix_serialized.push_back(0xff); // TXIN_GEN + write_varint(height, prefix_serialized); // gen.height + write_varint(0, prefix_serialized); // vout.size() + write_varint(2, prefix_serialized); // extra.size() + prefix_serialized.push_back(0x02); // extra[0] + prefix_serialized.push_back(0x00); // extra[1] + write_varint(2, prefix_serialized); // type = PROTOCOL + + // Hash the three components as salviumd does for v2+ transactions + hash prefix_hash, base_rct_hash; + uint8_t rct_type = 0; + + keccak(prefix_serialized.data(), prefix_serialized.size(), prefix_hash.h); + keccak(&rct_type, 1, base_rct_hash.h); + + // Combine: prefix_hash + base_rct_hash + null_hash (32 zeros) + uint8_t combined[HASH_SIZE * 3]; + memcpy(combined, prefix_hash.h, HASH_SIZE); + memcpy(combined + HASH_SIZE, base_rct_hash.h, HASH_SIZE); + memset(combined + HASH_SIZE * 2, 0, HASH_SIZE); // null hash for RCTTypeNull + + keccak(combined, sizeof(combined), result.h); +} + +} // namespace p2pool + diff --git a/src/protocol_tx_hash.h b/src/protocol_tx_hash.h new file mode 100644 index 0000000..9a29ff8 --- /dev/null +++ b/src/protocol_tx_hash.h @@ -0,0 +1,10 @@ +#pragma once + +#include "common.h" +#include + +namespace p2pool { + +void calculate_protocol_tx_hash(uint64_t height, hash& result); + +} // namespace p2pool diff --git a/src/wallet.cpp b/src/wallet.cpp index 7740675..e76dc03 100644 --- a/src/wallet.cpp +++ b/src/wallet.cpp @@ -121,7 +121,7 @@ bool Wallet::decode(const char* address) } const size_t addr_len = strlen(address); - if (addr_len < 97 || addr_len > 98) { + if (addr_len < 94 || addr_len > 140) { return false; } @@ -199,6 +199,24 @@ bool Wallet::decode(const char* address) memcpy(m_spendPublicKey.h, data + varint_len, HASH_SIZE); memcpy(m_viewPublicKey.h, data + varint_len + HASH_SIZE, HASH_SIZE); + // DEBUG: Print what we decoded + { + static constexpr char log_category_prefix[] = "Wallet "; + char spend_hex[65] = {0}, view_hex[65] = {0}, data_hex[200] = {0}, prefix_hex[20] = {0}; + for (int i = 0; i < 32; i++) { + sprintf(spend_hex + i*2, "%02x", m_spendPublicKey.h[i]); + sprintf(view_hex + i*2, "%02x", m_viewPublicKey.h[i]); + } + for (int i = 0; i < std::min(data_index, 80); i++) { + sprintf(data_hex + i*2, "%02x", data[i]); + } + sprintf(prefix_hex, "0x%lx", m_prefix); + LOGINFO(0, "decode varint_len=" << varint_len << " data_index=" << data_index << " prefix=" << static_cast(prefix_hex)); + LOGINFO(0, "decode raw_data: " << static_cast(data_hex)); + LOGINFO(0, "decode spend: " << static_cast(spend_hex)); + LOGINFO(0, "decode view: " << static_cast(view_hex)); + } + // Load checksum from correct position (at end of decoded data) memcpy(&m_checksum, data + data_index - sizeof(m_checksum), sizeof(m_checksum));