Optimized division by constants

makefile

@@ -11,7 +11,7 @@ SRCDIR=src
 OBJDIR=obj
 LDFLAGS=-lpthread
 TOBJS=$(addprefix $(OBJDIR)/,instructionsPortable.o TestAluFpu.o)
-ROBJS=$(addprefix $(OBJDIR)/,argon2_core.o argon2_ref.o AssemblyGeneratorX86.o blake2b.o CompiledVirtualMachine.o dataset.o JitCompilerX86.o instructionsPortable.o Instruction.o InterpretedVirtualMachine.o main.o Program.o softAes.o VirtualMachine.o t1ha2.o Cache.o virtualMemory.o)
+ROBJS=$(addprefix $(OBJDIR)/,argon2_core.o argon2_ref.o AssemblyGeneratorX86.o blake2b.o CompiledVirtualMachine.o dataset.o JitCompilerX86.o instructionsPortable.o Instruction.o InterpretedVirtualMachine.o main.o Program.o softAes.o VirtualMachine.o t1ha2.o Cache.o virtualMemory.o divideByConstantCodegen.o)
 ifeq ($(PLATFORM),x86_64)
     ROBJS += $(OBJDIR)/JitCompilerX86-static.o
 endif
@@ -57,6 +57,9 @@ $(OBJDIR)/CompiledVirtualMachine.o: $(addprefix $(SRCDIR)/,CompiledVirtualMachin
 $(OBJDIR)/dataset.o: $(addprefix $(SRCDIR)/,dataset.cpp common.hpp Pcg32.hpp) | $(OBJDIR)
 	$(CXX) $(CXXFLAGS) -c $(SRCDIR)/dataset.cpp -o $@
 
+$(OBJDIR)/divideByConstantCodegen.o: $(addprefix $(SRCDIR)/,divideByConstantCodegen.c divideByConstantCodegen.h) | $(OBJDIR)
+	$(CC) $(CCFLAGS) -c $(SRCDIR)/divideByConstantCodegen.c -o $@
+
 $(OBJDIR)/JitCompilerX86.o: $(addprefix $(SRCDIR)/,JitCompilerX86.cpp JitCompilerX86.hpp Instruction.hpp instructionWeights.hpp) | $(OBJDIR)
 	$(CXX) $(CXXFLAGS) -c $(SRCDIR)/JitCompilerX86.cpp -o $@
 

src/AssemblyGeneratorX86.cpp

@@ -17,10 +17,14 @@ You should have received a copy of the GNU General Public License
 along with RandomX.  If not, see<http://www.gnu.org/licenses/>.
 */
 //#define TRACE
+//#define MAGIC_DIVISION
 #include "AssemblyGeneratorX86.hpp"
 #include "Pcg32.hpp"
 #include "common.hpp"
 #include "instructions.hpp"
+#ifdef MAGIC_DIVISION
+#include "divideByConstantCodegen.h"
+#endif
 
 namespace RandomX {
 
@@ -315,34 +319,118 @@ namespace RandomX {
 	void AssemblyGeneratorX86::h_DIV_64(Instruction& instr, int i) {
 		genar(instr, i);
 		if ((instr.locb & 7) >= 6) {
+#ifdef MAGIC_DIVISION
+			if (instr.imm32 != 0) {
+				uint32_t divisor = instr.imm32;
+				asmCode << "\t; magic divide by " << divisor << std::endl;
+				if (divisor & (divisor - 1)) {
+					magicu_info mi = compute_unsigned_magic_info(divisor, sizeof(uint64_t) * 8);
+					if (mi.pre_shift > 0)
+						asmCode << "\tshr rax, " << mi.pre_shift << std::endl;
+					if (mi.increment) {
+						asmCode << "\tadd rax, 1" << std::endl;
+						asmCode << "\tsbb rax, 0" << std::endl;
+					}
+					asmCode << "\tmov rcx, " << mi.multiplier << std::endl;
+					asmCode << "\tmul rcx" << std::endl;
+					asmCode << "\tmov rax, rdx" << std::endl;
+					if (mi.post_shift > 0)
+						asmCode << "\tshr rax, " << mi.post_shift << std::endl;
+				}
+				else { //divisor is a power of two
+					int shift = 0;
+					while (divisor >>= 1)
+						++shift;
+					if(shift > 0)
+						asmCode << "\tshr rax, " << shift << std::endl;
+				}
+			}
+#else
 			if (instr.imm32 == 0) {
 				asmCode << "\tmov ecx, 1" << std::endl;
 			}
 			else {
 				asmCode << "\tmov ecx, " << instr.imm32 << std::endl;
 			}
+#endif
 		}
 		else {
 			asmCode << "\tmov ecx, 1" << std::endl;
 			asmCode << "\tmov edx, " << regR32[instr.regb % RegistersCount] << std::endl;
 			asmCode << "\ttest edx, edx" << std::endl;
 			asmCode << "\tcmovne ecx, edx" << std::endl;
+#ifdef MAGIC_DIVISION
+			asmCode << "\txor edx, edx" << std::endl;
+			asmCode << "\tdiv rcx" << std::endl;
+#endif
 		}
+#ifndef MAGIC_DIVISION
 		asmCode << "\txor edx, edx" << std::endl;
 		asmCode << "\tdiv rcx" << std::endl;
+#endif
 		gencr(instr);
 	}
 
 	void AssemblyGeneratorX86::h_IDIV_64(Instruction& instr, int i) {
 		genar(instr, i);
+#ifdef MAGIC_DIVISION
+		if ((instr.locb & 7) >= 6) {
+			int64_t divisor = instr.imm32;
+			asmCode << "\t; magic divide by " << divisor << std::endl;
+			if ((divisor & -divisor) == divisor || (divisor & -divisor) == -divisor) {
+				// +/- power of two
+				bool negative = divisor < 0;
+				if (negative)
+					divisor = -divisor;
+				int shift = 0;
+				uint64_t unsignedDivisor = divisor;
+				while (unsignedDivisor >>= 1)
+					++shift;
+				if (shift > 0) {
+					asmCode << "\tmov rcx, rax" << std::endl;
+					asmCode << "\tsar rcx, 63" << std::endl;
+					uint32_t mask = (1ULL << shift) + 0xFFFFFFFF;
+					asmCode << "\tand ecx, 0" << std::hex << mask << std::dec << "h" << std::endl;
+					asmCode << "\tadd rax, rcx" << std::endl;
+					asmCode << "\tsar rax, " << shift << std::endl;
+				}
+				if(negative)
+					asmCode << "\tneg rax" << std::endl;
+			} else if(divisor != 0) {
+				magics_info mi = compute_signed_magic_info(divisor);
+				if ((divisor >= 0) != (mi.multiplier >= 0))
+					asmCode << "\tmov rcx, rax" << std::endl;
+				asmCode << "\tmov rdx, " << mi.multiplier << std::endl;
+				asmCode << "\timul rdx" << std::endl;
+				asmCode << "\tmov rax, rdx" << std::endl;
+				asmCode << "\txor edx, edx" << std::endl;
+				bool haveSF = false;
+				if (divisor > 0 && mi.multiplier < 0) {
+					asmCode << "\tadd rax, rcx" << std::endl;
+					haveSF = true;
+				}
+				if (divisor < 0 && mi.multiplier > 0) {
+					asmCode << "\tsub rax, rcx" << std::endl;
+					haveSF = true;
+				}
+				if (mi.shift > 0) {
+					asmCode << "\tsar rax, " << mi.shift << std::endl;
+					haveSF = true;
+				}
+				if (!haveSF)
+					asmCode << "\ttest rax, rax" << std::endl;
+				asmCode << "\tsets dl" << std::endl;
+				asmCode << "\tadd rax, rdx" << std::endl;
+			}
+		}
+		else {
+#endif
 		asmCode << "\tmov edx, ";
 		genbr132(instr);
 		asmCode << "\tcmp edx, -1" << std::endl;
 		asmCode << "\tjne short safe_idiv_" << i << std::endl;
-		asmCode << "\tmov rcx, rax" << std::endl;
-		asmCode << "\trol rcx, 1" << std::endl;
-		asmCode << "\tdec rcx" << std::endl;
-		asmCode << "\tjz short result_idiv_" << i << std::endl;
+		asmCode << "\tneg rax" << std::endl;
+		asmCode << "\tjmp short result_idiv_" << i << std::endl;
 		asmCode << "safe_idiv_" << i << ":" << std::endl;
 		asmCode << "\tmov ecx, 1" << std::endl;
 		asmCode << "\ttest edx, edx" << std::endl;
@@ -351,6 +439,9 @@ namespace RandomX {
 		asmCode << "\tcqo" << std::endl;
 		asmCode << "\tidiv rcx" << std::endl;
 		asmCode << "result_idiv_" << i << ":" << std::endl;
+#ifdef MAGIC_DIVISION
+		}
+#endif
 		gencr(instr);
 	}
 

src/divideByConstantCodegen.c

@@ -0,0 +1,169 @@
+/*
+  Reference implementations of computing and using the "magic number" approach to dividing
+  by constants, including codegen instructions. The unsigned division incorporates the
+  "round down" optimization per ridiculous_fish.
+
+  This is free and unencumbered software. Any copyright is dedicated to the Public Domain.
+*/
+
+#include <limits.h> //for CHAR_BIT
+#include <assert.h>
+
+#include "divideByConstantCodegen.h"
+
+struct magicu_info compute_unsigned_magic_info(uint D, unsigned num_bits) {
+
+	//The numerator must fit in a uint
+	assert(num_bits > 0 && num_bits <= sizeof(uint) * CHAR_BIT);
+
+	// D must be larger than zero and not a power of 2
+	assert(D & (D - 1));
+
+	// The eventual result
+	struct magicu_info result;
+
+	// Bits in a uint
+	const unsigned UINT_BITS = sizeof(uint) * CHAR_BIT;
+
+	// The extra shift implicit in the difference between UINT_BITS and num_bits
+	const unsigned extra_shift = UINT_BITS - num_bits;
+
+	// The initial power of 2 is one less than the first one that can possibly work
+	const uint initial_power_of_2 = (uint)1 << (UINT_BITS - 1);
+
+	// The remainder and quotient of our power of 2 divided by d
+	uint quotient = initial_power_of_2 / D, remainder = initial_power_of_2 % D;
+
+	// ceil(log_2 D)
+	unsigned ceil_log_2_D;
+
+	// The magic info for the variant "round down" algorithm
+	uint down_multiplier = 0;
+	unsigned down_exponent = 0;
+	int has_magic_down = 0;
+
+	// Compute ceil(log_2 D)
+	ceil_log_2_D = 0;
+	uint tmp;
+	for (tmp = D; tmp > 0; tmp >>= 1)
+		ceil_log_2_D += 1;
+
+
+	// Begin a loop that increments the exponent, until we find a power of 2 that works.
+	unsigned exponent;
+	for (exponent = 0; ; exponent++) {
+		// Quotient and remainder is from previous exponent; compute it for this exponent.
+		if (remainder >= D - remainder) {
+			// Doubling remainder will wrap around D
+			quotient = quotient * 2 + 1;
+			remainder = remainder * 2 - D;
+		}
+		else {
+			// Remainder will not wrap
+			quotient = quotient * 2;
+			remainder = remainder * 2;
+		}
+
+		// We're done if this exponent works for the round_up algorithm.
+		// Note that exponent may be larger than the maximum shift supported,
+		// so the check for >= ceil_log_2_D is critical.
+		if ((exponent + extra_shift >= ceil_log_2_D) || (D - remainder) <= ((uint)1 << (exponent + extra_shift)))
+			break;
+
+		// Set magic_down if we have not set it yet and this exponent works for the round_down algorithm
+		if (!has_magic_down && remainder <= ((uint)1 << (exponent + extra_shift))) {
+			has_magic_down = 1;
+			down_multiplier = quotient;
+			down_exponent = exponent;
+		}
+	}
+
+	if (exponent < ceil_log_2_D) {
+		// magic_up is efficient
+		result.multiplier = quotient + 1;
+		result.pre_shift = 0;
+		result.post_shift = exponent;
+		result.increment = 0;
+	}
+	else if (D & 1) {
+		// Odd divisor, so use magic_down, which must have been set
+		assert(has_magic_down);
+		result.multiplier = down_multiplier;
+		result.pre_shift = 0;
+		result.post_shift = down_exponent;
+		result.increment = 1;
+	}
+	else {
+		// Even divisor, so use a prefix-shifted dividend
+		unsigned pre_shift = 0;
+		uint shifted_D = D;
+		while ((shifted_D & 1) == 0) {
+			shifted_D >>= 1;
+			pre_shift += 1;
+		}
+		result = compute_unsigned_magic_info(shifted_D, num_bits - pre_shift);
+		assert(result.increment == 0 && result.pre_shift == 0); //expect no increment or pre_shift in this path
+		result.pre_shift = pre_shift;
+	}
+	return result;
+}
+
+struct magics_info compute_signed_magic_info(sint D) {
+	// D must not be zero and must not be a power of 2 (or its negative)
+	assert(D != 0 && (D & -D) != D && (D & -D) != -D);
+
+	// Our result
+	struct magics_info result;
+
+	// Bits in an sint
+	const unsigned SINT_BITS = sizeof(sint) * CHAR_BIT;
+
+	// Absolute value of D (we know D is not the most negative value since that's a power of 2)
+	const uint abs_d = (D < 0 ? -D : D);
+
+	// The initial power of 2 is one less than the first one that can possibly work
+	// "two31" in Warren
+	unsigned exponent = SINT_BITS - 1;
+	const uint initial_power_of_2 = (uint)1 << exponent;
+
+	// Compute the absolute value of our "test numerator,"
+	// which is the largest dividend whose remainder with d is d-1.
+	// This is called anc in Warren.
+	const uint tmp = initial_power_of_2 + (D < 0);
+	const uint abs_test_numer = tmp - 1 - tmp % abs_d;
+
+	// Initialize our quotients and remainders (q1, r1, q2, r2 in Warren)
+	uint quotient1 = initial_power_of_2 / abs_test_numer, remainder1 = initial_power_of_2 % abs_test_numer;
+	uint quotient2 = initial_power_of_2 / abs_d, remainder2 = initial_power_of_2 % abs_d;
+	uint delta;
+
+	// Begin our loop
+	do {
+		// Update the exponent
+		exponent++;
+
+		// Update quotient1 and remainder1
+		quotient1 *= 2;
+		remainder1 *= 2;
+		if (remainder1 >= abs_test_numer) {
+			quotient1 += 1;
+			remainder1 -= abs_test_numer;
+		}
+
+		// Update quotient2 and remainder2
+		quotient2 *= 2;
+		remainder2 *= 2;
+		if (remainder2 >= abs_d) {
+			quotient2 += 1;
+			remainder2 -= abs_d;
+		}
+
+		// Keep going as long as (2**exponent) / abs_d <= delta
+		delta = abs_d - remainder2;
+	} while (quotient1 < delta || (quotient1 == delta && remainder1 == 0));
+
+	result.multiplier = quotient2 + 1;
+	if (D < 0) result.multiplier = -result.multiplier;
+	result.shift = exponent - SINT_BITS;
+	return result;
+}

src/divideByConstantCodegen.h

@@ -0,0 +1,117 @@
+/*
+Copyright (c) 2018 tevador
+
+This file is part of RandomX.
+
+RandomX is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+RandomX is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with RandomX.  If not, see<http://www.gnu.org/licenses/>.
+*/
+
+#pragma once
+#include <stdint.h>
+
+#if defined(__cplusplus)
+extern "C" {
+#endif
+
+	typedef uint64_t uint;
+	typedef int64_t sint;
+
+	/* Computes "magic info" for performing signed division by a fixed integer D.
+	   The type 'sint' is assumed to be defined as a signed integer type large enough
+	   to hold both the dividend and the divisor.
+	   Here >> is arithmetic (signed) shift, and >>> is logical shift.
+
+	   To emit code for n/d, rounding towards zero, use the following sequence:
+
+		 m = compute_signed_magic_info(D)
+		 emit("result = (m.multiplier * n) >> SINT_BITS");
+		 if d > 0 and m.multiplier < 0: emit("result += n")
+		 if d < 0 and m.multiplier > 0: emit("result -= n")
+		 if m.post_shift > 0: emit("result >>= m.shift")
+		 emit("result += (result < 0)")
+
+	  The shifts by SINT_BITS may be "free" if the high half of the full multiply
+	  is put in a separate register.
+
+	  The final add can of course be implemented via the sign bit, e.g.
+		  result += (result >>> (SINT_BITS - 1))
+	   or
+		  result -= (result >> (SINT_BITS - 1))
+
+	   This code is heavily indebted to Hacker's Delight by Henry Warren.
+	   See http://www.hackersdelight.org/HDcode/magic.c.txt
+	   Used with permission from http://www.hackersdelight.org/permissions.htm
+	 */
+
+	struct magics_info {
+		sint multiplier; // the "magic number" multiplier
+		unsigned shift; // shift for the dividend after multiplying
+	};
+	struct magics_info compute_signed_magic_info(sint D);
+
+
+	/* Computes "magic info" for performing unsigned division by a fixed positive integer D.
+	   The type 'uint' is assumed to be defined as an unsigned integer type large enough
+	   to hold both the dividend and the divisor. num_bits can be set appropriately if n is
+	   known to be smaller than the largest uint; if this is not known then pass
+	   (sizeof(uint) * CHAR_BIT) for num_bits.
+
+	   Assume we have a hardware register of width UINT_BITS, a known constant D which is
+	   not zero and not a power of 2, and a variable n of width num_bits (which may be
+	   up to UINT_BITS). To emit code for n/d, use one of the two following sequences
+	   (here >>> refers to a logical bitshift):
+
+		 m = compute_unsigned_magic_info(D, num_bits)
+		 if m.pre_shift > 0: emit("n >>>= m.pre_shift")
+		 if m.increment: emit("n = saturated_increment(n)")
+		 emit("result = (m.multiplier * n) >>> UINT_BITS")
+		 if m.post_shift > 0: emit("result >>>= m.post_shift")
+
+	   or
+
+		 m = compute_unsigned_magic_info(D, num_bits)
+		 if m.pre_shift > 0: emit("n >>>= m.pre_shift")
+		 emit("result = m.multiplier * n")
+		 if m.increment: emit("result = result + m.multiplier")
+		 emit("result >>>= UINT_BITS")
+		 if m.post_shift > 0: emit("result >>>= m.post_shift")
+
+	  The shifts by UINT_BITS may be "free" if the high half of the full multiply
+	  is put in a separate register.
+
+	  saturated_increment(n) means "increment n unless it would wrap to 0," i.e.
+		if n == (1 << UINT_BITS)-1: result = n
+		else: result = n+1
+	  A common way to implement this is with the carry bit. For example, on x86:
+		 add 1
+		 sbb 0
+
+	  Some invariants:
+	   1: At least one of pre_shift and increment is zero
+	   2: multiplier is never zero
+
+	   This code incorporates the "round down" optimization per ridiculous_fish.
+	 */
+
+	struct magicu_info {
+		uint multiplier; // the "magic number" multiplier
+		unsigned pre_shift; // shift for the dividend before multiplying
+		unsigned post_shift; //shift for the dividend after multiplying
+		int increment; // 0 or 1; if set then increment the numerator, using one of the two strategies
+	};
+	struct magicu_info compute_unsigned_magic_info(uint D, unsigned num_bits);
+
+#if defined(__cplusplus)
+}
+#endif