// Copyright 2010 Google Inc. All rights reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. // Implements multiword CRC for GCC on AMD64. // // Accoding to "Software Optimization Guide for AMD Family 10h Processors" // http://www.amd.com/us-en/assets/content_type/white_papers_and_tech_docs/40546.pdf // instead of // movzbq %al, %rsi // shrq $8, %rax // [use %rsi] // movzbq %al, %rsi // shrq $8, %rax // [use %rsi] // it is better to use 32-bit registers // (high 32 bits will be cleared on assignment), i.e. // movzbl %al, %esi // [use %rsi] // movzbl %ah, %esi // shrq $16, %rax // [use %rsi] // Makes instructions shorter and removes one shift // (the latter is not such a big deal as it's execution time // is nicely masked by [use %rsi] instruction). // // Performance difference: // About 10% degradation on bytes = 8 .. 16 // (clobbering registers that should be saved) // Break even at 32 bytes. // 3% improvement starting from 64 bytes. #include "generic_crc.h" #if defined(__GNUC__) && CRCUTIL_USE_ASM && HAVE_AMD64 namespace crcutil { template<> uint64 GenericCrc::CrcMultiwordGccAmd64( const void *data, size_t bytes, const uint64 &start) const; template<> uint64 GenericCrc::CrcMultiword( const void *data, size_t bytes, const uint64 &start) const { if (bytes <= 6 * sizeof(Word) - 1) { const uint8 *src = static_cast(data); uint64 crc = start ^ this->Base().Canonize(); const uint8 *end = src + bytes; #define PROCESS_ONE_WORD() do { \ Word buf = reinterpret_cast(src)[0]; \ CRC_WORD(this, crc, buf); \ src += sizeof(Word); \ } while (0) if (bytes >= 1 * sizeof(Word)) { PROCESS_ONE_WORD(); if (bytes >= 2 * sizeof(Word)) { PROCESS_ONE_WORD(); if (bytes >= 3 * sizeof(Word)) { PROCESS_ONE_WORD(); if (bytes >= 4 * sizeof(Word)) { PROCESS_ONE_WORD(); if (bytes >= 5 * sizeof(Word)) { PROCESS_ONE_WORD(); } } } } } for (; src < end; ++src) { CRC_BYTE(this, crc, *src); } return (crc ^ this->Base().Canonize()); } return this->CrcMultiwordGccAmd64(data, bytes, start); } #define TMP0 "%%rsi" #define TMP0W "%%esi" #define BUF0 "%%rax" #define BUF0L "%%al" #define BUF0H "%%ah" #define BUF1 "%%rbx" #define BUF1L "%%bl" #define BUF1H "%%bh" #define BUF2 "%%rcx" #define BUF2L "%%cl" #define BUF2H "%%ch" #define BUF3 "%%rdx" #define BUF3L "%%dl" #define BUF3H "%%dh" #define CRC_WORD_ASM() \ "xorq %[crc0], " BUF0 "\n" \ "movzbq " BUF0L ", " TMP0 "\n" \ "movq (%[table_word], " TMP0 ", 8), %[crc0]\n" \ "movzbl " BUF0H ", " TMP0W "\n" \ "shrq $16, " BUF0 "\n" \ "xorq 1*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \ "movzbq " BUF0L ", " TMP0 "\n" \ "xorq 2*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \ "movzbl " BUF0H ", " TMP0W "\n" \ "shrq $16, " BUF0 "\n" \ "xorq 3*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \ "movzbq " BUF0L ", " TMP0 "\n" \ "xorq 4*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \ "movzbl " BUF0H ", " TMP0W "\n" \ "shrq $16, " BUF0 "\n" \ "xorq 5*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \ "movzbq " BUF0L ", " TMP0 "\n" \ "xorq 6*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" \ "movzbl " BUF0H ", " TMP0W "\n" \ "xorq 7*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" template<> uint64 GenericCrc::CrcMultiwordGccAmd64( const void *data, size_t bytes, const uint64 &start) const { const uint8 *src = static_cast(data); const uint8 *end = src + bytes; uint64 crc0 = start ^ this->Base().Canonize(); ALIGN_ON_WORD_BOUNDARY_IF_NEEDED(bytes, this, src, end, crc0, uint64); if (src >= end) { return (crc0 ^ this->Base().Canonize()); } uint64 crc1; uint64 crc2; uint64 crc3; asm( "sub $2*4*8 - 1, %[end]\n" "cmpq %[src], %[end]\n" "jbe 2f\n" "xorq %[crc1], %[crc1]\n" "movq (%[src]), " BUF0 "\n" "movq 1*8(%[src]), " BUF1 "\n" "movq 2*8(%[src]), " BUF2 "\n" "movq 3*8(%[src]), " BUF3 "\n" "movq %[crc1], %[crc2]\n" "movq %[crc1], %[crc3]\n" "1:\n" #if HAVE_SSE && CRCUTIL_PREFETCH_WIDTH > 0 "prefetcht0 " TO_STRING(CRCUTIL_PREFETCH_WIDTH) "(%[src])\n" #endif // HAVE_SSE "add $4*8, %[src]\n" // Set buffer data. "xorq %[crc0], " BUF0 "\n" "xorq %[crc1], " BUF1 "\n" "xorq %[crc2], " BUF2 "\n" "xorq %[crc3], " BUF3 "\n" // LOAD crc of byte 0 and shift buffers. "movzbl " BUF0L ", " TMP0W "\n" "movq (%[table], " TMP0 ", 8), %[crc0]\n" "movzbl " BUF1L ", " TMP0W "\n" "movq (%[table], " TMP0 ", 8), %[crc1]\n" "movzbl " BUF2L ", " TMP0W "\n" "movq (%[table], " TMP0 ", 8), %[crc2]\n" "movzbl " BUF3L ", " TMP0W "\n" "movq (%[table], " TMP0 ", 8), %[crc3]\n" #define XOR1(byte1) \ "movzbl " BUF0L ", " TMP0W "\n" \ "xorq " #byte1 "*256*8(%[table], " TMP0 ", 8), %[crc0]\n" \ "movzbl " BUF1L ", " TMP0W "\n" \ "xorq " #byte1 "*256*8(%[table], " TMP0 ", 8), %[crc1]\n" \ "movzbl " BUF2L ", " TMP0W "\n" \ "xorq " #byte1 "*256*8(%[table], " TMP0 ", 8), %[crc2]\n" \ "movzbl " BUF3L ", " TMP0W "\n" \ "xorq " #byte1 "*256*8(%[table], " TMP0 ", 8), %[crc3]\n" #define XOR2(byte2) \ "movzbl " BUF0H ", " TMP0W "\n" \ "shrq $16, " BUF0 "\n" \ "xorq " #byte2 "*256*8(%[table], " TMP0 ", 8), %[crc0]\n" \ "movzbl " BUF1H ", " TMP0W "\n" \ "shrq $16, " BUF1 "\n" \ "xorq " #byte2 "*256*8(%[table], " TMP0 ", 8), %[crc1]\n" \ "movzbl " BUF2H ", " TMP0W "\n" \ "shrq $16, " BUF2 "\n" \ "xorq " #byte2 "*256*8(%[table], " TMP0 ", 8), %[crc2]\n" \ "movzbl " BUF3H ", " TMP0W "\n" \ "shrq $16, " BUF3 "\n" \ "xorq " #byte2 "*256*8(%[table], " TMP0 ", 8), %[crc3]\n" XOR2(1) XOR1(2) XOR2(3) XOR1(4) XOR2(5) XOR1(6) // Update CRC registers and load buffers. "movzbl " BUF0H ", " TMP0W "\n" "xorq 7*256*8(%[table], " TMP0 ", 8), %[crc0]\n" "movq (%[src]), " BUF0 "\n" "movzbl " BUF1H ", " TMP0W "\n" "xorq 7*256*8(%[table], " TMP0 ", 8), %[crc1]\n" "movq 1*8(%[src]), " BUF1 "\n" "movzbl " BUF2H ", " TMP0W "\n" "xorq 7*256*8(%[table], " TMP0 ", 8), %[crc2]\n" "movq 2*8(%[src]), " BUF2 "\n" "movzbl " BUF3H ", " TMP0W "\n" "xorq 7*256*8(%[table], " TMP0 ", 8), %[crc3]\n" "movq 3*8(%[src]), " BUF3 "\n" "cmpq %[src], %[end]\n" "ja 1b\n" CRC_WORD_ASM() "xorq %[crc1], " BUF1 "\n" "movq " BUF1 ", " BUF0 "\n" CRC_WORD_ASM() "xorq %[crc2], " BUF2 "\n" "movq " BUF2 ", " BUF0 "\n" CRC_WORD_ASM() "xorq %[crc3], " BUF3 "\n" "movq " BUF3 ", " BUF0 "\n" CRC_WORD_ASM() "add $4*8, %[src]\n" "2:\n" "add $2*4*8 - 8, %[end]\n" "cmpq %[src], %[end]\n" "jbe 4f\n" "3:\n" "movq (%[src]), " BUF0 "\n" "add $8, %[src]\n" CRC_WORD_ASM() "cmpq %[src], %[end]\n" "ja 3b\n" "4:\n" "add $7, %[end]\n" "cmpq %[src], %[end]\n" "jbe 6f\n" "5:\n" "movzbq (%[src]), " BUF0 "\n" "movzbq %b[crc0], " TMP0 "\n" "shrq $8, %[crc0]\n" "xorq " BUF0 ", " TMP0 "\n" "add $1, %[src]\n" "xorq 7*256*8(%[table_word], " TMP0 ", 8), %[crc0]\n" "cmpq %[src], %[end]\n" "ja 5b\n" "6:\n" : // outputs [src] "+r" (src), [end] "+r" (end), [crc0] "+r" (crc0), [crc1] "=&r" (crc1), [crc2] "=&r" (crc2), [crc3] "=&r" (crc3) : // inputs [table] "r" (&this->crc_word_interleaved_[0][0]), [table_word] "r" (&this->crc_word_[0][0]) : // clobbers "%rax", // BUF0 "%rbx", // BUF1 "%rcx", // BUF2 "%rdx", // BUF3 "%rsi" // TMP0 ); return (crc0 ^ this->Base().Canonize()); } } // namespace crcutil #endif // defined(__GNUC__) && HAVE_AMD64 && CRCUTIL_USE_ASM