asm_amd64.s

// Code generated by command: go run avo.go -out ../../asm_amd64.s -stubs ../../stubs_amd64.go -pkg vectorcmp --dispatcher-amd64 ../../dispatch_amd64.go --dispatcher-other ../../dispatch_other.go. DO NOT EDIT.

//go:build !purego

#include "textflag.h"

DATA constants<>+0(SB)/4, $0x55555555
DATA constants<>+4(SB)/4, $0x11111111
DATA constants<>+8(SB)/4, $0x01010101
GLOBL constants<>(SB), RODATA|NOPTR, $12

DATA const_zeroes<>+0(SB)/4, $0x00000000
DATA const_zeroes<>+4(SB)/4, $0x00000000
DATA const_zeroes<>+8(SB)/4, $0x00000000
DATA const_zeroes<>+12(SB)/4, $0x00000000
GLOBL const_zeroes<>(SB), RODATA|NOPTR, $16

DATA const_onezero<>+0(SB)/4, $0x01000100
DATA const_onezero<>+4(SB)/4, $0x01000100
DATA const_onezero<>+8(SB)/4, $0x01000100
DATA const_onezero<>+12(SB)/4, $0x01000100
GLOBL const_onezero<>(SB), RODATA|NOPTR, $16

DATA const_three_through_zero<>+0(SB)/4, $0x03020100
DATA const_three_through_zero<>+4(SB)/4, $0x03020100
DATA const_three_through_zero<>+8(SB)/4, $0x03020100
DATA const_three_through_zero<>+12(SB)/4, $0x03020100
GLOBL const_three_through_zero<>(SB), RODATA|NOPTR, $16

DATA const_seven_through_zero<>+0(SB)/8, $0x0706050403020100
DATA const_seven_through_zero<>+8(SB)/8, $0x0706050403020100
GLOBL const_seven_through_zero<>(SB), RODATA|NOPTR, $16

// func asmAVX2Equals8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2Equals8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQB Y1, Y0, Y1
	VPCMPEQB Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	RET

// func asmAVXEquals8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXEquals8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQB X1, X0, X1
	VPCMPEQB X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	RET

// func asmAVX2GreaterThan8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2GreaterThan8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB Y1, Y0, Y1
	VPCMPGTB Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	RET

// func asmAVXGreaterThan8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterThan8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB X1, X0, X1
	VPCMPGTB X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	RET

// func asmAVX2LessThan8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2LessThan8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB Y0, Y1, Y1
	VPCMPGTB Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	RET

// func asmAVXLessThan8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXLessThan8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB X0, X1, X1
	VPCMPGTB X0, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	RET

// func asmAVX2GreaterEquals8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2GreaterEquals8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB Y0, Y1, Y1
	VPCMPGTB Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTL BX
	NOTL SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	RET

// func asmAVXGreaterEquals8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXGreaterEquals8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB X0, X1, X1
	VPCMPGTB X0, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTW BX
	NOTW SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	RET

// func asmAVX2LesserEquals8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, AVX2
TEXT ·asmAVX2LesserEquals8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTB b+24(FP), Y0

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB Y1, Y0, Y1
	VPCMPGTB Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTL BX
	NOTL SI

	// Write the registers to dstMask
	MOVL BX, (AX)
	MOVL SI, 4(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000008, AX

	// Decrement loop counter
	SUBQ $0x00000040, DX
	JNZ  loop
	RET

// func asmAVXLesserEquals8(dstMask []byte, b uint8, rows []uint8)
// Requires: AVX, SSE2
TEXT ·asmAVXLesserEquals8(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVB    b+24(FP), BL
	MOVQ    BX, X0
	VPSHUFB const_zeroes<>+0(SB), X0, X0

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTB X1, X0, X1
	VPCMPGTB X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTW BX
	NOTW SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	RET

// func asmAVX2Equals16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2Equals16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQW Y1, Y0, Y1
	VPCMPEQW Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	RET

// func asmAVXEquals16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXEquals16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQW X1, X0, X1
	VPCMPEQW X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	RET

// func asmAVX2GreaterThan16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterThan16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW Y1, Y0, Y1
	VPCMPGTW Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	RET

// func asmAVXGreaterThan16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXGreaterThan16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW X1, X0, X1
	VPCMPGTW X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	RET

// func asmAVX2LessThan16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LessThan16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW Y0, Y1, Y1
	VPCMPGTW Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	RET

// func asmAVXLessThan16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXLessThan16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW X0, X1, X1
	VPCMPGTW X0, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	RET

// func asmAVX2GreaterEquals16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterEquals16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW Y0, Y1, Y1
	VPCMPGTW Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTW BX
	NOTW SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	RET

// func asmAVXGreaterEquals16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXGreaterEquals16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW X0, X1, X1
	VPCMPGTW X0, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTB BL
	NOTB SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	RET

// func asmAVX2LesserEquals16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LesserEquals16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTW b+24(FP), Y0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW Y1, Y0, Y1
	VPCMPGTW Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTW BX
	NOTW SI

	// Write the registers to dstMask
	MOVW BX, (AX)
	MOVW SI, 2(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000004, AX

	// Decrement loop counter
	SUBQ $0x00000020, DX
	JNZ  loop
	RET

// func asmAVXLesserEquals16(dstMask []byte, b uint16, rows []uint16)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXLesserEquals16(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVW    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_onezero<>+0(SB), X0, X0

	// Load the mask 01010101... which we will use with PEXT to drop half the bits
	MOVL constants<>+0(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTW X1, X0, X1
	VPCMPGTW X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTB BL
	NOTB SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	RET

// func asmAVX2Equals32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2Equals32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQD Y1, Y0, Y1
	VPCMPEQD Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	RET

// func asmAVXEquals32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXEquals32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQD X1, X0, X1
	VPCMPEQD X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	RET

// func asmAVX2GreaterThan32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterThan32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD Y1, Y0, Y1
	VPCMPGTD Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	RET

// func asmAVXGreaterThan32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXGreaterThan32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD X1, X0, X1
	VPCMPGTD X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	RET

// func asmAVX2LessThan32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LessThan32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD Y0, Y1, Y1
	VPCMPGTD Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	RET

// func asmAVXLessThan32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXLessThan32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD X0, X1, X1
	VPCMPGTD X0, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	RET

// func asmAVX2GreaterEquals32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterEquals32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD Y0, Y1, Y1
	VPCMPGTD Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTB BL
	NOTB SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	RET

// func asmAVXGreaterEquals32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXGreaterEquals32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD X0, X1, X1
	VPCMPGTD X0, X2, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	NOTB BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	RET

// func asmAVX2LesserEquals32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LesserEquals32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTD b+24(FP), Y0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD Y1, Y0, Y1
	VPCMPGTD Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	NOTB BL
	NOTB SI

	// Write the registers to dstMask
	MOVB BL, (AX)
	MOVB SI, 1(AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000002, AX

	// Decrement loop counter
	SUBQ $0x00000010, DX
	JNZ  loop
	RET

// func asmAVXLesserEquals32(dstMask []byte, b uint32, rows []uint32)
// Requires: AVX, BMI2, SSE2
TEXT ·asmAVXLesserEquals32(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into XMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	MOVL    b+24(FP), BX
	MOVQ    BX, X0
	VPSHUFB const_three_through_zero<>+0(SB), X0, X0

	// Load the mask 00010001... which we will use with PEXT to drop 75% of the bits
	MOVL constants<>+4(SB), DI

loop:
	// Load 2 128-bit chunks into XMM registers
	VMOVDQU (CX), X1
	VMOVDQU 16(CX), X2

	// Compare all bytes in each XMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTD X1, X0, X1
	VPCMPGTD X2, X0, X2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB X1, BX
	VPMOVMSKB X2, SI

	// Drop every second-fourth bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	NOTB BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000020, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	RET

// func asmAVX2Equals64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2Equals64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPEQQ Y1, Y0, Y1
	VPCMPEQQ Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-seventh bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	RET

// func asmAVX2GreaterThan64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterThan64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ Y1, Y0, Y1
	VPCMPGTQ Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-seventh bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	RET

// func asmAVX2LessThan64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LessThan64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ Y0, Y1, Y1
	VPCMPGTQ Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-seventh bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	RET

// func asmAVX2GreaterEquals64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2GreaterEquals64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ Y0, Y1, Y1
	VPCMPGTQ Y0, Y2, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-seventh bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get GreaterEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	NOTB BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	RET

// func asmAVX2LesserEquals64(dstMask []byte, b uint64, rows []uint64)
// Requires: AVX, AVX2, BMI2
TEXT ·asmAVX2LesserEquals64(SB), NOSPLIT, $0-56
	MOVQ dstMask_base+0(FP), AX
	MOVQ rows_base+32(FP), CX
	MOVQ rows_len+40(FP), DX

	// Read param b into YMM register. If b is 0x07, YMM becomes {0x07, 0x07, 0x07...}
	VPBROADCASTQ b+24(FP), Y0

	// Load the mask 00000001... which we will use with PEXT to drop 7/8th of the bits
	MOVL constants<>+8(SB), DI

loop:
	// Load 2 256-bit chunks into YMM registers
	VMOVDQU (CX), Y1
	VMOVDQU 32(CX), Y2

	// Compare all bytes in each YMM register to b. Each byte in the YMMs becomes 0x00 (mismatch) or 0xff (match)
	VPCMPGTQ Y1, Y0, Y1
	VPCMPGTQ Y2, Y0, Y2

	// Take one bit of each byte and pack it into an R32
	VPMOVMSKB Y1, BX
	VPMOVMSKB Y2, SI

	// Drop every second-seventh bit from these registers
	PEXTL DI, BX, BX
	PEXTL DI, SI, SI

	// To get LesserEquals semantics, we flipped the arguments of VPCMPGT and now invert the result
	// Each register contains 4 bits, so we first combine pairs before writing them back
	SHLB $0x04, SI
	ORB  SI, BL
	NOTB BL
	MOVB BL, (AX)

	// Update our offsets into rows and dstMask
	ADDQ $0x00000040, CX
	ADDQ $0x00000001, AX

	// Decrement loop counter
	SUBQ $0x00000008, DX
	JNZ  loop
	RET