From 4804f4a4798046030bcf9aa8404c121502256dcb Mon Sep 17 00:00:00 2001 From: korthaj Date: Sat, 17 Jun 2017 13:44:37 +0200 Subject: [PATCH 1/4] Use RotateLeft from GO 1.9 --- hash.go | 2 + hash_math_bits.go | 214 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 hash_math_bits.go diff --git a/hash.go b/hash.go index 9571ef6..4057696 100644 --- a/hash.go +++ b/hash.go @@ -1,3 +1,5 @@ +// +build !go1.9 + package bloom // MurmurHash3 implementation adapted from Sébastien Paolacci diff --git a/hash_math_bits.go b/hash_math_bits.go new file mode 100644 index 0000000..3a394d8 --- /dev/null +++ b/hash_math_bits.go @@ -0,0 +1,214 @@ +// +build go1.9 + +package bloom + +import "math/bits" + +// MurmurHash3 implementation adapted from Sébastien Paolacci +// github.com/spaolacci/murmur3, released under BSD-3-Clause. + +const ( + c1 = 0x87c37b91114253d5 + c2 = 0x4cf5ad432745937f +) + +func fmix(k uint64) uint64 { + k ^= k >> 33 + k *= 0xff51afd7ed558ccd + k ^= k >> 33 + k *= 0xc4ceb9fe1a85ec53 + k ^= k >> 33 + return k +} + +func uint64byte(b []byte) uint64 { + return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | + uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 +} + +func uint64string(s string) uint64 { + return uint64(s[0]) | uint64(s[1])<<8 | uint64(s[2])<<16 | uint64(s[3])<<24 | + uint64(s[4])<<32 | uint64(s[5])<<40 | uint64(s[6])<<48 | uint64(s[7])<<56 +} + +func hash(b []byte) (h1, h2 uint64) { + nblocks := len(b) / 16 + for i := 0; i < nblocks; i++ { + j := 16 * i + k1, k2 := uint64byte(b[j:j+8]), uint64byte(b[j+8:j+16]) + + k1 *= c1 + k1 = bits.RotateLeft64(k1, 31) + k1 *= c2 + + h1 ^= k1 + h1 = bits.RotateLeft64(h1, 27) + h1 += h2 + h1 = h1*5 + 0x52dce729 + + k2 *= c2 + k2 = bits.RotateLeft64(k2, 33) + k2 *= c1 + + h2 ^= k2 + h2 = bits.RotateLeft64(h2, 31) + h2 += h1 + h2 = h2*5 + 0x38495ab5 + } + + tail := b[nblocks*16:] + var k1, k2 uint64 + switch len(tail) { + case 15: + k2 ^= uint64(tail[14]) << 48 + fallthrough + case 14: + k2 ^= uint64(tail[13]) << 40 + fallthrough + case 13: + k2 ^= uint64(tail[12]) << 32 + fallthrough + case 12: + k2 ^= uint64(tail[11]) << 24 + fallthrough + case 11: + k2 ^= uint64(tail[10]) << 16 + fallthrough + case 10: + k2 ^= uint64(tail[9]) << 8 + fallthrough + case 9: + k2 ^= uint64(tail[8]) << 0 + k2 *= c2 + k2 = bits.RotateLeft64(k2, 33) + k2 *= c1 + h2 ^= k2 + fallthrough + case 8: + k1 ^= uint64(tail[7]) << 56 + fallthrough + case 7: + k1 ^= uint64(tail[6]) << 48 + fallthrough + case 6: + k1 ^= uint64(tail[5]) << 40 + fallthrough + case 5: + k1 ^= uint64(tail[4]) << 32 + fallthrough + case 4: + k1 ^= uint64(tail[3]) << 24 + fallthrough + case 3: + k1 ^= uint64(tail[2]) << 16 + fallthrough + case 2: + k1 ^= uint64(tail[1]) << 8 + fallthrough + case 1: + k1 ^= uint64(tail[0]) << 0 + k1 *= c1 + k1 = bits.RotateLeft64(k1, 31) + k1 *= c2 + h1 ^= k1 + } + h1 ^= uint64(len(b)) + h2 ^= uint64(len(b)) + h1 += h2 + h2 += h1 + h1, h2 = fmix(h1), fmix(h2) + h1 += h2 + h2 += h1 + return +} + +func hashString(s string) (h1, h2 uint64) { + nblocks := len(s) / 16 + for i := 0; i < nblocks; i++ { + j := 16 * i + k1, k2 := uint64string(s[j:j+8]), uint64string(s[j+8:j+16]) + + k1 *= c1 + k1 = bits.RotateLeft64(k1, 31) + k1 *= c2 + + h1 ^= k1 + h1 = bits.RotateLeft64(h1, 27) + h1 += h2 + h1 = h1*5 + 0x52dce729 + + k2 *= c2 + k2 = bits.RotateLeft64(k2, 33) + k2 *= c1 + + h2 ^= k2 + h2 = bits.RotateLeft64(h2, 31) + h2 += h1 + h2 = h2*5 + 0x38495ab5 + } + + tail := s[nblocks*16:] + var k1, k2 uint64 + switch len(tail) { + case 15: + k2 ^= uint64(tail[14]) << 48 + fallthrough + case 14: + k2 ^= uint64(tail[13]) << 40 + fallthrough + case 13: + k2 ^= uint64(tail[12]) << 32 + fallthrough + case 12: + k2 ^= uint64(tail[11]) << 24 + fallthrough + case 11: + k2 ^= uint64(tail[10]) << 16 + fallthrough + case 10: + k2 ^= uint64(tail[9]) << 8 + fallthrough + case 9: + k2 ^= uint64(tail[8]) << 0 + k2 *= c2 + k2 = bits.RotateLeft64(k2, 33) + k2 *= c1 + h2 ^= k2 + fallthrough + case 8: + k1 ^= uint64(tail[7]) << 56 + fallthrough + case 7: + k1 ^= uint64(tail[6]) << 48 + fallthrough + case 6: + k1 ^= uint64(tail[5]) << 40 + fallthrough + case 5: + k1 ^= uint64(tail[4]) << 32 + fallthrough + case 4: + k1 ^= uint64(tail[3]) << 24 + fallthrough + case 3: + k1 ^= uint64(tail[2]) << 16 + fallthrough + case 2: + k1 ^= uint64(tail[1]) << 8 + fallthrough + case 1: + k1 ^= uint64(tail[0]) << 0 + k1 *= c1 + k1 = bits.RotateLeft64(k1, 31) + k1 *= c2 + h1 ^= k1 + } + h1 ^= uint64(len(s)) + h2 ^= uint64(len(s)) + h1 += h2 + h2 += h1 + h1, h2 = fmix(h1), fmix(h2) + h1 += h2 + h2 += h1 + return +} From f39a6b0b10d12643426a4756c95ee4fecbb47449 Mon Sep 17 00:00:00 2001 From: korthaj Date: Sat, 17 Jun 2017 13:53:31 +0200 Subject: [PATCH 2/4] Use OnesCount64 from Go 1.9 --- filter.go | 2 + filter_math_bits.go | 176 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 filter_math_bits.go diff --git a/filter.go b/filter.go index 85b7808..4423009 100644 --- a/filter.go +++ b/filter.go @@ -1,3 +1,5 @@ +// +build !go1.9 + // Package bloom provides a Bloom filter implementation. // // Bloom filters diff --git a/filter_math_bits.go b/filter_math_bits.go new file mode 100644 index 0000000..efd866c --- /dev/null +++ b/filter_math_bits.go @@ -0,0 +1,176 @@ +// +build go1.9 + +// Package bloom provides a Bloom filter implementation. +// +// Bloom filters +// +// A Bloom filter is a fast and space-efficient probabilistic data structure +// used to test set membership. +// +// A membership test returns either ”likely member” or ”definitely not +// a member”. Only false positives can occur: an element that has been added +// to the filter will always be identified as ”likely member”. +// +// The probabilities of different outcomes of a membership test at +// a false-positives rate of 1/100 are: +// +// Test(s) true false +// -------------------------------------- +// s has been added 1 0 +// s has not been added 0.01 0.99 +// +// Elements can be added, but not removed. With more elements in the filter, +// the probability of false positives increases. +// +// Performance +// +// A full filter with a false-positives rate of 1/p uses roughly +// 0.26ln(p) bytes per element and performs ⌈1.4ln(p)⌉ bit array lookups +// per test: +// +// p bytes lookups +// ------------------------- +// 4 0.4 2 +// 8 0.5 3 +// 16 0.7 4 +// 32 0.9 5 +// 64 1.1 6 +// 128 1.3 7 +// 256 1.5 8 +// 512 1.6 9 +// 1024 1.8 10 +// +// Each membership test makes a single call to a 128-bit hash function. +// This improves speed without increasing the false-positives rate +// as shown by Kirsch and Mitzenmacher. +// +// Limitations +// +// This implementation is not intended for cryptographic use. +// +// The internal data representation is different for big-endian +// and little-endian machines. +// +// Typical use case +// +// The Basics example contains a typcial use case: +// a blacklist of shady websites. +// +package bloom + +import ( + "math" + "math/bits" +) + +const ( + shift = 6 + mask = 0x3f +) + +// Filter represents a Bloom filter. +type Filter struct { + data []uint64 // Bit array, the length is a power of 2. + lookups int // Lookups per query + count int64 // Estimate number of elements +} + +// New creates an empty Bloom filter with room for n elements +// at a false-positives rate less than 1/p. +func New(n int, p int) *Filter { + minWords := int(0.0325 * math.Log(float64(p)) * float64(n)) + words := 1 + for words < minWords { + words *= 2 + } + return &Filter{ + data: make([]uint64, words), + lookups: int(1.4*math.Log(float64(p)) + 1), + } +} + +// AddByte adds b to the filter and tells if b was already a likely member. +func (f *Filter) AddByte(b []byte) bool { + return f.add(hash(b)) +} + +// Add adds s to the filter and tells if s was already a likely member. +func (f *Filter) Add(s string) bool { + return f.add(hashString(s)) +} + +func (f *Filter) add(h1, h2 uint64) bool { + trunc := uint64(len(f.data))< 0; i-- { + h1 += h2 + n := h1 & trunc + k, b := n>>shift, uint64(1< 0; i-- { + h1 += h2 + n := h1 & trunc + k, b := n>>shift, uint64(1< Date: Mon, 19 Jun 2017 10:12:16 +0200 Subject: [PATCH 3/4] Add godoc badge --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ed4b078..6caa829 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ ### Golang probabilistic set data structure +[![GoDoc](https://godoc.org/github.com/yourbasic/bloom?status.svg)][godoc-bloom] + A Bloom filter is a fast and space-efficient probabilistic data structure used to test set membership. A membership test returns either ”likely member” or ”definitely not a member”. From 75d417d08476ea6a5208a92f09edad9d757091d4 Mon Sep 17 00:00:00 2001 From: korthaj Date: Mon, 19 Jun 2017 19:00:40 +0200 Subject: [PATCH 4/4] Move godoc badge --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index 6caa829..603e85a 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,7 @@ -# Your basic Bloom filter +# Your basic Bloom filter [![GoDoc](https://godoc.org/github.com/yourbasic/bloom?status.svg)][godoc-bloom] ### Golang probabilistic set data structure -[![GoDoc](https://godoc.org/github.com/yourbasic/bloom?status.svg)][godoc-bloom] - A Bloom filter is a fast and space-efficient probabilistic data structure used to test set membership. A membership test returns either ”likely member” or ”definitely not a member”.