Update
This commit is contained in:
bluepython508 committed
1 parent
3530afb854
commit
6903fcff27
2146 files changed
+112256
-190703
No files matched your search
+230
-134
@@ -1,183 +1,279 @@
|
||||
# package bart
|
||||
|
||||
|
||||
[](https://pkg.go.dev/github.com/gaissmai/bart#section-documentation)
|
||||
[](https://github.com/avelino/awesome-go)
|
||||
[](https://github.com/gaissmai/bart/actions/workflows/go.yml)
|
||||
[](https://coveralls.io/github/gaissmai/bart)
|
||||
[](https://goreportcard.com/report/github.com/gaissmai/bart)
|
||||
[](https://opensource.org/licenses/MIT)
|
||||
[](https://stand-with-ukraine.pp.ua)
|
||||
|
||||
# package bart
|
||||
|
||||
The bart package provides some **Balanced Routing Tables (BART)** for
|
||||
fastest IP-to-CIDR lookups and related tasks such as:
|
||||
|
||||
- **ACL** determine extremely fast whether an IP address matches any of millions of CIDR rules.
|
||||
- **RIB** handle very large routing tables with low memory overhead, while keeping lookups fast.
|
||||
- **FIB** high-speed lookups, achieve LPM in constant-time for packet forwarding in the datapath.
|
||||
|
||||
BART is designed for workloads where both speed and/or memory efficiency matter,
|
||||
making it a best fit for firewalls, routers, or any system that needs large-scale
|
||||
IP prefix matching.
|
||||
|
||||
## Overview
|
||||
|
||||
`package bart` provides a Balanced-Routing-Table (BART).
|
||||
BART is implemented as a multibit trie with a fixed stride of 8 bits,
|
||||
using a fast mapping function derived from Donald E. Knuth’s
|
||||
**Allotment Routing Table** (ART) algorithm, to map the possible prefixes
|
||||
at each level into a complete binary tree.
|
||||
|
||||
BART is balanced in terms of memory usage and lookup time
|
||||
for the longest-prefix match.
|
||||
BART implements three different routing tables, each optimized for specific
|
||||
use cases:
|
||||
- **bart.Lite**
|
||||
- **bart.Table**
|
||||
- **bart.Fast**
|
||||
|
||||
BART is a multibit-trie with fixed stride length of 8 bits,
|
||||
using the _baseIndex_ function from the ART algorithm to
|
||||
build the complete-binary-tree (CBT) of prefixes for each stride.
|
||||
For **bart.Table** this binary tree is represented with popcount‑compressed
|
||||
sparse arrays for **level compression**.
|
||||
Combined with a **novel path and fringe compression**, this design reduces
|
||||
memory consumption by nearly two orders of magnitude compared to classical ART.
|
||||
|
||||
The CBT is implemented as a bit-vector, backtracking is just
|
||||
a matter of fast cache friendly bitmask operations.
|
||||
For **bart.Fast**, an additional 256 bytes per node are used to
|
||||
achieve faster level traversal in the multibit trie.
|
||||
|
||||
The Table is implemented with popcount compressed sparse arrays
|
||||
together with path compression. This reduces storage consumption
|
||||
by almost two orders of magnitude in comparison to ART with
|
||||
comparable or even better lookup times for longest prefix match.
|
||||
**bart.Lite** is a special form of **bart.Table**, but without a payload, and therefore
|
||||
has the lowest memory overhead while maintaining the same lookup times.
|
||||
|
||||
The algorithm is also excellent for determining whether two tables
|
||||
contain overlapping IP addresses.
|
||||
## Getting Started
|
||||
|
||||
## Example
|
||||
### Example: simple ACL with bart.Lite
|
||||
|
||||
```golang
|
||||
func ExampleTable_Contains() {
|
||||
// Create a new routing table
|
||||
table := new(bart.Table[struct{}])
|
||||
```go
|
||||
package main
|
||||
|
||||
// Insert some prefixes
|
||||
prefixes := []string{
|
||||
"192.168.0.0/16", // corporate
|
||||
"192.168.1.0/24", // department
|
||||
"2001:7c0:3100::/40", // corporate
|
||||
"2001:7c0:3100:1::/64", // department
|
||||
"fc00::/7", // unique local
|
||||
}
|
||||
import (
|
||||
"net/netip"
|
||||
|
||||
for _, s := range prefixes {
|
||||
pfx := netip.MustParsePrefix(s)
|
||||
table.Insert(pfx, struct{}{})
|
||||
}
|
||||
"github.com/gaissmai/bart"
|
||||
)
|
||||
|
||||
// Test some IP addresses for black/whitelist containment
|
||||
ips := []string{
|
||||
"192.168.1.100", // must match, department
|
||||
"192.168.2.1", // must match, corporate
|
||||
"2001:7c0:3100:1::1", // must match, department
|
||||
"2001:7c0:3100:2::1", // must match, corporate
|
||||
"fc00::1", // must match, unique local
|
||||
//
|
||||
"172.16.0.1", // must NOT match
|
||||
"2003:dead:beef::1", // must NOT match
|
||||
}
|
||||
var a = netip.MustParseAddr
|
||||
var p = netip.MustParsePrefix
|
||||
|
||||
for _, s := range ips {
|
||||
ip := netip.MustParseAddr(s)
|
||||
fmt.Printf("%-20s is contained: %t\n", ip, table.Contains(ip))
|
||||
}
|
||||
func main() {
|
||||
allowList := new(bart.Lite)
|
||||
|
||||
// Output:
|
||||
// 192.168.1.100 is contained: true
|
||||
// 192.168.2.1 is contained: true
|
||||
// 2001:7c0:3100:1::1 is contained: true
|
||||
// 2001:7c0:3100:2::1 is contained: true
|
||||
// fc00::1 is contained: true
|
||||
// 172.16.0.1 is contained: false
|
||||
// 2003:dead:beef::1 is contained: false
|
||||
// Add allowed networks
|
||||
allowList.Insert(p("192.168.0.0/16"))
|
||||
allowList.Insert(p("2001:db8::/32"))
|
||||
allowList.Insert(p("10.0.0.0/24"))
|
||||
|
||||
// Test some IPs with Contains (longest-prefix match)
|
||||
testIPs := []netip.Addr{
|
||||
a("192.168.1.100"), // allowed (matches 192.168.0.0/16)
|
||||
a("2001:db8::1"), // allowed (matches 2001:db8::/32)
|
||||
a("172.16.0.1"), // denied (no match)
|
||||
}
|
||||
|
||||
for _, ip := range testIPs {
|
||||
if allowList.Contains(ip) {
|
||||
fmt.Printf("✅ %s is allowed\n", ip)
|
||||
} else {
|
||||
fmt.Printf("❌ %s is denied\n", ip)
|
||||
}
|
||||
}
|
||||
|
||||
// Exact-prefix checks with Get (not longest-prefix match)
|
||||
exactPrefixes := []netip.Prefix{
|
||||
p("192.168.0.0/16"), // exists exactly
|
||||
p("192.168.1.0/24"), // doesn't exist (would be covered by /16, but not stored exactly)
|
||||
p("10.0.0.0/24"), // exists exactly
|
||||
p("10.0.0.0/16"), // doesn't exist (only /24 is stored)
|
||||
}
|
||||
|
||||
fmt.Println("\nExact-prefix checks:")
|
||||
for _, pfx := range exactPrefixes {
|
||||
if allowList.Get(pfx) {
|
||||
fmt.Printf("✅ Exact prefix %s exists in table\n", pfx)
|
||||
} else {
|
||||
fmt.Printf("❌ Exact prefix %s not found in table\n", pfx)
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Comparison
|
||||
|
||||
| Aspect | Table | Lite | Fast |
|
||||
|--------|-------------|-------------|-------------|
|
||||
| **Per-level Speed** | ⚡ **O(1)** | ⚡ **O(1)** | 🚀 **O(1), ~25% faster** |
|
||||
| **Overall Lookup** | O(trie_depth) | O(trie_depth) | O(trie_depth) |
|
||||
| **IPv4 Performance** | ~3 level traversals | ~3 level traversals | ~3 level traversals |
|
||||
| **IPv6 Performance** | ~6 level traversals | ~6 level traversals | ~6 level traversals |
|
||||
| **IPv6 vs IPv4** | ~2× slower | ~2× slower | ~2× slower |
|
||||
| **Memory** | efficient | very efficient | less efficient |
|
||||
| **Update** | very fast | best | fast |
|
||||
|
||||
A more detailed description can be found in [NODETYPES.md](NODETYPES.md).
|
||||
|
||||
## When to Use Each Type
|
||||
|
||||
### 🎯 **bart.Table[V]** - The Balanced Choice
|
||||
- **Recommended** for most routing table use cases
|
||||
- Near-optimal per-level performance with excellent memory efficiency
|
||||
- Very fast updates (insert/delete)
|
||||
- Perfect balance for both IPv4 and IPv6 routing tables (use it for RIB)
|
||||
|
||||
### 🪶 **bart.Lite** - The Minimalist
|
||||
- **Specialized** for prefix-only operations, no payload
|
||||
- Same per-level performance as *bart.Table[V]* but ~35% less memory
|
||||
- Optimal updates (insert/delete), ~25% faster than *bart.Table[V]*
|
||||
- Ideal for IPv4/IPv6 allowlists and set-based operations (use it for ACL)
|
||||
|
||||
### 🚀 **bart.Fast[V]** - The Lookup Performance Champion
|
||||
- **~25% faster** when memory constraints allow
|
||||
- Fast updates (insert/delete), ~50% slower than *bart.Table[V]*
|
||||
- Best choice for lookup-intensive applications (use it for FIB)
|
||||
|
||||
## Bitset Efficiency
|
||||
|
||||
The BART algorithm is based on fixed-size bit vectors and precomputed lookup tables.
|
||||
Lookups are executed entirely with fast, cache-resident bitmask operations, which
|
||||
modern CPUs accelerate using specialized instructions such as POPCNT, LZCNT, and TZCNT.
|
||||
|
||||
For maximum performance, specify the CPU feature set when compiling.
|
||||
See the [Go minimum requirements](https://go.dev/wiki/MinimumRequirements#architectures) for details.
|
||||
|
||||
```bash
|
||||
# On ARM64, Go auto-selects CPU instructions.
|
||||
# Example for AMD64, choose v2/v3/v4 to match your CPU features.
|
||||
GOAMD64=v3 go build
|
||||
```
|
||||
Critical loops over these fixed-size bitsets can be unrolled for additional speed,
|
||||
ensuring predictable memory access and efficient use of CPU pipelines.
|
||||
|
||||
```go
|
||||
func (b *BitSet256) popcnt() (cnt int) {
|
||||
cnt += bits.OnesCount64(b[0])
|
||||
cnt += bits.OnesCount64(b[1])
|
||||
cnt += bits.OnesCount64(b[2])
|
||||
cnt += bits.OnesCount64(b[3])
|
||||
return
|
||||
}
|
||||
```
|
||||
Future Go versions with SIMD intrinsics for `uint64` vectors may unlock
|
||||
additional speedups on compatible hardware.
|
||||
|
||||
## Concurrency model
|
||||
|
||||
There are examples demonstrating how to use bart concurrently with multiple readers and writers.
|
||||
Readers can always access the table lock‑free. Writers synchronize with a mutex so that only one writer
|
||||
modifies the persistent table at a time, without relying on CAS, which can be problematic with multiple
|
||||
long‑running writers.
|
||||
|
||||
The combination of lock-free concurrency, fast lookup and update times and low memory consumption
|
||||
provides clear advantages for any routing daemon.
|
||||
|
||||
But as always, it depends on the specific use case.
|
||||
|
||||
See the concurrent tests for concrete examples of this pattern:
|
||||
- [ExampleFast](example_fast_concurrent_test.go)
|
||||
- [ExampleLite](example_lite_concurrent_test.go)
|
||||
- [ExampleTable](example_table_concurrent_test.go)
|
||||
|
||||
|
||||
## Additional Use Cases
|
||||
|
||||
Beyond high-performance prefix matching, BART also excels at detecting overlaps
|
||||
between two routing tables.
|
||||
In internal benchmarks the check runs in a few nanoseconds per query with zero
|
||||
heap allocations on a modern CPU.
|
||||
|
||||
## API
|
||||
|
||||
Release v0.18 requires at least go1.23 and we use the `iter.Seq2[netip.Prefix, V]` types for iterators.
|
||||
The lock-free versions of insert, update and delete are added, but still experimentell.
|
||||
BART has a rich API for CRUD, lookup, comparison, iteration,
|
||||
serialization and persistence.
|
||||
|
||||
```golang
|
||||
import "github.com/gaissmai/bart"
|
||||
|
||||
type Table[V any] struct {
|
||||
// Has unexported fields.
|
||||
}
|
||||
Table is an IPv4 and IPv6 routing table with payload V. The zero value is
|
||||
ready to use.
|
||||
**Table** and **Fast** expose the identical API, while **Lite** deviates in
|
||||
its methods from the common API when it comes to the payload, since *Lite*
|
||||
has no payload.
|
||||
|
||||
The Table is safe for concurrent readers but not for concurrent readers
|
||||
and/or writers. Either the update operations must be protected by an
|
||||
external lock mechanism or the various ...Persist functions must be used
|
||||
which return a modified routing table by leaving the original unchanged
|
||||
```go
|
||||
import "github.com/gaissmai/bart"
|
||||
|
||||
A Table must not be copied by value, see Table.Clone.
|
||||
type Table[V any] struct {
|
||||
// Has unexported fields.
|
||||
}
|
||||
|
||||
func (t *Table[V]) Insert(pfx netip.Prefix, val V)
|
||||
func (t *Table[V]) Delete(pfx netip.Prefix)
|
||||
func (t *Table[V]) Update(pfx netip.Prefix, cb func(val V, ok bool) V) (newVal V)
|
||||
func (t *Table[V]) Contains(netip.Addr) bool
|
||||
func (t *Table[V]) Lookup(netip.Addr) (V, bool)
|
||||
|
||||
func (t *Table[V]) InsertPersist(pfx netip.Prefix, val V) *Table[V]
|
||||
func (t *Table[V]) DeletePersist(pfx netip.Prefix) *Table[V]
|
||||
func (t *Table[V]) UpdatePersist(pfx netip.Prefix, cb func(val V, ok bool) V) (pt *Table[V], newVal V)
|
||||
func (t *Table[V]) LookupPrefix(netip.Prefix) (V, bool)
|
||||
func (t *Table[V]) LookupPrefixLPM(netip.Prefix) (netip.Prefix, V, bool)
|
||||
|
||||
func (t *Table[V]) Get(pfx netip.Prefix) (val V, ok bool)
|
||||
func (t *Table[V]) GetAndDelete(pfx netip.Prefix) (val V, ok bool)
|
||||
func (t *Table[V]) GetAndDeletePersist(pfx netip.Prefix) (pt *Table[V], val V, ok bool)
|
||||
func (t *Table[V]) Get(netip.Prefix) (V, bool)
|
||||
func (t *Table[V]) Insert(netip.Prefix, V)
|
||||
func (t *Table[V]) Delete(netip.Prefix)
|
||||
func (t *Table[V]) Modify(netip.Prefix, cb func(V, bool) (V, bool))
|
||||
|
||||
func (t *Table[V]) Union(o *Table[V])
|
||||
func (t *Table[V]) Clone() *Table[V]
|
||||
func (t *Table[V]) InsertPersist(netip.Prefix, V) *Table[V]
|
||||
func (t *Table[V]) DeletePersist(netip.Prefix) *Table[V]
|
||||
func (t *Table[V]) ModifyPersist(netip.Prefix, cb func(V, bool) (V, bool)) *Table[V]
|
||||
|
||||
func (t *Table[V]) Contains(ip netip.Addr) bool
|
||||
func (t *Table[V]) Lookup(ip netip.Addr) (val V, ok bool)
|
||||
func (t *Table[V]) LookupPrefix(pfx netip.Prefix) (val V, ok bool)
|
||||
func (t *Table[V]) LookupPrefixLPM(pfx netip.Prefix) (lpm netip.Prefix, val V, ok bool)
|
||||
func (t *Table[V]) Clone() *Table[V]
|
||||
func (t *Table[V]) Union(o *Table[V])
|
||||
func (t *Table[V]) UnionPersist(o *Table[V]) *Table[V]
|
||||
|
||||
func (t *Table[V]) OverlapsPrefix(pfx netip.Prefix) bool
|
||||
func (t *Table[V]) OverlapsPrefix(netip.Prefix) bool
|
||||
|
||||
func (t *Table[V]) Overlaps(o *Table[V]) bool
|
||||
func (t *Table[V]) Overlaps4(o *Table[V]) bool
|
||||
func (t *Table[V]) Overlaps6(o *Table[V]) bool
|
||||
func (t *Table[V]) Overlaps(o *Table[V]) bool
|
||||
func (t *Table[V]) Overlaps4(o *Table[V]) bool
|
||||
func (t *Table[V]) Overlaps6(o *Table[V]) bool
|
||||
|
||||
func (t *Table[V]) Subnets(pfx netip.Prefix) iter.Seq2[netip.Prefix, V]
|
||||
func (t *Table[V]) Supernets(pfx netip.Prefix) iter.Seq2[netip.Prefix, V]
|
||||
func (t *Table[V]) Equal(o *Table[V]) bool
|
||||
|
||||
func (t *Table[V]) All() iter.Seq2[netip.Prefix, V]
|
||||
func (t *Table[V]) All4() iter.Seq2[netip.Prefix, V]
|
||||
func (t *Table[V]) All6() iter.Seq2[netip.Prefix, V]
|
||||
func (t *Table[V]) Subnets(netip.Prefix) iter.Seq2[netip.Prefix, V]
|
||||
func (t *Table[V]) Supernets(netip.Prefix) iter.Seq2[netip.Prefix, V]
|
||||
|
||||
func (t *Table[V]) AllSorted() iter.Seq2[netip.Prefix, V]
|
||||
func (t *Table[V]) AllSorted4() iter.Seq2[netip.Prefix, V]
|
||||
func (t *Table[V]) AllSorted6() iter.Seq2[netip.Prefix, V]
|
||||
func (t *Table[V]) All() iter.Seq2[netip.Prefix, V]
|
||||
func (t *Table[V]) All4() iter.Seq2[netip.Prefix, V]
|
||||
func (t *Table[V]) All6() iter.Seq2[netip.Prefix, V]
|
||||
|
||||
func (t *Table[V]) Size() int
|
||||
func (t *Table[V]) Size4() int
|
||||
func (t *Table[V]) Size6() int
|
||||
func (t *Table[V]) AllSorted() iter.Seq2[netip.Prefix, V]
|
||||
func (t *Table[V]) AllSorted4() iter.Seq2[netip.Prefix, V]
|
||||
func (t *Table[V]) AllSorted6() iter.Seq2[netip.Prefix, V]
|
||||
|
||||
func (t *Table[V]) String() string
|
||||
func (t *Table[V]) Fprint(w io.Writer) error
|
||||
func (t *Table[V]) MarshalText() ([]byte, error)
|
||||
func (t *Table[V]) MarshalJSON() ([]byte, error)
|
||||
func (t *Table[V]) Size() int
|
||||
func (t *Table[V]) Size4() int
|
||||
func (t *Table[V]) Size6() int
|
||||
|
||||
func (t *Table[V]) DumpList4() []DumpListNode[V]
|
||||
func (t *Table[V]) DumpList6() []DumpListNode[V]
|
||||
func (t *Table[V]) Fprint(w io.Writer) error
|
||||
func (t *Table[V]) MarshalText() ([]byte, error)
|
||||
func (t *Table[V]) MarshalJSON() ([]byte, error)
|
||||
|
||||
func (t *Table[V]) DumpList4() []DumpListNode[V]
|
||||
func (t *Table[V]) DumpList6() []DumpListNode[V]
|
||||
```
|
||||
|
||||
## benchmarks
|
||||
## Benchmarks
|
||||
|
||||
Please see the extensive [benchmarks](https://github.com/gaissmai/iprbench) comparing `bart` with other IP routing table implementations.
|
||||
Please see the extensive [benchmarks](https://github.com/gaissmai/iprbench)
|
||||
comparing `bart` with other IP routing table implementations.
|
||||
|
||||
Just a teaser, Contains and Lookups against the full Internet routing table with random IP address probes:
|
||||
Just a teaser, `Fast.Lookup` against the Tier1 full
|
||||
Internet routing table with random IP address probes:
|
||||
|
||||
```
|
||||
```text
|
||||
$ GOAMD64=v3 go test -run=xxx -bench=FullFastM/Lookup$ -cpu=1
|
||||
goos: linux
|
||||
goarch: amd64
|
||||
pkg: github.com/gaissmai/bart
|
||||
cpu: Intel(R) Core(TM) i5-8250U CPU @ 1.60GHz
|
||||
BenchmarkFullMatchV4/Contains 55906228 21.39 ns/op 0 B/op 0 allocs/op
|
||||
BenchmarkFullMatchV6/Contains 100000000 11.38 ns/op 0 B/op 0 allocs/op
|
||||
BenchmarkFullMissV4/Contains 52927538 22.69 ns/op 0 B/op 0 allocs/op
|
||||
BenchmarkFullMissV6/Contains 250386540 4.883 ns/op 0 B/op 0 allocs/op
|
||||
cpu: AMD Ryzen 7 PRO 4750U with Radeon Graphics
|
||||
BenchmarkFullFastMatch4/Lookup 129484557 9.274 ns/op
|
||||
BenchmarkFullFastMatch6/Lookup 65185531 17.99 ns/op
|
||||
BenchmarkFullFastMiss4/Lookup 92448910 12.81 ns/op
|
||||
BenchmarkFullFastMiss6/Lookup 64323817 18.66 ns/op
|
||||
PASS
|
||||
ok github.com/gaissmai/bart 11.291s
|
||||
|
||||
goos: linux
|
||||
goarch: amd64
|
||||
pkg: github.com/gaissmai/bart
|
||||
cpu: Intel(R) Core(TM) i5-8250U CPU @ 1.60GHz
|
||||
BenchmarkFullMatchV4/Lookup 53578766 21.77 ns/op 0 B/op 0 allocs/op
|
||||
BenchmarkFullMatchV6/Lookup 57054618 21.07 ns/op 0 B/op 0 allocs/op
|
||||
BenchmarkFullMissV4/Lookup 48289306 25.60 ns/op 0 B/op 0 allocs/op
|
||||
BenchmarkFullMissV6/Lookup 142917571 8.387 ns/op 0 B/op 0 allocs/op
|
||||
PASS
|
||||
ok github.com/gaissmai/bart 10.455s
|
||||
ok github.com/gaissmai/bart 6.647s
|
||||
```
|
||||
|
||||
## Compatibility Guarantees
|
||||
@@ -188,20 +284,20 @@ to be revisited to better support the use cases that the library solves for.
|
||||
|
||||
These occurrences are expected to be rare in frequency and the API is already quite stable.
|
||||
|
||||
## CONTRIBUTION
|
||||
## Contribution
|
||||
|
||||
Please open an issue for discussion before sending a pull request.
|
||||
|
||||
## CREDIT
|
||||
## Credit
|
||||
|
||||
Standing on the shoulders of giants.
|
||||
|
||||
Credits for many inspirations go to
|
||||
|
||||
- the clever guys at tailscale,
|
||||
- the clever folks at Tailscale,
|
||||
- to Daniel Lemire for his inspiring blog,
|
||||
- to Donald E. Knuth for the **ART** routing algorithm and
|
||||
- to Yoichi Hariguchi who deciphered it for us mere mortals
|
||||
- to Donald E. Knuth for the Allotment Routing Table [ART](https://github.com/hariguchi/art/tree/master/orig) algorithm and
|
||||
- to Yoichi Hariguchi who [deciphered it](https://github.com/hariguchi/art/blob/master/docs/art.pdf) for us mere mortals
|
||||
|
||||
And last but not least to the Go team who do a wonderful job!
|
||||
|
||||
|
||||
Reference in new issue
Block a user