Skip to content

Commit d0157ba

Browse files
studyzystudyzy
committed
style: Fix lint error
--story=1
1 parent dcf4358 commit d0157ba

13 files changed

+114
-49
lines changed

‎README.md‎

Lines changed: 39 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# shardingdb
22

3-
This package enables LevelDB to support sharding and concurrent reads/writes, and can be used as a drop-in replacement for LevelDB.
4-
3+
ShardingDB is an open-source, sharded database enhancing LevelDB with concurrent reads/writes support.
4+
It significantly improves performance, boosting PutData by 60x and GetData by 7x, making it an ideal drop-in replacement for LevelDB.
55
-----------
66

77
## Requirements
@@ -64,7 +64,7 @@ if err != nil {
6464
t.Fatal(err)
6565
}
6666
// Create a new sharding db
67-
sdb, err := NewShardingDb(Sha256Sharding, db1, db2)
67+
sdb, err := shardingdb.NewShardingDb(shardingdb.WithDbHandles(db1,db2), shardingdb.WithShardingFunc(MurmurSharding))
6868
...
6969
```
7070
@@ -80,57 +80,57 @@ generate data by command:
8080
go test -timeout 60m -run "TestCompareDbPerformance"
8181
```
8282
Test case: total 1000000 key-value pairs, 100 go routines, 100 key-value pairs per batch.
83-
83+
result means the time cost(second) of the whole operation.
8484
### 1. PutData
8585
8686
| Data Size | LevelDB | ShardingDB(3 folders) | ShardingDB(6 folders) | ShardingDB(encrypt 3 folders) |
8787
|:---------:|:-------:|:---------------------:|:---------------------:|:-----------------------------:|
88-
| 100B | 2.27s | 0.659s | 0.581s | 0.953s |
89-
| 200B | 4.45s | 1.07s | 0.683s | 1.9s |
90-
| 500B | 15.3s | 3.36s | 1.49s | 6.4s |
91-
| 1KB | 48.9s | 9.42s | 3.74s | 17.69s |
92-
| 10KB | 1117s | 351s | 123s | 308s |
88+
| 100B | 2.27 | 0.659 | 0.581 | 0.953 |
89+
| 200B | 4.45 | 1.07 | 0.683 | 1.9 |
90+
| 500B | 15.3 | 3.36 | 1.49 | 6.4 |
91+
| 1KB | 48.9 | 9.42 | 3.74 | 17.69 |
92+
| 10KB | 1117 | 351 | 123 | 308 |
9393
9494
### 2. GetData
9595
9696
| Data Size | LevelDB | ShardingDB(3 folders) | ShardingDB(6 folders) | ShardingDB(encrypt 3 folders) |
9797
|:---------:|:-------:|:---------------------:|:---------------------:|:-----------------------------:|
98-
| 100B | 2.23s | 1.25s | 1.02s | 1.86s |
99-
| 200B | 3.09s | 1.42s | 1.27s | 2.24s |
100-
| 500B | 4.17s | 1.91s | 1.62s | 3.73s |
101-
| 1KB | 7.97s | 2.37s | 2.26s | 4.53s |
102-
| 10KB | 12.75s | 9.54s | 11.03s | 13.85s |
98+
| 100B | 2.23 | 1.25 | 1.02 | 1.86 |
99+
| 200B | 3.09 | 1.42 | 1.27 | 2.24 |
100+
| 500B | 4.17 | 1.91 | 1.62 | 3.73 |
101+
| 1KB | 7.97 | 2.37 | 2.26 | 4.53 |
102+
| 10KB | 12.75 | 9.54 | 11.03 | 13.85 |
103103
104104
### 3. GetData not found
105105
106106
| Data Size | LevelDB | ShardingDB(3 folders) | ShardingDB(6 folders) | ShardingDB(encrypt 3 folders) |
107107
|:---------:|:-------:|:---------------------:|:---------------------:|:-----------------------------:|
108-
| 100B | 2.14s | 1.36s | 0.87s | 1.43s |
109-
| 200B | 2.07s | 1.47s | 0.9s | 1.6s |
110-
| 500B | 2.05s | 1.51s | 0.93s | 1.81s |
111-
| 1KB | 2.35s | 1.64s | 0.891s | 2.28s |
112-
| 10KB | 8.68s | 5.56s | 2.48s | 7.75s |
108+
| 100B | 2.14 | 1.36 | 0.87 | 1.43 |
109+
| 200B | 2.07 | 1.47 | 0.9 | 1.6 |
110+
| 500B | 2.05 | 1.51 | 0.93 | 1.81 |
111+
| 1KB | 2.35 | 1.64 | 0.891 | 2.28 |
112+
| 10KB | 8.68 | 5.56 | 2.48 | 7.75 |
113113
114114
115115
### 4. DeleteData
116116
117117
| Data Size | LevelDB | ShardingDB(3 folders) | ShardingDB(6 folders) | ShardingDB(encrypt 3 folders) |
118118
|:---------:|:-------:|:---------------------:|:---------------------:|:-----------------------------:|
119-
| 100B | 3.82s | 2.76s | 1.02s | 1.72s |
120-
| 200B | 3.81s | 1.71s | 1.02s | 1.74s |
121-
| 500B | 3.85s | 1.76s | 1.05s | 1.69s |
122-
| 1KB | 3.84s | 1.72s | 1.04s | 1.74s |
123-
| 10KB | 3.844s | 1.78s | 1.06s | 1.76s |
119+
| 100B | 3.82 | 2.76 | 1.02 | 1.72 |
120+
| 200B | 3.81 | 1.71 | 1.02 | 1.74 |
121+
| 500B | 3.85 | 1.76 | 1.05 | 1.69 |
122+
| 1KB | 3.84 | 1.72 | 1.04 | 1.74 |
123+
| 10KB | 3.844 | 1.78 | 1.06 | 1.76 |
124124
125125
### 5. Iterator
126126
127127
| Data Size | LevelDB | ShardingDB(3 folders) | ShardingDB(6 folders) | ShardingDB(encrypt 3 folders) |
128128
|:---------:|:-------:|:---------------------:|:---------------------:|:-----------------------------:|
129-
| 100B | 0.133s | 0.184s | 0.222s | 0.18s |
130-
| 200B | 0.151s | 0.246s | 0.246s | 0.191s |
131-
| 500B | 0.282s | 0.351s | 0.41s | 0.344s |
132-
| 1KB | 0.514s | 0.419s | 0.472s | 0.541s |
133-
| 10KB | 2.46s | 2.39s | 1.96s | 2.3s |
129+
| 100B | 0.133 | 0.184 | 0.222 | 0.18 |
130+
| 200B | 0.151 | 0.246 | 0.246 | 0.191 |
131+
| 500B | 0.282 | 0.351 | 0.41 | 0.344 |
132+
| 1KB | 0.514 | 0.419 | 0.472 | 0.541 |
133+
| 10KB | 2.46 | 2.39 | 1.96 | 2.3 |
134134
135135
136136
### 6. Sharding count compare
@@ -142,19 +142,19 @@ go test -timeout 60m -run "TestCompareShardingCountPerformance"
142142
#### 6.1 PutData
143143
| Data Size | ShardingDB(3 folders) | ShardingDB(6 folders) | ShardingDB(9 folders) | ShardingDB(30 folders) | ShardingDB(60 folders) |
144144
|:---------:|:---------------------:|:---------------------:|:---------------------:|:----------------------:|:----------------------:|
145-
| 100B | 0.659s | 0.581s | 0.506s | 0.564s | 0.728s |
146-
| 200B | 1.07s | 0.683s | 0.624s | 0.685s | 0.782s |
147-
| 500B | 3.36s | 1.49s | 1.20s | 1.18s | 1.21s |
148-
| 1KB | 9.42s | 3.74s | 2.33s | 1.92s | 1.96s |
149-
| 10KB | 351s | 123s | 54s | 26s | 18.2s |
145+
| 100B | 0.659 | 0.581 | 0.506 | 0.564 | 0.728 |
146+
| 200B | 1.07 | 0.683 | 0.624 | 0.685 | 0.782 |
147+
| 500B | 3.36 | 1.49 | 1.20 | 1.18 | 1.21 |
148+
| 1KB | 9.42 | 3.74 | 2.33 | 1.92 | 1.96 |
149+
| 10KB | 351 | 123 | 54 | 26 | 18.2 |
150150
#### 6.2 GetData
151151
| Data Size | ShardingDB(3 folders) | ShardingDB(6 folders) | ShardingDB(9 folders) | ShardingDB(30 folders) | ShardingDB(60 folders) |
152152
|:---------:|:---------------------:|:---------------------:|:---------------------:|:----------------------:|:----------------------:|
153-
| 100B | 1.25s | 1.02s | 1.03s | 0.343s | 0.366s |
154-
| 200B | 1.42s | 1.27s | 1.01s | 0.66s | 0.373s |
155-
| 500B | 1.91s | 1.62s | 1.21s | 0.96s | 1.34s |
156-
| 1KB | 2.37s | 2.26s | 1.83s | 1.18s | 1.19s |
157-
| 10KB | 9.54s | 11.03s | 7.67s | 4.8s | 3.4s |
153+
| 100B | 1.25 | 1.02 | 1.03 | 0.343 | 0.366 |
154+
| 200B | 1.42 | 1.27 | 1.01 | 0.66 | 0.373 |
155+
| 500B | 1.91 | 1.62 | 1.21 | 0.96 | 1.34 |
156+
| 1KB | 2.37 | 2.26 | 1.83 | 1.18 | 1.19 |
157+
| 10KB | 9.54 | 11.03 | 7.67 | 4.8 | 3.4 |
158158
159159
160160
Most interfaces are the same as [goleveldb](https://github.com/syndtr/goleveldb). For my interface definition, please refer to [DbHandle](https://github.com/studyzy/shardingdb/blob/main/interfaces.go).

‎batch.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,15 @@ package shardingdb
1818

1919
import "github.com/syndtr/goleveldb/leveldb"
2020

21+
// ShardingBatch is a batch of multiple db
2122
type ShardingBatch struct {
2223
batchHandles []*leveldb.Batch
2324
length uint16
2425
shardingFunc ShardingFunc
2526
encryptor Encryptor
2627
}
2728

29+
// NewShardingBatch returns a new ShardingBatch
2830
func NewShardingBatch(len uint16, shardingFunc ShardingFunc, e Encryptor) *ShardingBatch {
2931
batches := make([]*leveldb.Batch, len)
3032
for i := uint16(0); i < len; i++ {
@@ -38,6 +40,7 @@ func NewShardingBatch(len uint16, shardingFunc ShardingFunc, e Encryptor) *Shard
3840
}
3941
}
4042

43+
// Put sets the value for the given key
4144
func (s *ShardingBatch) Put(key, value []byte) {
4245
dbIndex := s.shardingFunc(key, s.length)
4346
if s.encryptor != nil && len(value) > 0 {
@@ -51,10 +54,13 @@ func (s *ShardingBatch) Put(key, value []byte) {
5154
}
5255
}
5356

57+
// Delete deletes the value for the given key
5458
func (s *ShardingBatch) Delete(key []byte) {
5559
dbIndex := s.shardingFunc(key, s.length)
5660
s.batchHandles[dbIndex].Delete(key)
5761
}
62+
63+
// GetSplitBatch returns a map of db index to batch
5864
func (s *ShardingBatch) GetSplitBatch() map[uint16]*leveldb.Batch {
5965
batches := make(map[uint16]*leveldb.Batch)
6066
for idx, dbHandle := range s.batchHandles {

‎encrypt.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,17 @@ import (
2424

2525
var _ Encryptor = (*AESCryptor)(nil)
2626

27+
// AESCryptor is a encryptor using AES
2728
type AESCryptor struct {
2829
key []byte
2930
}
3031

32+
// NewAESCryptor creates a new AESCryptor
3133
func NewAESCryptor(key []byte) *AESCryptor {
3234
return &AESCryptor{key: key}
3335
}
3436

37+
// Encrypt encrypts data with AES
3538
func (a *AESCryptor) Encrypt(data []byte) ([]byte, error) {
3639
block, err := aes.NewCipher(a.key)
3740
if err != nil {
@@ -56,6 +59,7 @@ func (a *AESCryptor) Encrypt(data []byte) ([]byte, error) {
5659
return ciphertext, nil
5760
}
5861

62+
// Decrypt decrypts data
5963
func (a *AESCryptor) Decrypt(data []byte) ([]byte, error) {
6064
block, err := aes.NewCipher(a.key)
6165
if err != nil {

‎interfaces.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"github.com/syndtr/goleveldb/leveldb/util"
2424
)
2525

26+
// CommonDbHandle is the interface that wraps the basic methods of a leveldb.DB
2627
type CommonDbHandle interface {
2728
// Get returns the value for the given key.
2829
// @param key
@@ -99,24 +100,41 @@ type ShardingDbHandle interface {
99100
// Resharding resharding the DB.
100101
// @return error
101102
Resharding() error
103+
// ShardCount returns the shard count of the DB.
102104
ShardCount() uint16
103105
}
104106

107+
// Transaction is the interface that wraps the basic methods of a leveldb.Transaction
105108
type Transaction interface {
109+
// Get returns the value for the given key.
106110
Get(key []byte, ro *opt.ReadOptions) ([]byte, error)
111+
// Has returns whether the DB does contains the given key.
107112
Has(key []byte, ro *opt.ReadOptions) (bool, error)
113+
// NewIterator returns an iterator for the latest snapshot of the DB.
108114
NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator
115+
// Put sets the value for the given key.
109116
Put(key, value []byte, wo *opt.WriteOptions) error
117+
// Delete deletes the value for the given key.
110118
Delete(key []byte, wo *opt.WriteOptions) error
119+
// Write writes the given batch to the DB.
111120
Write(b *leveldb.Batch, wo *opt.WriteOptions) error
121+
// Commit commits the transaction.
112122
Commit() error
123+
// Discard discards the transaction.
113124
Discard()
114125
}
126+
127+
// Snapshot is the interface that wraps the basic methods of a leveldb.Snapshot
115128
type Snapshot interface {
129+
// String returns a string representation of the snapshot
116130
String() string
131+
// Get returns the value for the given key
117132
Get(key []byte, ro *opt.ReadOptions) (value []byte, err error)
133+
// Has returns whether the DB does contains the given key
118134
Has(key []byte, ro *opt.ReadOptions) (ret bool, err error)
135+
// NewIterator returns an iterator for the latest snapshot of the DB
119136
NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator
137+
// Release releases the snapshot
120138
Release()
121139
}
122140

@@ -146,11 +164,19 @@ type Batch interface {
146164
// Reset resets the batch contents.
147165
Reset()
148166
}
167+
168+
// Logger is the interface that wraps the basic methods of a logger
149169
type Logger interface {
170+
// Debug logs a debug message.
150171
Debug(msg string)
172+
// Info logs an info message.
151173
Info(msg string)
152174
}
175+
176+
// Encryptor is the interface that wraps the basic methods of a encryptor
153177
type Encryptor interface {
178+
// Encrypt encrypts the given data.
154179
Encrypt(data []byte) ([]byte, error)
180+
// Decrypt decrypts the given data.
155181
Decrypt(data []byte) ([]byte, error)
156182
}

‎leveldb_interface.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,12 @@ import (
2020
"github.com/syndtr/goleveldb/leveldb"
2121
)
2222

23+
// LevelDbHandle is the interface that wraps the basic LevelDB methods.
2324
type LevelDbHandle interface {
2425
CommonDbHandle
26+
// GetSnapshot returns a new snapshot of the DB.
2527
GetSnapshot() (*leveldb.Snapshot, error)
28+
// OpenTransaction opens a transaction.
2629
OpenTransaction() (*leveldb.Transaction, error)
2730
}
2831

‎options.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,18 @@ package shardingdb
1818

1919
import "github.com/syndtr/goleveldb/leveldb"
2020

21+
// DbOption is used to set options for ShardingDb
2122
type DbOption func(db *ShardingDb)
2223

24+
// WithDbHandles sets dbHandles for ShardingDb
2325
func WithDbHandles(dbHandles ...LevelDbHandle) DbOption {
2426
return func(s *ShardingDb) {
2527
s.dbHandles = dbHandles
2628
s.length = uint16(len(dbHandles))
2729
}
2830
}
2931

32+
// WithDbPaths sets dbHandles for ShardingDb
3033
func WithDbPaths(paths ...string) DbOption {
3134
dbHandles := make([]LevelDbHandle, len(paths))
3235
for i := 0; i < len(paths); i++ {
@@ -41,18 +44,22 @@ func WithDbPaths(paths ...string) DbOption {
4144
s.length = uint16(len(dbHandles))
4245
}
4346
}
47+
48+
// WithShardingFunc sets shardingFunc for ShardingDb
4449
func WithShardingFunc(f ShardingFunc) DbOption {
4550
return func(s *ShardingDb) {
4651
s.shardingFunc = f
4752
}
4853
}
4954

55+
// WithLogger sets logger for ShardingDb
5056
func WithLogger(l Logger) DbOption {
5157
return func(s *ShardingDb) {
5258
s.logger = l
5359
}
5460
}
5561

62+
// WithEncryptor sets encryptor for ShardingDb
5663
func WithEncryptor(e Encryptor) DbOption {
5764
return func(s *ShardingDb) {
5865
s.encryptor = e

‎performance_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ func testDbPerformance(t *testing.T, db CommonDbHandle, testName string) {
252252
}
253253
wg.Wait()
254254
fmt.Printf("%s write batch[%d] thread[%d] loop[%d] cost:%v\n",
255-
testName, batchSize, thread, loop, time.Now().Sub(start))
255+
testName, batchSize, thread, loop, time.Since(start))
256256
//Test Get performance
257257
wg = sync.WaitGroup{}
258258
wg.Add(thread)
@@ -270,7 +270,7 @@ func testDbPerformance(t *testing.T, db CommonDbHandle, testName string) {
270270
}
271271
wg.Wait()
272272
fmt.Printf("%s get batch[%d] thread[%d] loop[%d] cost:%v\n",
273-
testName, batchSize, thread, loop, time.Now().Sub(start))
273+
testName, batchSize, thread, loop, time.Since(start))
274274
//Test Get not found performance
275275
wg = sync.WaitGroup{}
276276
wg.Add(thread)
@@ -288,7 +288,7 @@ func testDbPerformance(t *testing.T, db CommonDbHandle, testName string) {
288288
}
289289
wg.Wait()
290290
fmt.Printf("%s get not found batch[%d] thread[%d] loop[%d] cost:%v\n",
291-
testName, batchSize, thread, loop, time.Now().Sub(start))
291+
testName, batchSize, thread, loop, time.Since(start))
292292
//Test delete performance
293293
wg = sync.WaitGroup{}
294294
wg.Add(thread)
@@ -308,7 +308,7 @@ func testDbPerformance(t *testing.T, db CommonDbHandle, testName string) {
308308
}
309309
wg.Wait()
310310
fmt.Printf("%s delete batch[%d] thread[%d] loop[%d] cost:%v\n",
311-
testName, batchSize, thread, loop, time.Now().Sub(start))
311+
testName, batchSize, thread, loop, time.Since(start))
312312

313313
//Test Iterator performance
314314
wg = sync.WaitGroup{}
@@ -331,7 +331,7 @@ func testDbPerformance(t *testing.T, db CommonDbHandle, testName string) {
331331
}
332332
wg.Wait()
333333
fmt.Printf("%s iterator batch[%d] thread[%d] loop[%d] cost:%v\n",
334-
testName, batchSize, thread, loop, time.Now().Sub(start))
334+
testName, batchSize, thread, loop, time.Since(start))
335335

336336
}
337337

‎shardingdb.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
* limitations under the License.
1515
*/
1616

17+
// Package shardingdb provides a sharding db based on goleveldb
1718
package shardingdb
1819

1920
import (
@@ -30,6 +31,7 @@ import (
3031

3132
var _ ShardingDbHandle = (*ShardingDb)(nil)
3233

34+
// ShardingDb is a db of multiple db
3335
type ShardingDb struct {
3436
dbHandles []LevelDbHandle
3537
length uint16
@@ -39,6 +41,7 @@ type ShardingDb struct {
3941
encryptor Encryptor
4042
}
4143

44+
// ShardCount returns the number of shards
4245
func (sdb *ShardingDb) ShardCount() uint16 {
4346
return sdb.length
4447
}

‎shardingdb_main.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,6 @@ func NewShardingDb(options ...DbOption) (*ShardingDb, error) {
4747
sdb.shardingFunc = MurmurSharding
4848
}
4949
}
50-
5150
return sdb, nil
5251
}
5352

0 commit comments

Comments
 (0)