DB as a service
Fixes https://github.com/tendermint/tendermint/issues/1162
Databases as a service!
Can now access Databases as a remote service
via gRPC for performance and easy deployment.
The caveat is that each service is stateful in regards to
the DB i.e. each unique service uses only one unique DB
but nonetheless multiple clients can access it.
A full standalone example
```go
package main
import (
"bytes"
"context"
"log"
grpcdb "github.com/tendermint/tmlibs/grpcdb"
protodb "github.com/tendermint/tmlibs/proto"
)
func main() {
addr := ":8998"
go func() {
if err := grpcdb.BindRemoteDBServer(addr); err != nil {
log.Fatalf("BindRemoteDBServer: %v", err)
}
}()
client, err := grpcdb.NewClient(addr, false)
if err != nil {
log.Fatalf("Failed to create grpcDB client: %v", err)
}
ctx := context.Background()
// 1. Initialize the DB
in := &protodb.Init{
Type: "leveldb",
Name: "grpc-uno-test",
Dir: ".",
}
if _, err := client.Init(ctx, in); err != nil {
log.Fatalf("Init error: %v", err)
}
// 2. Now it can be used!
query1 := &protodb.Entity{Key: []byte("Project"), Value:
[]byte("Tmlibs-on-gRPC")}
if _, err := client.SetSync(ctx, query1); err != nil {
log.Fatalf("SetSync err: %v", err)
}
query2 := &protodb.Entity{Key: []byte("Project")}
read, err := client.Get(ctx, query2)
if err != nil {
log.Fatalf("Get err: %v", err)
}
if g, w := read.Value, []byte("Tmlibs-on-gRPC"); !bytes.Equal(g, w) {
log.Fatalf("got= (%q ==> % X)\nwant=(%q ==> % X)", g, g, w, w)
}
}
```
2018-03-12 00:39:34 -07:00
|
|
|
package grpcdb
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"net"
|
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"google.golang.org/grpc"
|
2018-05-07 23:16:06 +02:00
|
|
|
"google.golang.org/grpc/credentials"
|
DB as a service
Fixes https://github.com/tendermint/tendermint/issues/1162
Databases as a service!
Can now access Databases as a remote service
via gRPC for performance and easy deployment.
The caveat is that each service is stateful in regards to
the DB i.e. each unique service uses only one unique DB
but nonetheless multiple clients can access it.
A full standalone example
```go
package main
import (
"bytes"
"context"
"log"
grpcdb "github.com/tendermint/tmlibs/grpcdb"
protodb "github.com/tendermint/tmlibs/proto"
)
func main() {
addr := ":8998"
go func() {
if err := grpcdb.BindRemoteDBServer(addr); err != nil {
log.Fatalf("BindRemoteDBServer: %v", err)
}
}()
client, err := grpcdb.NewClient(addr, false)
if err != nil {
log.Fatalf("Failed to create grpcDB client: %v", err)
}
ctx := context.Background()
// 1. Initialize the DB
in := &protodb.Init{
Type: "leveldb",
Name: "grpc-uno-test",
Dir: ".",
}
if _, err := client.Init(ctx, in); err != nil {
log.Fatalf("Init error: %v", err)
}
// 2. Now it can be used!
query1 := &protodb.Entity{Key: []byte("Project"), Value:
[]byte("Tmlibs-on-gRPC")}
if _, err := client.SetSync(ctx, query1); err != nil {
log.Fatalf("SetSync err: %v", err)
}
query2 := &protodb.Entity{Key: []byte("Project")}
read, err := client.Get(ctx, query2)
if err != nil {
log.Fatalf("Get err: %v", err)
}
if g, w := read.Value, []byte("Tmlibs-on-gRPC"); !bytes.Equal(g, w) {
log.Fatalf("got= (%q ==> % X)\nwant=(%q ==> % X)", g, g, w, w)
}
}
```
2018-03-12 00:39:34 -07:00
|
|
|
|
|
|
|
"github.com/tendermint/tmlibs/db"
|
|
|
|
protodb "github.com/tendermint/tmlibs/proto"
|
|
|
|
)
|
|
|
|
|
2018-03-17 16:58:49 -07:00
|
|
|
// ListenAndServe is a blocking function that sets up a gRPC based
|
2018-03-16 14:54:15 -07:00
|
|
|
// server at the address supplied, with the gRPC options passed in.
|
|
|
|
// Normally in usage, invoke it in a goroutine like you would for http.ListenAndServe.
|
2018-05-08 15:45:49 +02:00
|
|
|
func ListenAndServe(addr, cert, key string, opts ...grpc.ServerOption) error {
|
DB as a service
Fixes https://github.com/tendermint/tendermint/issues/1162
Databases as a service!
Can now access Databases as a remote service
via gRPC for performance and easy deployment.
The caveat is that each service is stateful in regards to
the DB i.e. each unique service uses only one unique DB
but nonetheless multiple clients can access it.
A full standalone example
```go
package main
import (
"bytes"
"context"
"log"
grpcdb "github.com/tendermint/tmlibs/grpcdb"
protodb "github.com/tendermint/tmlibs/proto"
)
func main() {
addr := ":8998"
go func() {
if err := grpcdb.BindRemoteDBServer(addr); err != nil {
log.Fatalf("BindRemoteDBServer: %v", err)
}
}()
client, err := grpcdb.NewClient(addr, false)
if err != nil {
log.Fatalf("Failed to create grpcDB client: %v", err)
}
ctx := context.Background()
// 1. Initialize the DB
in := &protodb.Init{
Type: "leveldb",
Name: "grpc-uno-test",
Dir: ".",
}
if _, err := client.Init(ctx, in); err != nil {
log.Fatalf("Init error: %v", err)
}
// 2. Now it can be used!
query1 := &protodb.Entity{Key: []byte("Project"), Value:
[]byte("Tmlibs-on-gRPC")}
if _, err := client.SetSync(ctx, query1); err != nil {
log.Fatalf("SetSync err: %v", err)
}
query2 := &protodb.Entity{Key: []byte("Project")}
read, err := client.Get(ctx, query2)
if err != nil {
log.Fatalf("Get err: %v", err)
}
if g, w := read.Value, []byte("Tmlibs-on-gRPC"); !bytes.Equal(g, w) {
log.Fatalf("got= (%q ==> % X)\nwant=(%q ==> % X)", g, g, w, w)
}
}
```
2018-03-12 00:39:34 -07:00
|
|
|
ln, err := net.Listen("tcp", addr)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-05-07 23:16:06 +02:00
|
|
|
srv, err := NewServer(cert, key, opts...)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2018-03-17 16:58:49 -07:00
|
|
|
return srv.Serve(ln)
|
|
|
|
}
|
|
|
|
|
2018-05-08 15:45:49 +02:00
|
|
|
func NewServer(cert, key string, opts ...grpc.ServerOption) (*grpc.Server, error) {
|
2018-05-07 23:16:06 +02:00
|
|
|
creds, err := credentials.NewServerTLSFromFile(cert, key)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
opts = append(opts, grpc.Creds(creds))
|
DB as a service
Fixes https://github.com/tendermint/tendermint/issues/1162
Databases as a service!
Can now access Databases as a remote service
via gRPC for performance and easy deployment.
The caveat is that each service is stateful in regards to
the DB i.e. each unique service uses only one unique DB
but nonetheless multiple clients can access it.
A full standalone example
```go
package main
import (
"bytes"
"context"
"log"
grpcdb "github.com/tendermint/tmlibs/grpcdb"
protodb "github.com/tendermint/tmlibs/proto"
)
func main() {
addr := ":8998"
go func() {
if err := grpcdb.BindRemoteDBServer(addr); err != nil {
log.Fatalf("BindRemoteDBServer: %v", err)
}
}()
client, err := grpcdb.NewClient(addr, false)
if err != nil {
log.Fatalf("Failed to create grpcDB client: %v", err)
}
ctx := context.Background()
// 1. Initialize the DB
in := &protodb.Init{
Type: "leveldb",
Name: "grpc-uno-test",
Dir: ".",
}
if _, err := client.Init(ctx, in); err != nil {
log.Fatalf("Init error: %v", err)
}
// 2. Now it can be used!
query1 := &protodb.Entity{Key: []byte("Project"), Value:
[]byte("Tmlibs-on-gRPC")}
if _, err := client.SetSync(ctx, query1); err != nil {
log.Fatalf("SetSync err: %v", err)
}
query2 := &protodb.Entity{Key: []byte("Project")}
read, err := client.Get(ctx, query2)
if err != nil {
log.Fatalf("Get err: %v", err)
}
if g, w := read.Value, []byte("Tmlibs-on-gRPC"); !bytes.Equal(g, w) {
log.Fatalf("got= (%q ==> % X)\nwant=(%q ==> % X)", g, g, w, w)
}
}
```
2018-03-12 00:39:34 -07:00
|
|
|
srv := grpc.NewServer(opts...)
|
|
|
|
protodb.RegisterDBServer(srv, new(server))
|
2018-05-07 23:16:06 +02:00
|
|
|
return srv, nil
|
DB as a service
Fixes https://github.com/tendermint/tendermint/issues/1162
Databases as a service!
Can now access Databases as a remote service
via gRPC for performance and easy deployment.
The caveat is that each service is stateful in regards to
the DB i.e. each unique service uses only one unique DB
but nonetheless multiple clients can access it.
A full standalone example
```go
package main
import (
"bytes"
"context"
"log"
grpcdb "github.com/tendermint/tmlibs/grpcdb"
protodb "github.com/tendermint/tmlibs/proto"
)
func main() {
addr := ":8998"
go func() {
if err := grpcdb.BindRemoteDBServer(addr); err != nil {
log.Fatalf("BindRemoteDBServer: %v", err)
}
}()
client, err := grpcdb.NewClient(addr, false)
if err != nil {
log.Fatalf("Failed to create grpcDB client: %v", err)
}
ctx := context.Background()
// 1. Initialize the DB
in := &protodb.Init{
Type: "leveldb",
Name: "grpc-uno-test",
Dir: ".",
}
if _, err := client.Init(ctx, in); err != nil {
log.Fatalf("Init error: %v", err)
}
// 2. Now it can be used!
query1 := &protodb.Entity{Key: []byte("Project"), Value:
[]byte("Tmlibs-on-gRPC")}
if _, err := client.SetSync(ctx, query1); err != nil {
log.Fatalf("SetSync err: %v", err)
}
query2 := &protodb.Entity{Key: []byte("Project")}
read, err := client.Get(ctx, query2)
if err != nil {
log.Fatalf("Get err: %v", err)
}
if g, w := read.Value, []byte("Tmlibs-on-gRPC"); !bytes.Equal(g, w) {
log.Fatalf("got= (%q ==> % X)\nwant=(%q ==> % X)", g, g, w, w)
}
}
```
2018-03-12 00:39:34 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
type server struct {
|
|
|
|
mu sync.Mutex
|
|
|
|
db db.DB
|
|
|
|
}
|
|
|
|
|
|
|
|
var _ protodb.DBServer = (*server)(nil)
|
|
|
|
|
2018-03-16 14:54:15 -07:00
|
|
|
// Init initializes the server's database. Only one type of database
|
|
|
|
// can be initialized per server.
|
|
|
|
//
|
|
|
|
// Dir is the directory on the file system in which the DB will be stored(if backed by disk) (TODO: remove)
|
|
|
|
//
|
|
|
|
// Name is representative filesystem entry's basepath
|
|
|
|
//
|
|
|
|
// Type can be either one of:
|
|
|
|
// * cleveldb (if built with gcc enabled)
|
|
|
|
// * fsdb
|
|
|
|
// * memdB
|
|
|
|
// * leveldb
|
|
|
|
// See https://godoc.org/github.com/tendermint/tmlibs/db#DBBackendType
|
DB as a service
Fixes https://github.com/tendermint/tendermint/issues/1162
Databases as a service!
Can now access Databases as a remote service
via gRPC for performance and easy deployment.
The caveat is that each service is stateful in regards to
the DB i.e. each unique service uses only one unique DB
but nonetheless multiple clients can access it.
A full standalone example
```go
package main
import (
"bytes"
"context"
"log"
grpcdb "github.com/tendermint/tmlibs/grpcdb"
protodb "github.com/tendermint/tmlibs/proto"
)
func main() {
addr := ":8998"
go func() {
if err := grpcdb.BindRemoteDBServer(addr); err != nil {
log.Fatalf("BindRemoteDBServer: %v", err)
}
}()
client, err := grpcdb.NewClient(addr, false)
if err != nil {
log.Fatalf("Failed to create grpcDB client: %v", err)
}
ctx := context.Background()
// 1. Initialize the DB
in := &protodb.Init{
Type: "leveldb",
Name: "grpc-uno-test",
Dir: ".",
}
if _, err := client.Init(ctx, in); err != nil {
log.Fatalf("Init error: %v", err)
}
// 2. Now it can be used!
query1 := &protodb.Entity{Key: []byte("Project"), Value:
[]byte("Tmlibs-on-gRPC")}
if _, err := client.SetSync(ctx, query1); err != nil {
log.Fatalf("SetSync err: %v", err)
}
query2 := &protodb.Entity{Key: []byte("Project")}
read, err := client.Get(ctx, query2)
if err != nil {
log.Fatalf("Get err: %v", err)
}
if g, w := read.Value, []byte("Tmlibs-on-gRPC"); !bytes.Equal(g, w) {
log.Fatalf("got= (%q ==> % X)\nwant=(%q ==> % X)", g, g, w, w)
}
}
```
2018-03-12 00:39:34 -07:00
|
|
|
func (s *server) Init(ctx context.Context, in *protodb.Init) (*protodb.Entity, error) {
|
|
|
|
s.mu.Lock()
|
|
|
|
defer s.mu.Unlock()
|
|
|
|
|
|
|
|
s.db = db.NewDB(in.Name, db.DBBackendType(in.Type), in.Dir)
|
2018-05-07 22:12:26 +02:00
|
|
|
return &protodb.Entity{CreatedAt: time.Now().Unix()}, nil
|
DB as a service
Fixes https://github.com/tendermint/tendermint/issues/1162
Databases as a service!
Can now access Databases as a remote service
via gRPC for performance and easy deployment.
The caveat is that each service is stateful in regards to
the DB i.e. each unique service uses only one unique DB
but nonetheless multiple clients can access it.
A full standalone example
```go
package main
import (
"bytes"
"context"
"log"
grpcdb "github.com/tendermint/tmlibs/grpcdb"
protodb "github.com/tendermint/tmlibs/proto"
)
func main() {
addr := ":8998"
go func() {
if err := grpcdb.BindRemoteDBServer(addr); err != nil {
log.Fatalf("BindRemoteDBServer: %v", err)
}
}()
client, err := grpcdb.NewClient(addr, false)
if err != nil {
log.Fatalf("Failed to create grpcDB client: %v", err)
}
ctx := context.Background()
// 1. Initialize the DB
in := &protodb.Init{
Type: "leveldb",
Name: "grpc-uno-test",
Dir: ".",
}
if _, err := client.Init(ctx, in); err != nil {
log.Fatalf("Init error: %v", err)
}
// 2. Now it can be used!
query1 := &protodb.Entity{Key: []byte("Project"), Value:
[]byte("Tmlibs-on-gRPC")}
if _, err := client.SetSync(ctx, query1); err != nil {
log.Fatalf("SetSync err: %v", err)
}
query2 := &protodb.Entity{Key: []byte("Project")}
read, err := client.Get(ctx, query2)
if err != nil {
log.Fatalf("Get err: %v", err)
}
if g, w := read.Value, []byte("Tmlibs-on-gRPC"); !bytes.Equal(g, w) {
log.Fatalf("got= (%q ==> % X)\nwant=(%q ==> % X)", g, g, w, w)
}
}
```
2018-03-12 00:39:34 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
func (s *server) Delete(ctx context.Context, in *protodb.Entity) (*protodb.Nothing, error) {
|
|
|
|
s.db.Delete(in.Key)
|
|
|
|
return nothing, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
var nothing = new(protodb.Nothing)
|
2018-03-16 14:54:15 -07:00
|
|
|
|
DB as a service
Fixes https://github.com/tendermint/tendermint/issues/1162
Databases as a service!
Can now access Databases as a remote service
via gRPC for performance and easy deployment.
The caveat is that each service is stateful in regards to
the DB i.e. each unique service uses only one unique DB
but nonetheless multiple clients can access it.
A full standalone example
```go
package main
import (
"bytes"
"context"
"log"
grpcdb "github.com/tendermint/tmlibs/grpcdb"
protodb "github.com/tendermint/tmlibs/proto"
)
func main() {
addr := ":8998"
go func() {
if err := grpcdb.BindRemoteDBServer(addr); err != nil {
log.Fatalf("BindRemoteDBServer: %v", err)
}
}()
client, err := grpcdb.NewClient(addr, false)
if err != nil {
log.Fatalf("Failed to create grpcDB client: %v", err)
}
ctx := context.Background()
// 1. Initialize the DB
in := &protodb.Init{
Type: "leveldb",
Name: "grpc-uno-test",
Dir: ".",
}
if _, err := client.Init(ctx, in); err != nil {
log.Fatalf("Init error: %v", err)
}
// 2. Now it can be used!
query1 := &protodb.Entity{Key: []byte("Project"), Value:
[]byte("Tmlibs-on-gRPC")}
if _, err := client.SetSync(ctx, query1); err != nil {
log.Fatalf("SetSync err: %v", err)
}
query2 := &protodb.Entity{Key: []byte("Project")}
read, err := client.Get(ctx, query2)
if err != nil {
log.Fatalf("Get err: %v", err)
}
if g, w := read.Value, []byte("Tmlibs-on-gRPC"); !bytes.Equal(g, w) {
log.Fatalf("got= (%q ==> % X)\nwant=(%q ==> % X)", g, g, w, w)
}
}
```
2018-03-12 00:39:34 -07:00
|
|
|
func (s *server) DeleteSync(ctx context.Context, in *protodb.Entity) (*protodb.Nothing, error) {
|
|
|
|
s.db.DeleteSync(in.Key)
|
|
|
|
return nothing, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *server) Get(ctx context.Context, in *protodb.Entity) (*protodb.Entity, error) {
|
|
|
|
value := s.db.Get(in.Key)
|
|
|
|
return &protodb.Entity{Value: value}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *server) GetStream(ds protodb.DB_GetStreamServer) error {
|
|
|
|
// Receive routine
|
|
|
|
responsesChan := make(chan *protodb.Entity)
|
|
|
|
go func() {
|
|
|
|
defer close(responsesChan)
|
|
|
|
ctx := context.Background()
|
|
|
|
for {
|
|
|
|
in, err := ds.Recv()
|
|
|
|
if err != nil {
|
|
|
|
responsesChan <- &protodb.Entity{Err: err.Error()}
|
|
|
|
return
|
|
|
|
}
|
|
|
|
out, err := s.Get(ctx, in)
|
|
|
|
if err != nil {
|
|
|
|
if out == nil {
|
|
|
|
out = new(protodb.Entity)
|
|
|
|
out.Key = in.Key
|
|
|
|
}
|
|
|
|
out.Err = err.Error()
|
|
|
|
responsesChan <- out
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
// Otherwise continue on
|
|
|
|
responsesChan <- out
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
// Send routine, block until we return
|
|
|
|
for out := range responsesChan {
|
|
|
|
if err := ds.Send(out); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *server) Has(ctx context.Context, in *protodb.Entity) (*protodb.Entity, error) {
|
|
|
|
exists := s.db.Has(in.Key)
|
|
|
|
return &protodb.Entity{Exists: exists}, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *server) Set(ctx context.Context, in *protodb.Entity) (*protodb.Nothing, error) {
|
|
|
|
s.db.Set(in.Key, in.Value)
|
|
|
|
return nothing, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *server) SetSync(ctx context.Context, in *protodb.Entity) (*protodb.Nothing, error) {
|
|
|
|
s.db.SetSync(in.Key, in.Value)
|
|
|
|
return nothing, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *server) Iterator(query *protodb.Entity, dis protodb.DB_IteratorServer) error {
|
|
|
|
it := s.db.Iterator(query.Start, query.End)
|
|
|
|
return s.handleIterator(it, dis.Send)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *server) handleIterator(it db.Iterator, sendFunc func(*protodb.Iterator) error) error {
|
|
|
|
for it.Valid() {
|
|
|
|
start, end := it.Domain()
|
|
|
|
out := &protodb.Iterator{
|
2018-05-07 22:12:26 +02:00
|
|
|
Domain: &protodb.Domain{Start: start, End: end},
|
DB as a service
Fixes https://github.com/tendermint/tendermint/issues/1162
Databases as a service!
Can now access Databases as a remote service
via gRPC for performance and easy deployment.
The caveat is that each service is stateful in regards to
the DB i.e. each unique service uses only one unique DB
but nonetheless multiple clients can access it.
A full standalone example
```go
package main
import (
"bytes"
"context"
"log"
grpcdb "github.com/tendermint/tmlibs/grpcdb"
protodb "github.com/tendermint/tmlibs/proto"
)
func main() {
addr := ":8998"
go func() {
if err := grpcdb.BindRemoteDBServer(addr); err != nil {
log.Fatalf("BindRemoteDBServer: %v", err)
}
}()
client, err := grpcdb.NewClient(addr, false)
if err != nil {
log.Fatalf("Failed to create grpcDB client: %v", err)
}
ctx := context.Background()
// 1. Initialize the DB
in := &protodb.Init{
Type: "leveldb",
Name: "grpc-uno-test",
Dir: ".",
}
if _, err := client.Init(ctx, in); err != nil {
log.Fatalf("Init error: %v", err)
}
// 2. Now it can be used!
query1 := &protodb.Entity{Key: []byte("Project"), Value:
[]byte("Tmlibs-on-gRPC")}
if _, err := client.SetSync(ctx, query1); err != nil {
log.Fatalf("SetSync err: %v", err)
}
query2 := &protodb.Entity{Key: []byte("Project")}
read, err := client.Get(ctx, query2)
if err != nil {
log.Fatalf("Get err: %v", err)
}
if g, w := read.Value, []byte("Tmlibs-on-gRPC"); !bytes.Equal(g, w) {
log.Fatalf("got= (%q ==> % X)\nwant=(%q ==> % X)", g, g, w, w)
}
}
```
2018-03-12 00:39:34 -07:00
|
|
|
Valid: it.Valid(),
|
|
|
|
Key: it.Key(),
|
|
|
|
Value: it.Value(),
|
|
|
|
}
|
|
|
|
if err := sendFunc(out); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Finally move the iterator forward
|
|
|
|
it.Next()
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *server) ReverseIterator(query *protodb.Entity, dis protodb.DB_ReverseIteratorServer) error {
|
|
|
|
it := s.db.ReverseIterator(query.Start, query.End)
|
|
|
|
return s.handleIterator(it, dis.Send)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (s *server) Stats(context.Context, *protodb.Nothing) (*protodb.Stats, error) {
|
|
|
|
stats := s.db.Stats()
|
|
|
|
return &protodb.Stats{Data: stats, TimeAt: time.Now().Unix()}, nil
|
|
|
|
}
|