From 304be4ec2fe153fac1a4a240c40f368c4763d760 Mon Sep 17 00:00:00 2001 From: rigel rozanski Date: Tue, 6 Jun 2017 04:00:36 -0400 Subject: [PATCH 01/37] date parse functionality --- common/date.go | 67 +++++++++++++++++++++++++++++++++++++++ common/date_test.go | 76 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 common/date.go create mode 100644 common/date_test.go diff --git a/common/date.go b/common/date.go new file mode 100644 index 00000000..05f207f7 --- /dev/null +++ b/common/date.go @@ -0,0 +1,67 @@ +package common + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" +) + +// ParseDate parses a date string of the format YYYY-MM-DD +func ParseDate(date string) (t time.Time, err error) { + + //get the time of invoice + str := strings.Split(date, "-") + var ymd = []int{} + for _, i := range str { + j, err := strconv.Atoi(i) + if err != nil { + return t, err + } + ymd = append(ymd, j) + } + if len(ymd) != 3 { + return t, fmt.Errorf("Bad date parsing, not 3 segments") //never stack trace + } + if ymd[1] < 1 || ymd[1] > 12 { + return t, fmt.Errorf("Month not between 1 and 12") //never stack trace + } + if ymd[2] > 31 { + return t, fmt.Errorf("Day over 31") //never stack trace + } + + t = time.Date(ymd[0], time.Month(ymd[1]), ymd[2], 0, 0, 0, 0, time.UTC) + + return t, nil +} + +// ParseDateRange parses a date range string of the format start:end +// where the start and end date are of the format YYYY-MM-DD. +// The parsed dates are *time.Time and will return nil pointers for +// unbounded dates, ex: +// unbounded start: :2000-12-31 +// unbounded end: 2000-12-31: +func ParseDateRange(dateRange string) (startDate, endDate *time.Time, err error) { + dates := strings.Split(dateRange, ":") + if len(dates) != 2 { + return nil, nil, errors.New("bad date range, must be in format date:date") + } + parseDate := func(date string) (*time.Time, error) { + if len(date) == 0 { + return nil, nil + } + d, err := ParseDate(date) + return &d, err + } + startDate, err = parseDate(dates[0]) + if err != nil { + return nil, nil, err + } + endDate, err = parseDate(dates[1]) + if err != nil { + return nil, nil, err + } + return +} diff --git a/common/date_test.go b/common/date_test.go new file mode 100644 index 00000000..42fd91aa --- /dev/null +++ b/common/date_test.go @@ -0,0 +1,76 @@ +package common + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +var ( + date = time.Date(2015, time.Month(12), 31, 0, 0, 0, 0, time.UTC) + date2 = time.Date(2016, time.Month(12), 31, 0, 0, 0, 0, time.UTC) +) + +func TestParseDate(t *testing.T) { + assert := assert.New(t) + + var testDates = []struct { + dateStr string + date time.Time + errNil bool + }{ + {"2015-12-31", date, true}, + {"2015-31-12", date, false}, + {"12-31-2015", date, false}, + {"31-12-2015", date, false}, + } + + for _, test := range testDates { + parsed, err := ParseDate(test.dateStr) + switch test.errNil { + case true: + assert.Nil(err) + assert.True(parsed.Equal(test.date), "parsed: %v, want %v", parsed, test.date) + case false: + assert.NotNil(err, "parsed %v, expected err %v", parsed, err) + } + } +} + +func TestParseDateRange(t *testing.T) { + assert := assert.New(t) + + var testDates = []struct { + dateStr string + start *time.Time + end *time.Time + errNil bool + }{ + {"2015-12-31:2016-12-31", &date, &date2, true}, + {"2015-12-31:", &date, nil, true}, + {":2016-12-31", nil, &date2, true}, + {"2016-12-31", nil, nil, false}, + {"2016-31-12:", nil, nil, false}, + {":2016-31-12", nil, nil, false}, + } + + for _, test := range testDates { + start, end, err := ParseDateRange(test.dateStr) + switch test.errNil { + case true: + assert.Nil(err) + testPtr := func(want, have *time.Time) { + if want == nil { + assert.Nil(have) + } else { + assert.True((*have).Equal(*want)) + } + } + testPtr(test.start, start) + testPtr(test.end, end) + case false: + assert.NotNil(err) + } + } +} From f913ed8134448f65f572014b90ca78b24c12c03a Mon Sep 17 00:00:00 2001 From: rigel rozanski Date: Tue, 6 Jun 2017 16:08:57 -0400 Subject: [PATCH 02/37] date simplify --- common/date.go | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/common/date.go b/common/date.go index 05f207f7..7b7b05b2 100644 --- a/common/date.go +++ b/common/date.go @@ -1,8 +1,6 @@ package common import ( - "fmt" - "strconv" "strings" "time" @@ -11,30 +9,8 @@ import ( // ParseDate parses a date string of the format YYYY-MM-DD func ParseDate(date string) (t time.Time, err error) { - - //get the time of invoice - str := strings.Split(date, "-") - var ymd = []int{} - for _, i := range str { - j, err := strconv.Atoi(i) - if err != nil { - return t, err - } - ymd = append(ymd, j) - } - if len(ymd) != 3 { - return t, fmt.Errorf("Bad date parsing, not 3 segments") //never stack trace - } - if ymd[1] < 1 || ymd[1] > 12 { - return t, fmt.Errorf("Month not between 1 and 12") //never stack trace - } - if ymd[2] > 31 { - return t, fmt.Errorf("Day over 31") //never stack trace - } - - t = time.Date(ymd[0], time.Month(ymd[1]), ymd[2], 0, 0, 0, 0, time.UTC) - - return t, nil + layout := "2006-01-02" //this represents YYYY-MM-DD + return time.Parse(layout, date) } // ParseDateRange parses a date range string of the format start:end From c76dca0456bb38c16c10bc69088d0bf9ace157c3 Mon Sep 17 00:00:00 2001 From: rigel rozanski Date: Fri, 26 May 2017 13:48:37 -0400 Subject: [PATCH 03/37] cli stderr output for Execute --- cli/setup.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/setup.go b/cli/setup.go index 21b29a49..30fb5e78 100644 --- a/cli/setup.go +++ b/cli/setup.go @@ -93,9 +93,9 @@ func (e Executor) Execute() error { if err != nil { // TODO: something cooler with log-levels if viper.GetBool(TraceFlag) { - fmt.Printf("ERROR: %+v\n", err) + fmt.Fprintf(os.Stderr, "ERROR: %+v\n", err) } else { - fmt.Println("ERROR:", err.Error()) + fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) } } return err From 33d0dd0bfc3451643a78eb267faafd704c224a79 Mon Sep 17 00:00:00 2001 From: rigel rozanski Date: Tue, 6 Jun 2017 18:36:28 -0400 Subject: [PATCH 04/37] add stderr to tests --- .gitignore | 1 + cli/helper.go | 33 ++++++++++++++++++++++----------- cli/setup_test.go | 6 ++++-- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 62f28681..6e098685 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +*.swp vendor .glide diff --git a/cli/helper.go b/cli/helper.go index 79654bc3..959f8a5f 100644 --- a/cli/helper.go +++ b/cli/helper.go @@ -57,28 +57,39 @@ func RunWithArgs(cmd Executable, args []string, env map[string]string) error { // RunCaptureWithArgs executes the given command with the specified command line args // and environmental variables set. It returns whatever was writen to // stdout along with any error returned from cmd.Execute() -func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (output string, err error) { - old := os.Stdout // keep backup of the real stdout - r, w, _ := os.Pipe() - os.Stdout = w +func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (stdout, stderr string, err error) { + oldout, olderr := os.Stdout, os.Stderr // keep backup of the real stdout + rOut, wOut, _ := os.Pipe() + rErr, wErr, _ := os.Pipe() + os.Stdout, os.Stderr = wOut, wErr defer func() { - os.Stdout = old // restoring the real stdout + os.Stdout, os.Stderr = oldout, olderr // restoring the real stdout }() - outC := make(chan string) // copy the output in a separate goroutine so printing can't block indefinitely + outC := make(chan string) go func() { var buf bytes.Buffer - // io.Copy will end when we call w.Close() below - io.Copy(&buf, r) + // io.Copy will end when we call wOut.Close() below + io.Copy(&buf, rOut) outC <- buf.String() }() + errC := make(chan string) + go func() { + var buf bytes.Buffer + // io.Copy will end when we call wErr.Close() below + io.Copy(&buf, rErr) + errC <- buf.String() + }() + // now run the command err = RunWithArgs(cmd, args, env) // and grab the stdout to return - w.Close() - output = <-outC - return output, err + wOut.Close() + wErr.Close() + stdout = <-outC + stderr = <-errC + return stdout, stderr, err } diff --git a/cli/setup_test.go b/cli/setup_test.go index 8fb4ce14..791bc799 100644 --- a/cli/setup_test.go +++ b/cli/setup_test.go @@ -212,9 +212,11 @@ func TestSetupTrace(t *testing.T) { viper.Reset() args := append([]string{cmd.Use}, tc.args...) - out, err := RunCaptureWithArgs(cmd, args, tc.env) + stdout, stderr, err := RunCaptureWithArgs(cmd, args, tc.env) require.NotNil(err, i) - msg := strings.Split(out, "\n") + require.Equal("", stdout, i) + require.NotEqual("", stderr, i) + msg := strings.Split(stderr, "\n") desired := fmt.Sprintf("ERROR: %s", tc.expected) assert.Equal(desired, msg[0], i) if tc.long && assert.True(len(msg) > 2, i) { From 94c0172618a5d2d856edd2568034dd6b3b69b6c0 Mon Sep 17 00:00:00 2001 From: rigel rozanski Date: Wed, 7 Jun 2017 05:05:11 -0400 Subject: [PATCH 05/37] doc update --- cli/helper.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cli/helper.go b/cli/helper.go index 959f8a5f..12f94ec7 100644 --- a/cli/helper.go +++ b/cli/helper.go @@ -54,9 +54,10 @@ func RunWithArgs(cmd Executable, args []string, env map[string]string) error { return cmd.Execute() } -// RunCaptureWithArgs executes the given command with the specified command line args -// and environmental variables set. It returns whatever was writen to -// stdout along with any error returned from cmd.Execute() +// RunCaptureWithArgs executes the given command with the specified command +// line args and environmental variables set. It returns string fields +// representing output written to stdout and stderr, additionally any error +// from cmd.Execute() is also returned func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (stdout, stderr string, err error) { oldout, olderr := os.Stdout, os.Stderr // keep backup of the real stdout rOut, wOut, _ := os.Pipe() From 0a3a08a3bc3d3397c259dc79f6e9bc15dac2bba2 Mon Sep 17 00:00:00 2001 From: rigel rozanski Date: Sat, 17 Jun 2017 18:35:05 -0400 Subject: [PATCH 06/37] stderr PR revisions --- CHANGELOG.md | 6 ++++++ cli/helper.go | 27 ++++++++++++--------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a97aa128..c5b2a155 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Develop-Branch changes (unreleased) + +BREAKING CHANGES: + +- [run] NewBaseService takes the new logger + ## 0.2.1 (June 2, 2017) FEATURES: diff --git a/cli/helper.go b/cli/helper.go index 12f94ec7..8bdb9e49 100644 --- a/cli/helper.go +++ b/cli/helper.go @@ -68,21 +68,18 @@ func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (s }() // copy the output in a separate goroutine so printing can't block indefinitely - outC := make(chan string) - go func() { - var buf bytes.Buffer - // io.Copy will end when we call wOut.Close() below - io.Copy(&buf, rOut) - outC <- buf.String() - }() - - errC := make(chan string) - go func() { - var buf bytes.Buffer - // io.Copy will end when we call wErr.Close() below - io.Copy(&buf, rErr) - errC <- buf.String() - }() + copyStd := func(reader *os.File) *(chan string) { + stdC := make(chan string) + go func() { + var buf bytes.Buffer + // io.Copy will end when we call reader.Close() below + io.Copy(&buf, *reader) + stdC <- buf.String() + }() + return stdC + } + outC := copyStd(&rOut) + errC := copyStd(&rErr) // now run the command err = RunWithArgs(cmd, args, env) From 34bcb30f1cb9d14d141fe02be45ced9b082fd403 Mon Sep 17 00:00:00 2001 From: rigel rozanski Date: Tue, 20 Jun 2017 16:40:32 -0400 Subject: [PATCH 07/37] changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c5b2a155..3db410cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,14 @@ # Changelog + ## Develop-Branch changes (unreleased) BREAKING CHANGES: - [run] NewBaseService takes the new logger +- [cli] RunCaptureWithArgs now captures stderr and stdout + - +func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (stdout, stderr string, err error) + - -func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (output string, err error) ## 0.2.1 (June 2, 2017) From f3eaf9b8703db67ddae0c7979a1610d340d6d382 Mon Sep 17 00:00:00 2001 From: rigel rozanski Date: Tue, 20 Jun 2017 16:52:22 -0400 Subject: [PATCH 08/37] quickfix --- cli/helper.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cli/helper.go b/cli/helper.go index 8bdb9e49..845c17db 100644 --- a/cli/helper.go +++ b/cli/helper.go @@ -73,13 +73,13 @@ func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (s go func() { var buf bytes.Buffer // io.Copy will end when we call reader.Close() below - io.Copy(&buf, *reader) + io.Copy(&buf, reader) stdC <- buf.String() }() - return stdC + return &stdC } - outC := copyStd(&rOut) - errC := copyStd(&rErr) + outC := copyStd(rOut) + errC := copyStd(rErr) // now run the command err = RunWithArgs(cmd, args, env) @@ -87,7 +87,7 @@ func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (s // and grab the stdout to return wOut.Close() wErr.Close() - stdout = <-outC - stderr = <-errC + stdout = <-*outC + stderr = <-*errC return stdout, stderr, err } From cc364b14e20965a82a59ba2061675eb98dd818c9 Mon Sep 17 00:00:00 2001 From: rigel rozanski Date: Tue, 20 Jun 2017 17:18:55 -0400 Subject: [PATCH 09/37] changelog and PR changes --- CHANGELOG.md | 7 ++++++ common/date.go | 28 +++++++++++------------ common/date_test.go | 56 +++++++++++---------------------------------- 3 files changed, 34 insertions(+), 57 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a97aa128..4261c466 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog + +## Develop-Branch changes (unreleased) + +FEATURES: + +- [common] Date range parsing from string (ex. "2015-12-31:2017-12-31") + ## 0.2.1 (June 2, 2017) FEATURES: diff --git a/common/date.go b/common/date.go index 7b7b05b2..e017a4b4 100644 --- a/common/date.go +++ b/common/date.go @@ -7,37 +7,37 @@ import ( "github.com/pkg/errors" ) -// ParseDate parses a date string of the format YYYY-MM-DD -func ParseDate(date string) (t time.Time, err error) { - layout := "2006-01-02" //this represents YYYY-MM-DD - return time.Parse(layout, date) -} +// TimeLayout helps to parse a date string of the format YYYY-MM-DD +// Intended to be used with the following function: +// time.Parse(TimeLayout, date) +var TimeLayout = "2006-01-02" //this represents YYYY-MM-DD // ParseDateRange parses a date range string of the format start:end // where the start and end date are of the format YYYY-MM-DD. -// The parsed dates are *time.Time and will return nil pointers for +// The parsed dates are time.Time and will return the zero time for // unbounded dates, ex: // unbounded start: :2000-12-31 // unbounded end: 2000-12-31: -func ParseDateRange(dateRange string) (startDate, endDate *time.Time, err error) { +func ParseDateRange(dateRange string) (startDate, endDate time.Time, err error) { dates := strings.Split(dateRange, ":") if len(dates) != 2 { - return nil, nil, errors.New("bad date range, must be in format date:date") + err = errors.New("bad date range, must be in format date:date") + return } - parseDate := func(date string) (*time.Time, error) { + parseDate := func(date string) (out time.Time, err error) { if len(date) == 0 { - return nil, nil + return } - d, err := ParseDate(date) - return &d, err + out, err = time.Parse(TimeLayout, date) + return } startDate, err = parseDate(dates[0]) if err != nil { - return nil, nil, err + return } endDate, err = parseDate(dates[1]) if err != nil { - return nil, nil, err + return } return } diff --git a/common/date_test.go b/common/date_test.go index 42fd91aa..2c063247 100644 --- a/common/date_test.go +++ b/common/date_test.go @@ -10,66 +10,36 @@ import ( var ( date = time.Date(2015, time.Month(12), 31, 0, 0, 0, 0, time.UTC) date2 = time.Date(2016, time.Month(12), 31, 0, 0, 0, 0, time.UTC) + zero time.Time ) -func TestParseDate(t *testing.T) { - assert := assert.New(t) - - var testDates = []struct { - dateStr string - date time.Time - errNil bool - }{ - {"2015-12-31", date, true}, - {"2015-31-12", date, false}, - {"12-31-2015", date, false}, - {"31-12-2015", date, false}, - } - - for _, test := range testDates { - parsed, err := ParseDate(test.dateStr) - switch test.errNil { - case true: - assert.Nil(err) - assert.True(parsed.Equal(test.date), "parsed: %v, want %v", parsed, test.date) - case false: - assert.NotNil(err, "parsed %v, expected err %v", parsed, err) - } - } -} - func TestParseDateRange(t *testing.T) { assert := assert.New(t) var testDates = []struct { dateStr string - start *time.Time - end *time.Time + start time.Time + end time.Time errNil bool }{ - {"2015-12-31:2016-12-31", &date, &date2, true}, - {"2015-12-31:", &date, nil, true}, - {":2016-12-31", nil, &date2, true}, - {"2016-12-31", nil, nil, false}, - {"2016-31-12:", nil, nil, false}, - {":2016-31-12", nil, nil, false}, + {"2015-12-31:2016-12-31", date, date2, true}, + {"2015-12-31:", date, zero, true}, + {":2016-12-31", zero, date2, true}, + {"2016-12-31", zero, zero, false}, + {"2016-31-12:", zero, zero, false}, + {":2016-31-12", zero, zero, false}, } for _, test := range testDates { start, end, err := ParseDateRange(test.dateStr) - switch test.errNil { - case true: + if test.errNil { assert.Nil(err) - testPtr := func(want, have *time.Time) { - if want == nil { - assert.Nil(have) - } else { - assert.True((*have).Equal(*want)) - } + testPtr := func(want, have time.Time) { + assert.True(have.Equal(want)) } testPtr(test.start, start) testPtr(test.end, end) - case false: + } else { assert.NotNil(err) } } From a99b8a6210071060600a43b04daed800d8f4d125 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 20 Jun 2017 17:25:42 +0400 Subject: [PATCH 10/37] new events package query parser use parser compiler to generate query parser I used https://github.com/pointlander/peg which has a nice API and seems to be the most popular Golang compiler parser using PEG on Github. More about PEG: - https://en.wikipedia.org/wiki/Parsing_expression_grammar - https://github.com/PhilippeSigaud/Pegged/wiki/PEG-Basics - https://github.com/PhilippeSigaud/Pegged/wiki/Grammar-Examples rename implement query match function match function uncomment test lines add more test cases for query#Matches fix int case rename events to pubsub add comment about cache assertReceive helper to not block on receive in tests fix bug with multiple conditions uncomment benchmark first results: ``` Benchmark10Clients-2 1000 1305493 ns/op 3957519 B/op 355 allocs/op Benchmark100Clients-2 100 12278304 ns/op 39571751 B/op 3505 allocs/op Benchmark1000Clients-2 10 124120909 ns/op 395714004 B/op 35005 allocs/op ``` 124ms to publish message to 1000 clients. A lot. use AST from query.peg.go separate pubsub and query packages by using Query interface in pubsub wrote docs and refactor code updates from Frey's review refactor type assertion to use type switch cleanup during shutdown subscriber should create output channel, not the server overflow strategies, server buffer capacity context as the first argument for Publish log error introduce Option type update NewServer comment move helpers into pubsub_test increase assertReceive timeout add query.MustParse add more false tests for parser add more false tests for query.Matches parse numbers as int64 / float64 try our best to convert from other types add number to panic output add more comments save commit introduce client argument as first argument to Subscribe > Why we do not specify buffer size on the output channel in Subscribe? The choice of buffer size of N here depends on knowing the number of messages server will receive and the number of messages downstream subscribers will consume. This is fragile: if we publish an additional message, or if one of the downstream subscribers reads any fewer messages, we will again have blocked goroutines. save commit remove reference counting fix test test client resubscribe test UnsubscribeAll client options [pubsub/query] fuzzy testing do not print msg as it creates data race! --- .gitignore | 2 + pubsub/example_test.go | 24 + pubsub/pubsub.go | 314 ++++++ pubsub/pubsub_test.go | 227 +++++ pubsub/query/Makefile | 11 + pubsub/query/empty.go | 14 + pubsub/query/empty_test.go | 16 + pubsub/query/fuzz_test/main.go | 30 + pubsub/query/parser_test.go | 81 ++ pubsub/query/query.go | 258 +++++ pubsub/query/query.peg | 33 + pubsub/query/query.peg.go | 1668 ++++++++++++++++++++++++++++++++ pubsub/query/query_test.go | 64 ++ 13 files changed, 2742 insertions(+) create mode 100644 pubsub/example_test.go create mode 100644 pubsub/pubsub.go create mode 100644 pubsub/pubsub_test.go create mode 100644 pubsub/query/Makefile create mode 100644 pubsub/query/empty.go create mode 100644 pubsub/query/empty_test.go create mode 100644 pubsub/query/fuzz_test/main.go create mode 100644 pubsub/query/parser_test.go create mode 100644 pubsub/query/query.go create mode 100644 pubsub/query/query.peg create mode 100644 pubsub/query/query.peg.go create mode 100644 pubsub/query/query_test.go diff --git a/.gitignore b/.gitignore index 62f28681..34d0bf1f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ vendor .glide + +pubsub/query/fuzz_test/output diff --git a/pubsub/example_test.go b/pubsub/example_test.go new file mode 100644 index 00000000..d64b96ea --- /dev/null +++ b/pubsub/example_test.go @@ -0,0 +1,24 @@ +package pubsub_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tmlibs/pubsub" + "github.com/tendermint/tmlibs/pubsub/query" +) + +func TestExample(t *testing.T) { + s := pubsub.NewServer() + s.SetLogger(log.TestingLogger()) + s.Start() + defer s.Stop() + + ch := make(chan interface{}, 1) + s.Subscribe("example-client", query.MustParse("abci.account.name=John"), ch) + err := s.PublishWithTags("Tombstone", map[string]interface{}{"abci.account.name": "John"}) + require.NoError(t, err) + assertReceive(t, "Tombstone", ch) +} diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go new file mode 100644 index 00000000..26484816 --- /dev/null +++ b/pubsub/pubsub.go @@ -0,0 +1,314 @@ +// Package pubsub implements a pub-sub model with a single publisher (Server) +// and multiple subscribers (clients). +// +// Though you can have multiple publishers by sharing a pointer to a server or +// by giving the same channel to each publisher and publishing messages from +// that channel (fan-in). +// +// Clients subscribe for messages, which could be of any type, using a query. +// When some message is published, we match it with all queries. If there is a +// match, this message will be pushed to all clients, subscribed to that query. +// See query subpackage for our implementation. +// +// Overflow strategies (incoming publish requests): +// +// 1) drop - drops publish requests when there are too many of them +// 2) wait - blocks until the server is ready to accept more publish requests (default) +// +// Subscribe/Unsubscribe calls are always blocking. +// +// Overflow strategies (outgoing messages): +// +// 1) skip - do not send a message if the client is busy or slow (default) +// 2) wait - wait until the client is ready to accept new messages +// +package pubsub + +import ( + "errors" + + cmn "github.com/tendermint/tmlibs/common" + "github.com/tendermint/tmlibs/log" +) + +type operation int + +const ( + sub operation = iota + pub + unsub + shutdown +) + +type overflowStrategy int + +const ( + drop overflowStrategy = iota + wait +) + +var ( + ErrorOverflow = errors.New("Server overflowed") +) + +type cmd struct { + op operation + query Query + ch chan<- interface{} + clientID string + msg interface{} + tags map[string]interface{} +} + +// Query defines an interface for a query to be used for subscribing. +type Query interface { + Matches(tags map[string]interface{}) bool +} + +// Server allows clients to subscribe/unsubscribe for messages, pubsling +// messages with or without tags, and manages internal state. +type Server struct { + cmn.BaseService + + cmds chan cmd + + overflowStrategy overflowStrategy + slowClientStrategy overflowStrategy +} + +// Option sets a parameter for the server. +type Option func(*Server) + +// NewServer returns a new server. See the commentary on the Option functions +// for a detailed description of how to configure buffering and overflow +// behavior. If no options are provided, the resulting server's queue is +// unbuffered and it blocks when overflowed. +func NewServer(options ...Option) *Server { + s := &Server{overflowStrategy: wait, slowClientStrategy: drop} + s.BaseService = *cmn.NewBaseService(nil, "PubSub", s) + + for _, option := range options { + option(s) + } + + if s.cmds == nil { // if BufferCapacity was not set, create unbuffered channel + s.cmds = make(chan cmd) + } + + return s +} + +// BufferCapacity allows you to specify capacity for the internal server's +// queue. Since the server, given Y subscribers, could only process X messages, +// this option could be used to survive spikes (e.g. high amount of +// transactions during peak hours). +func BufferCapacity(cap int) Option { + return func(s *Server) { + if cap > 0 { + s.cmds = make(chan cmd, cap) + } + } +} + +// OverflowStrategyDrop will tell the server to drop messages when it can't +// process more messages. +func OverflowStrategyDrop() Option { + return func(s *Server) { + s.overflowStrategy = drop + } +} + +// OverflowStrategyWait will tell the server to block and wait for some time +// for server to process other messages. Default strategy. +func OverflowStrategyWait() func(*Server) { + return func(s *Server) { + s.overflowStrategy = wait + } +} + +// WaitSlowClients will tell the server to block and wait until subscriber +// reads a messages even if it is fast enough to process them. +func WaitSlowClients() func(*Server) { + return func(s *Server) { + s.slowClientStrategy = wait + } +} + +// SkipSlowClients will tell the server to skip subscriber if it is busy +// processing previous message(s). Default strategy. +func SkipSlowClients() func(*Server) { + return func(s *Server) { + s.slowClientStrategy = drop + } +} + +// Subscribe returns a channel on which messages matching the given query can +// be received. If the subscription already exists old channel will be closed +// and new one returned. +func (s *Server) Subscribe(clientID string, query Query, out chan<- interface{}) { + s.cmds <- cmd{op: sub, clientID: clientID, query: query, ch: out} +} + +// Unsubscribe unsubscribes the given client from the query. +func (s *Server) Unsubscribe(clientID string, query Query) { + s.cmds <- cmd{op: unsub, clientID: clientID, query: query} +} + +// Unsubscribe unsubscribes the given channel. +func (s *Server) UnsubscribeAll(clientID string) { + s.cmds <- cmd{op: unsub, clientID: clientID} +} + +// Publish publishes the given message. +func (s *Server) Publish(msg interface{}) error { + return s.PublishWithTags(msg, make(map[string]interface{})) +} + +// PublishWithTags publishes the given message with a set of tags. This set of +// tags will be matched with client queries. If there is a match, the message +// will be sent to a client. +func (s *Server) PublishWithTags(msg interface{}, tags map[string]interface{}) error { + pubCmd := cmd{op: pub, msg: msg, tags: tags} + switch s.overflowStrategy { + case drop: + select { + case s.cmds <- pubCmd: + default: + s.Logger.Error("Server overflowed, dropping message...", "msg", msg) + return ErrorOverflow + } + case wait: + s.cmds <- pubCmd + } + return nil +} + +// OnStop implements Service.OnStop by shutting down the server. +func (s *Server) OnStop() { + s.cmds <- cmd{op: shutdown} +} + +// NOTE: not goroutine safe +type state struct { + // query -> client -> ch + queries map[Query]map[string]chan<- interface{} + // client -> query -> struct{} + clients map[string]map[Query]struct{} +} + +// OnStart implements Service.OnStart by creating a main loop. +func (s *Server) OnStart() error { + go s.loop(state{ + queries: make(map[Query]map[string]chan<- interface{}), + clients: make(map[string]map[Query]struct{}), + }) + return nil +} + +func (s *Server) loop(state state) { +loop: + for cmd := range s.cmds { + switch cmd.op { + case unsub: + if cmd.query != nil { + state.remove(cmd.clientID, cmd.query) + } else { + state.removeAll(cmd.clientID) + } + case shutdown: + state.reset() + break loop + case sub: + state.add(cmd.clientID, cmd.query, cmd.ch) + case pub: + state.send(cmd.msg, cmd.tags, s.slowClientStrategy, s.Logger) + } + } +} + +func (state *state) add(clientID string, q Query, ch chan<- interface{}) { + // add query if needed + if clientToChannelMap, ok := state.queries[q]; !ok { + state.queries[q] = make(map[string]chan<- interface{}) + } else { + // check if already subscribed + if oldCh, ok := clientToChannelMap[clientID]; ok { + close(oldCh) + } + } + state.queries[q][clientID] = ch + + // add client if needed + if _, ok := state.clients[clientID]; !ok { + state.clients[clientID] = make(map[Query]struct{}) + } + state.clients[clientID][q] = struct{}{} + + // create subscription + clientToChannelMap := state.queries[q] + clientToChannelMap[clientID] = ch +} + +func (state *state) remove(clientID string, q Query) { + clientToChannelMap, ok := state.queries[q] + if !ok { + return + } + + ch, ok := clientToChannelMap[clientID] + if ok { + close(ch) + + delete(state.clients[clientID], q) + + // if it not subscribed to anything else, remove the client + if len(state.clients[clientID]) == 0 { + delete(state.clients, clientID) + } + + delete(state.queries[q], clientID) + } +} + +func (state *state) removeAll(clientID string) { + queryMap, ok := state.clients[clientID] + if !ok { + return + } + + for q, _ := range queryMap { + ch := state.queries[q][clientID] + close(ch) + + delete(state.queries[q], clientID) + } + + delete(state.clients, clientID) +} + +func (state *state) reset() { + state.queries = make(map[Query]map[string]chan<- interface{}) + state.clients = make(map[string]map[Query]struct{}) +} + +func (state *state) send(msg interface{}, tags map[string]interface{}, slowClientStrategy overflowStrategy, logger log.Logger) { + for q, clientToChannelMap := range state.queries { + // NOTE we can use LRU cache to speed up common cases like query = " + // tm.events.type=NewBlock" and tags = {"tm.events.type": "NewBlock"} + if q.Matches(tags) { + for clientID, ch := range clientToChannelMap { + logger.Info("Sending message to client", "msg", msg, "client", clientID) + switch slowClientStrategy { + case drop: + select { + case ch <- msg: + default: + logger.Error("Client is busy, skipping...", "clientID", clientID) + } + case wait: + ch <- msg + } + } + } + } +} diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go new file mode 100644 index 00000000..570f76a8 --- /dev/null +++ b/pubsub/pubsub_test.go @@ -0,0 +1,227 @@ +package pubsub_test + +import ( + "fmt" + "runtime/debug" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/tendermint/tmlibs/log" + "github.com/tendermint/tmlibs/pubsub" + "github.com/tendermint/tmlibs/pubsub/query" +) + +const ( + clientID = "test-client" +) + +func TestSubscribe(t *testing.T) { + s := pubsub.NewServer() + s.SetLogger(log.TestingLogger()) + s.Start() + defer s.Stop() + + ch := make(chan interface{}, 1) + s.Subscribe(clientID, query.Empty{}, ch) + err := s.Publish("Ka-Zar") + require.NoError(t, err) + assertReceive(t, "Ka-Zar", ch) + + err = s.Publish("Quicksilver") + require.NoError(t, err) + assertReceive(t, "Quicksilver", ch) +} + +func TestDifferentClients(t *testing.T) { + s := pubsub.NewServer() + s.SetLogger(log.TestingLogger()) + s.Start() + defer s.Stop() + ch1 := make(chan interface{}, 1) + s.Subscribe("client-1", query.MustParse("tm.events.type=NewBlock"), ch1) + err := s.PublishWithTags("Iceman", map[string]interface{}{"tm.events.type": "NewBlock"}) + require.NoError(t, err) + assertReceive(t, "Iceman", ch1) + + ch2 := make(chan interface{}, 1) + s.Subscribe("client-2", query.MustParse("tm.events.type=NewBlock AND abci.account.name=Igor"), ch2) + err = s.PublishWithTags("Ultimo", map[string]interface{}{"tm.events.type": "NewBlock", "abci.account.name": "Igor"}) + require.NoError(t, err) + assertReceive(t, "Ultimo", ch1) + assertReceive(t, "Ultimo", ch2) + + ch3 := make(chan interface{}, 1) + s.Subscribe("client-3", query.MustParse("tm.events.type=NewRoundStep AND abci.account.name=Igor AND abci.invoice.number = 10"), ch3) + err = s.PublishWithTags("Valeria Richards", map[string]interface{}{"tm.events.type": "NewRoundStep"}) + require.NoError(t, err) + assert.Zero(t, len(ch3)) +} + +func TestClientResubscribes(t *testing.T) { + s := pubsub.NewServer() + s.SetLogger(log.TestingLogger()) + s.Start() + defer s.Stop() + + q := query.MustParse("tm.events.type=NewBlock") + + ch1 := make(chan interface{}, 1) + s.Subscribe(clientID, q, ch1) + err := s.PublishWithTags("Goblin Queen", map[string]interface{}{"tm.events.type": "NewBlock"}) + require.NoError(t, err) + assertReceive(t, "Goblin Queen", ch1) + + ch2 := make(chan interface{}, 1) + s.Subscribe(clientID, q, ch2) + + _, ok := <-ch1 + assert.False(t, ok) + + err = s.PublishWithTags("Spider-Man", map[string]interface{}{"tm.events.type": "NewBlock"}) + require.NoError(t, err) + assertReceive(t, "Spider-Man", ch2) +} + +func TestUnsubscribe(t *testing.T) { + s := pubsub.NewServer() + s.SetLogger(log.TestingLogger()) + s.Start() + defer s.Stop() + + ch := make(chan interface{}) + s.Subscribe(clientID, query.Empty{}, ch) + s.Unsubscribe(clientID, query.Empty{}) + + err := s.Publish("Nick Fury") + require.NoError(t, err) + assert.Zero(t, len(ch), "Should not receive anything after Unsubscribe") + + _, ok := <-ch + assert.False(t, ok) +} + +func TestUnsubscribeAll(t *testing.T) { + s := pubsub.NewServer() + s.SetLogger(log.TestingLogger()) + s.Start() + defer s.Stop() + + ch1, ch2 := make(chan interface{}, 1), make(chan interface{}, 1) + s.Subscribe(clientID, query.MustParse("tm.events.type=NewBlock"), ch1) + s.Subscribe(clientID, query.MustParse("tm.events.type=NewBlockHeader"), ch2) + + s.UnsubscribeAll(clientID) + + err := s.Publish("Nick Fury") + require.NoError(t, err) + assert.Zero(t, len(ch1), "Should not receive anything after UnsubscribeAll") + assert.Zero(t, len(ch2), "Should not receive anything after UnsubscribeAll") + + _, ok := <-ch1 + assert.False(t, ok) + _, ok = <-ch2 + assert.False(t, ok) +} + +func TestOverflowStrategyDrop(t *testing.T) { + s := pubsub.NewServer(pubsub.OverflowStrategyDrop()) + s.SetLogger(log.TestingLogger()) + + err := s.Publish("Veda") + if assert.Error(t, err) { + assert.Equal(t, pubsub.ErrorOverflow, err) + } +} + +func TestOverflowStrategyWait(t *testing.T) { + s := pubsub.NewServer(pubsub.OverflowStrategyWait()) + s.SetLogger(log.TestingLogger()) + + go func() { + time.Sleep(1 * time.Second) + s.Start() + defer s.Stop() + }() + + err := s.Publish("Veda") + assert.NoError(t, err) +} + +func TestBufferCapacity(t *testing.T) { + s := pubsub.NewServer(pubsub.BufferCapacity(2)) + s.SetLogger(log.TestingLogger()) + + err := s.Publish("Nighthawk") + require.NoError(t, err) + err = s.Publish("Sage") + require.NoError(t, err) +} + +func TestWaitSlowClients(t *testing.T) { + s := pubsub.NewServer(pubsub.WaitSlowClients()) + s.SetLogger(log.TestingLogger()) + s.Start() + defer s.Stop() + + ch := make(chan interface{}) + s.Subscribe(clientID, query.Empty{}, ch) + err := s.Publish("Wonderwoman") + require.NoError(t, err) + + time.Sleep(1 * time.Second) + + assertReceive(t, "Wonderwoman", ch) +} + +func TestSkipSlowClients(t *testing.T) { + s := pubsub.NewServer(pubsub.SkipSlowClients()) + s.SetLogger(log.TestingLogger()) + s.Start() + defer s.Stop() + + ch := make(chan interface{}) + s.Subscribe(clientID, query.Empty{}, ch) + err := s.Publish("Cyclops") + require.NoError(t, err) + assert.Zero(t, len(ch)) +} + +func Benchmark10Clients(b *testing.B) { benchmarkNClients(10, b) } +func Benchmark100Clients(b *testing.B) { benchmarkNClients(100, b) } +func Benchmark1000Clients(b *testing.B) { benchmarkNClients(1000, b) } + +func benchmarkNClients(n int, b *testing.B) { + s := pubsub.NewServer(pubsub.BufferCapacity(b.N)) + s.Start() + defer s.Stop() + + for i := 0; i < n; i++ { + ch := make(chan interface{}) + s.Subscribe(clientID, query.MustParse(fmt.Sprintf("abci.Account.Owner = Ivan AND abci.Invoices.Number = %d", i)), ch) + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.PublishWithTags("Gamora", map[string]interface{}{"abci.Account.Owner": "Ivan", "abci.Invoices.Number": i}) + } +} + +/////////////////////////////////////////////////////////////////////////////// +/// HELPERS +/////////////////////////////////////////////////////////////////////////////// + +func assertReceive(t *testing.T, expected interface{}, ch <-chan interface{}, msgAndArgs ...interface{}) { + select { + case actual := <-ch: + if actual != nil { + assert.Equal(t, expected, actual, msgAndArgs...) + } + case <-time.After(1 * time.Second): + t.Errorf("Expected to receive %v from the channel, got nothing after 1s", expected) + debug.PrintStack() + } +} diff --git a/pubsub/query/Makefile b/pubsub/query/Makefile new file mode 100644 index 00000000..ca3ff5b5 --- /dev/null +++ b/pubsub/query/Makefile @@ -0,0 +1,11 @@ +gen_query_parser: + @go get github.com/pointlander/peg + peg -inline -switch query.peg + +fuzzy_test: + @go get github.com/dvyukov/go-fuzz/go-fuzz + @go get github.com/dvyukov/go-fuzz/go-fuzz-build + go-fuzz-build github.com/tendermint/tmlibs/pubsub/query/fuzz_test + go-fuzz -bin=./fuzz_test-fuzz.zip -workdir=./fuzz_test/output + +.PHONY: gen_query_parser fuzzy_test diff --git a/pubsub/query/empty.go b/pubsub/query/empty.go new file mode 100644 index 00000000..2d60a892 --- /dev/null +++ b/pubsub/query/empty.go @@ -0,0 +1,14 @@ +package query + +// Empty query matches any set of tags. +type Empty struct { +} + +// Matches always returns true. +func (Empty) Matches(tags map[string]interface{}) bool { + return true +} + +func (Empty) String() string { + return "empty" +} diff --git a/pubsub/query/empty_test.go b/pubsub/query/empty_test.go new file mode 100644 index 00000000..663acb19 --- /dev/null +++ b/pubsub/query/empty_test.go @@ -0,0 +1,16 @@ +package query_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tendermint/tmlibs/pubsub/query" +) + +func TestEmptyQueryMatchesAnything(t *testing.T) { + q := query.Empty{} + assert.True(t, q.Matches(map[string]interface{}{})) + assert.True(t, q.Matches(map[string]interface{}{"Asher": "Roth"})) + assert.True(t, q.Matches(map[string]interface{}{"Route": 66})) + assert.True(t, q.Matches(map[string]interface{}{"Route": 66, "Billy": "Blue"})) +} diff --git a/pubsub/query/fuzz_test/main.go b/pubsub/query/fuzz_test/main.go new file mode 100644 index 00000000..3b0ef147 --- /dev/null +++ b/pubsub/query/fuzz_test/main.go @@ -0,0 +1,30 @@ +package fuzz_test + +import ( + "fmt" + + "github.com/tendermint/tmlibs/pubsub/query" +) + +func Fuzz(data []byte) int { + sdata := string(data) + q0, err := query.New(sdata) + if err != nil { + return 0 + } + + sdata1 := q0.String() + q1, err := query.New(sdata1) + if err != nil { + panic(err) + } + + sdata2 := q1.String() + if sdata1 != sdata2 { + fmt.Printf("q0: %q\n", sdata1) + fmt.Printf("q1: %q\n", sdata2) + panic("query changed") + } + + return 1 +} diff --git a/pubsub/query/parser_test.go b/pubsub/query/parser_test.go new file mode 100644 index 00000000..19496666 --- /dev/null +++ b/pubsub/query/parser_test.go @@ -0,0 +1,81 @@ +package query_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/tendermint/tmlibs/pubsub/query" +) + +// TODO: fuzzy testing? +func TestParser(t *testing.T) { + cases := []struct { + query string + valid bool + }{ + {"tm.events.type=NewBlock", true}, + {"tm.events.type = NewBlock", true}, + {"tm.events.type=TIME", true}, + {"tm.events.type=DATE", true}, + {"tm.events.type==", false}, + {">==", false}, + {"tm.events.type NewBlock =", false}, + {"tm.events.type>NewBlock", false}, + {"", false}, + {"=", false}, + {"=NewBlock", false}, + {"tm.events.type=", false}, + + {"tm.events.typeNewBlock", false}, + {"NewBlock", false}, + {"", false}, + + {"tm.events.type=NewBlock AND abci.account.name=Igor", true}, + {"tm.events.type=NewBlock AND", false}, + {"tm.events.type=NewBlock AN", false}, + {"tm.events.type=NewBlock AN tm.events.type=NewBlockHeader", false}, + {"AND tm.events.type=NewBlock ", false}, + + {"abci.account.name CONTAINS Igor", true}, + + {"tx.date > DATE 2013-05-03", true}, + {"tx.date < DATE 2013-05-03", true}, + {"tx.date <= DATE 2013-05-03", true}, + {"tx.date >= DATE 2013-05-03", true}, + {"tx.date >= DAT 2013-05-03", false}, + {"tx.date <= DATE2013-05-03", false}, + {"tx.date <= DATE -05-03", false}, + {"tx.date >= DATE 20130503", false}, + {"tx.date >= DATE 2013+01-03", false}, + // incorrect year, month, day + {"tx.date >= DATE 0013-01-03", false}, + {"tx.date >= DATE 2013-31-03", false}, + {"tx.date >= DATE 2013-01-83", false}, + + {"tx.date > TIME 2013-05-03T14:45:00+07:00", true}, + {"tx.date < TIME 2013-05-03T14:45:00-02:00", true}, + {"tx.date <= TIME 2013-05-03T14:45:00Z", true}, + {"tx.date >= TIME 2013-05-03T14:45:00Z", true}, + {"tx.date >= TIME2013-05-03T14:45:00Z", false}, + {"tx.date = IME 2013-05-03T14:45:00Z", false}, + {"tx.date = TIME 2013-05-:45:00Z", false}, + {"tx.date >= TIME 2013-05-03T14:45:00", false}, + {"tx.date >= TIME 0013-00-00T14:45:00Z", false}, + {"tx.date >= TIME 2013+05=03T14:45:00Z", false}, + + {"account.balance=100", true}, + {"account.balance >= 200", true}, + {"account.balance >= -300", false}, + {"account.balance >>= 400", false}, + {"account.balance=33.22.1", false}, + } + + for _, c := range cases { + _, err := query.New(c.query) + if c.valid { + assert.NoError(t, err, "Query was '%s'", c.query) + } else { + assert.Error(t, err, "Query was '%s'", c.query) + } + } +} diff --git a/pubsub/query/query.go b/pubsub/query/query.go new file mode 100644 index 00000000..f084a3f9 --- /dev/null +++ b/pubsub/query/query.go @@ -0,0 +1,258 @@ +// Package query provides a parser for a custom query format: +// +// abci.invoice.number=22 AND abci.invoice.owner=Ivan +// +// See query.peg for the grammar, which is a https://en.wikipedia.org/wiki/Parsing_expression_grammar. +// More: https://github.com/PhilippeSigaud/Pegged/wiki/PEG-Basics +// +// It has a support for numbers (integer and floating point), dates and times. +package query + +import ( + "fmt" + "reflect" + "strconv" + "strings" + "time" +) + +// Query holds the query string and the query parser. +type Query struct { + str string + parser *QueryParser +} + +// New parses the given string and returns a query or error if the string is +// invalid. +func New(s string) (*Query, error) { + p := &QueryParser{Buffer: fmt.Sprintf(`"%s"`, s)} + p.Init() + if err := p.Parse(); err != nil { + return nil, err + } + return &Query{str: s, parser: p}, nil +} + +// MustParse turns the given string into a query or panics; for tests or others +// cases where you know the string is valid. +func MustParse(s string) *Query { + q, err := New(s) + if err != nil { + panic(fmt.Sprintf("failed to parse %s: %v", s, err)) + } + return q +} + +// String returns the original string. +func (q *Query) String() string { + return q.str +} + +type operator uint8 + +const ( + opLessEqual operator = iota + opGreaterEqual + opLess + opGreater + opEqual + opContains +) + +// Matches returns true if the query matches the given set of tags, false otherwise. +// +// For example, query "name=John" matches tags = {"name": "John"}. More +// examples could be found in parser_test.go and query_test.go. +func (q *Query) Matches(tags map[string]interface{}) bool { + if len(tags) == 0 { + return false + } + + buffer, begin, end := q.parser.Buffer, 0, 0 + + var tag string + var op operator + + // tokens must be in the following order: tag ("tx.gas") -> operator ("=") -> operand ("7") + for _, token := range q.parser.Tokens() { + switch token.pegRule { + + case rulePegText: + begin, end = int(token.begin), int(token.end) + case ruletag: + tag = buffer[begin:end] + case rulele: + op = opLessEqual + case rulege: + op = opGreaterEqual + case rulel: + op = opLess + case ruleg: + op = opGreater + case ruleequal: + op = opEqual + case rulecontains: + op = opContains + case rulevalue: + // see if the triplet (tag, operator, operand) matches any tag + // "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" } + if !match(tag, op, reflect.ValueOf(buffer[begin:end]), tags) { + return false + } + case rulenumber: + number := buffer[begin:end] + if strings.Contains(number, ".") { // if it looks like a floating-point number + value, err := strconv.ParseFloat(number, 64) + if err != nil { + panic(fmt.Sprintf("got %v while trying to parse %s as float64 (should never happen if the grammar is correct)", err, number)) + } + if !match(tag, op, reflect.ValueOf(value), tags) { + return false + } + } else { + value, err := strconv.ParseInt(number, 10, 64) + if err != nil { + panic(fmt.Sprintf("got %v while trying to parse %s as int64 (should never happen if the grammar is correct)", err, number)) + } + if !match(tag, op, reflect.ValueOf(value), tags) { + return false + } + } + case ruletime: + value, err := time.Parse(time.RFC3339, buffer[begin:end]) + if err != nil { + panic(fmt.Sprintf("got %v while trying to parse %s as time.Time / RFC3339 (should never happen if the grammar is correct)", err, buffer[begin:end])) + } + if !match(tag, op, reflect.ValueOf(value), tags) { + return false + } + case ruledate: + value, err := time.Parse("2006-01-02", buffer[begin:end]) + if err != nil { + panic(fmt.Sprintf("got %v while trying to parse %s as time.Time / '2006-01-02' (should never happen if the grammar is correct)", err, buffer[begin:end])) + } + if !match(tag, op, reflect.ValueOf(value), tags) { + return false + } + } + } + + return true +} + +// match returns true if the given triplet (tag, operator, operand) matches any tag. +// +// First, it looks up the tag in tags and if it finds one, tries to compare the +// value from it to the operand using the operator. +// +// "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" } +func match(tag string, op operator, operand reflect.Value, tags map[string]interface{}) bool { + // look up the tag from the query in tags + value, ok := tags[tag] + if !ok { + return false + } + switch operand.Kind() { + case reflect.Struct: // time + operandAsTime := operand.Interface().(time.Time) + v, ok := value.(time.Time) + if !ok { // if value from tags is not time.Time + return false + } + switch op { + case opLessEqual: + return v.Before(operandAsTime) || v.Equal(operandAsTime) + case opGreaterEqual: + return v.Equal(operandAsTime) || v.After(operandAsTime) + case opLess: + return v.Before(operandAsTime) + case opGreater: + return v.After(operandAsTime) + case opEqual: + return v.Equal(operandAsTime) + } + case reflect.Float64: + operandFloat64 := operand.Interface().(float64) + var v float64 + // try our best to convert value from tags to float64 + switch vt := value.(type) { + case float64: + v = vt + case float32: + v = float64(vt) + case int: + v = float64(vt) + case int8: + v = float64(vt) + case int16: + v = float64(vt) + case int32: + v = float64(vt) + case int64: + v = float64(vt) + default: // fail for all other types + panic(fmt.Sprintf("Incomparable types: %T (%v) vs float64 (%v)", value, value, operandFloat64)) + } + switch op { + case opLessEqual: + return v <= operandFloat64 + case opGreaterEqual: + return v >= operandFloat64 + case opLess: + return v < operandFloat64 + case opGreater: + return v > operandFloat64 + case opEqual: + return v == operandFloat64 + } + case reflect.Int64: + operandInt := operand.Interface().(int64) + var v int64 + // try our best to convert value from tags to int64 + switch vt := value.(type) { + case int64: + v = vt + case int8: + v = int64(vt) + case int16: + v = int64(vt) + case int32: + v = int64(vt) + case int: + v = int64(vt) + case float64: + v = int64(vt) + case float32: + v = int64(vt) + default: // fail for all other types + panic(fmt.Sprintf("Incomparable types: %T (%v) vs int64 (%v)", value, value, operandInt)) + } + switch op { + case opLessEqual: + return v <= operandInt + case opGreaterEqual: + return v >= operandInt + case opLess: + return v < operandInt + case opGreater: + return v > operandInt + case opEqual: + return v == operandInt + } + case reflect.String: + v, ok := value.(string) + if !ok { // if value from tags is not string + return false + } + switch op { + case opEqual: + return v == operand.String() + case opContains: + return strings.Contains(v, operand.String()) + } + default: + panic(fmt.Sprintf("Unknown kind of operand %v", operand.Kind())) + } + + return false +} diff --git a/pubsub/query/query.peg b/pubsub/query/query.peg new file mode 100644 index 00000000..9654289c --- /dev/null +++ b/pubsub/query/query.peg @@ -0,0 +1,33 @@ +package query + +type QueryParser Peg { +} + +e <- '\"' condition ( ' '+ and ' '+ condition )* '\"' !. + +condition <- tag ' '* (le ' '* (number / time / date) + / ge ' '* (number / time / date) + / l ' '* (number / time / date) + / g ' '* (number / time / date) + / equal ' '* (number / time / date / value) + / contains ' '* value + ) + +tag <- < (![ \t\n\r\\()"=><] .)+ > +value <- < (![ \t\n\r\\()"=><] .)+ > +number <- < ('0' + / [1-9] digit* ('.' digit*)?) > +digit <- [0-9] +time <- "TIME " < year '-' month '-' day 'T' digit digit ':' digit digit ':' digit digit (('-' / '+') digit digit ':' digit digit / 'Z') > +date <- "DATE " < year '-' month '-' day > +year <- ('1' / '2') digit digit digit +month <- ('0' / '1') digit +day <- ('0' / '1' / '2' / '3') digit +and <- "AND" + +equal <- "=" +contains <- "CONTAINS" +le <- "<=" +ge <- ">=" +l <- "<" +g <- ">" diff --git a/pubsub/query/query.peg.go b/pubsub/query/query.peg.go new file mode 100644 index 00000000..5cd0a9e3 --- /dev/null +++ b/pubsub/query/query.peg.go @@ -0,0 +1,1668 @@ +package query + +import ( + "fmt" + "math" + "sort" + "strconv" +) + +const endSymbol rune = 1114112 + +/* The rule types inferred from the grammar are below. */ +type pegRule uint8 + +const ( + ruleUnknown pegRule = iota + rulee + rulecondition + ruletag + rulevalue + rulenumber + ruledigit + ruletime + ruledate + ruleyear + rulemonth + ruleday + ruleand + ruleequal + rulecontains + rulele + rulege + rulel + ruleg + rulePegText +) + +var rul3s = [...]string{ + "Unknown", + "e", + "condition", + "tag", + "value", + "number", + "digit", + "time", + "date", + "year", + "month", + "day", + "and", + "equal", + "contains", + "le", + "ge", + "l", + "g", + "PegText", +} + +type token32 struct { + pegRule + begin, end uint32 +} + +func (t *token32) String() string { + return fmt.Sprintf("\x1B[34m%v\x1B[m %v %v", rul3s[t.pegRule], t.begin, t.end) +} + +type node32 struct { + token32 + up, next *node32 +} + +func (node *node32) print(pretty bool, buffer string) { + var print func(node *node32, depth int) + print = func(node *node32, depth int) { + for node != nil { + for c := 0; c < depth; c++ { + fmt.Printf(" ") + } + rule := rul3s[node.pegRule] + quote := strconv.Quote(string(([]rune(buffer)[node.begin:node.end]))) + if !pretty { + fmt.Printf("%v %v\n", rule, quote) + } else { + fmt.Printf("\x1B[34m%v\x1B[m %v\n", rule, quote) + } + if node.up != nil { + print(node.up, depth+1) + } + node = node.next + } + } + print(node, 0) +} + +func (node *node32) Print(buffer string) { + node.print(false, buffer) +} + +func (node *node32) PrettyPrint(buffer string) { + node.print(true, buffer) +} + +type tokens32 struct { + tree []token32 +} + +func (t *tokens32) Trim(length uint32) { + t.tree = t.tree[:length] +} + +func (t *tokens32) Print() { + for _, token := range t.tree { + fmt.Println(token.String()) + } +} + +func (t *tokens32) AST() *node32 { + type element struct { + node *node32 + down *element + } + tokens := t.Tokens() + var stack *element + for _, token := range tokens { + if token.begin == token.end { + continue + } + node := &node32{token32: token} + for stack != nil && stack.node.begin >= token.begin && stack.node.end <= token.end { + stack.node.next = node.up + node.up = stack.node + stack = stack.down + } + stack = &element{node: node, down: stack} + } + if stack != nil { + return stack.node + } + return nil +} + +func (t *tokens32) PrintSyntaxTree(buffer string) { + t.AST().Print(buffer) +} + +func (t *tokens32) PrettyPrintSyntaxTree(buffer string) { + t.AST().PrettyPrint(buffer) +} + +func (t *tokens32) Add(rule pegRule, begin, end, index uint32) { + if tree := t.tree; int(index) >= len(tree) { + expanded := make([]token32, 2*len(tree)) + copy(expanded, tree) + t.tree = expanded + } + t.tree[index] = token32{ + pegRule: rule, + begin: begin, + end: end, + } +} + +func (t *tokens32) Tokens() []token32 { + return t.tree +} + +type QueryParser struct { + Buffer string + buffer []rune + rules [20]func() bool + parse func(rule ...int) error + reset func() + Pretty bool + tokens32 +} + +func (p *QueryParser) Parse(rule ...int) error { + return p.parse(rule...) +} + +func (p *QueryParser) Reset() { + p.reset() +} + +type textPosition struct { + line, symbol int +} + +type textPositionMap map[int]textPosition + +func translatePositions(buffer []rune, positions []int) textPositionMap { + length, translations, j, line, symbol := len(positions), make(textPositionMap, len(positions)), 0, 1, 0 + sort.Ints(positions) + +search: + for i, c := range buffer { + if c == '\n' { + line, symbol = line+1, 0 + } else { + symbol++ + } + if i == positions[j] { + translations[positions[j]] = textPosition{line, symbol} + for j++; j < length; j++ { + if i != positions[j] { + continue search + } + } + break search + } + } + + return translations +} + +type parseError struct { + p *QueryParser + max token32 +} + +func (e *parseError) Error() string { + tokens, error := []token32{e.max}, "\n" + positions, p := make([]int, 2*len(tokens)), 0 + for _, token := range tokens { + positions[p], p = int(token.begin), p+1 + positions[p], p = int(token.end), p+1 + } + translations := translatePositions(e.p.buffer, positions) + format := "parse error near %v (line %v symbol %v - line %v symbol %v):\n%v\n" + if e.p.Pretty { + format = "parse error near \x1B[34m%v\x1B[m (line %v symbol %v - line %v symbol %v):\n%v\n" + } + for _, token := range tokens { + begin, end := int(token.begin), int(token.end) + error += fmt.Sprintf(format, + rul3s[token.pegRule], + translations[begin].line, translations[begin].symbol, + translations[end].line, translations[end].symbol, + strconv.Quote(string(e.p.buffer[begin:end]))) + } + + return error +} + +func (p *QueryParser) PrintSyntaxTree() { + if p.Pretty { + p.tokens32.PrettyPrintSyntaxTree(p.Buffer) + } else { + p.tokens32.PrintSyntaxTree(p.Buffer) + } +} + +func (p *QueryParser) Init() { + var ( + max token32 + position, tokenIndex uint32 + buffer []rune + ) + p.reset = func() { + max = token32{} + position, tokenIndex = 0, 0 + + p.buffer = []rune(p.Buffer) + if len(p.buffer) == 0 || p.buffer[len(p.buffer)-1] != endSymbol { + p.buffer = append(p.buffer, endSymbol) + } + buffer = p.buffer + } + p.reset() + + _rules := p.rules + tree := tokens32{tree: make([]token32, math.MaxInt16)} + p.parse = func(rule ...int) error { + r := 1 + if len(rule) > 0 { + r = rule[0] + } + matches := p.rules[r]() + p.tokens32 = tree + if matches { + p.Trim(tokenIndex) + return nil + } + return &parseError{p, max} + } + + add := func(rule pegRule, begin uint32) { + tree.Add(rule, begin, position, tokenIndex) + tokenIndex++ + if begin != position && position > max.end { + max = token32{rule, begin, position} + } + } + + matchDot := func() bool { + if buffer[position] != endSymbol { + position++ + return true + } + return false + } + + /*matchChar := func(c byte) bool { + if buffer[position] == c { + position++ + return true + } + return false + }*/ + + /*matchRange := func(lower byte, upper byte) bool { + if c := buffer[position]; c >= lower && c <= upper { + position++ + return true + } + return false + }*/ + + _rules = [...]func() bool{ + nil, + /* 0 e <- <('"' condition (' '+ and ' '+ condition)* '"' !.)> */ + func() bool { + position0, tokenIndex0 := position, tokenIndex + { + position1 := position + if buffer[position] != rune('"') { + goto l0 + } + position++ + if !_rules[rulecondition]() { + goto l0 + } + l2: + { + position3, tokenIndex3 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l3 + } + position++ + l4: + { + position5, tokenIndex5 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l5 + } + position++ + goto l4 + l5: + position, tokenIndex = position5, tokenIndex5 + } + { + position6 := position + { + position7, tokenIndex7 := position, tokenIndex + if buffer[position] != rune('a') { + goto l8 + } + position++ + goto l7 + l8: + position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('A') { + goto l3 + } + position++ + } + l7: + { + position9, tokenIndex9 := position, tokenIndex + if buffer[position] != rune('n') { + goto l10 + } + position++ + goto l9 + l10: + position, tokenIndex = position9, tokenIndex9 + if buffer[position] != rune('N') { + goto l3 + } + position++ + } + l9: + { + position11, tokenIndex11 := position, tokenIndex + if buffer[position] != rune('d') { + goto l12 + } + position++ + goto l11 + l12: + position, tokenIndex = position11, tokenIndex11 + if buffer[position] != rune('D') { + goto l3 + } + position++ + } + l11: + add(ruleand, position6) + } + if buffer[position] != rune(' ') { + goto l3 + } + position++ + l13: + { + position14, tokenIndex14 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l14 + } + position++ + goto l13 + l14: + position, tokenIndex = position14, tokenIndex14 + } + if !_rules[rulecondition]() { + goto l3 + } + goto l2 + l3: + position, tokenIndex = position3, tokenIndex3 + } + if buffer[position] != rune('"') { + goto l0 + } + position++ + { + position15, tokenIndex15 := position, tokenIndex + if !matchDot() { + goto l15 + } + goto l0 + l15: + position, tokenIndex = position15, tokenIndex15 + } + add(rulee, position1) + } + return true + l0: + position, tokenIndex = position0, tokenIndex0 + return false + }, + /* 1 condition <- <(tag ' '* ((le ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number))) / (ge ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number))) / ((&('=') (equal ' '* (number / time / date / value))) | (&('>') (g ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('<') (l ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('C' | 'c') (contains ' '* value)))))> */ + func() bool { + position16, tokenIndex16 := position, tokenIndex + { + position17 := position + { + position18 := position + { + position19 := position + { + position22, tokenIndex22 := position, tokenIndex + { + switch buffer[position] { + case '<': + if buffer[position] != rune('<') { + goto l22 + } + position++ + break + case '>': + if buffer[position] != rune('>') { + goto l22 + } + position++ + break + case '=': + if buffer[position] != rune('=') { + goto l22 + } + position++ + break + case '"': + if buffer[position] != rune('"') { + goto l22 + } + position++ + break + case ')': + if buffer[position] != rune(')') { + goto l22 + } + position++ + break + case '(': + if buffer[position] != rune('(') { + goto l22 + } + position++ + break + case '\\': + if buffer[position] != rune('\\') { + goto l22 + } + position++ + break + case '\r': + if buffer[position] != rune('\r') { + goto l22 + } + position++ + break + case '\n': + if buffer[position] != rune('\n') { + goto l22 + } + position++ + break + case '\t': + if buffer[position] != rune('\t') { + goto l22 + } + position++ + break + default: + if buffer[position] != rune(' ') { + goto l22 + } + position++ + break + } + } + + goto l16 + l22: + position, tokenIndex = position22, tokenIndex22 + } + if !matchDot() { + goto l16 + } + l20: + { + position21, tokenIndex21 := position, tokenIndex + { + position24, tokenIndex24 := position, tokenIndex + { + switch buffer[position] { + case '<': + if buffer[position] != rune('<') { + goto l24 + } + position++ + break + case '>': + if buffer[position] != rune('>') { + goto l24 + } + position++ + break + case '=': + if buffer[position] != rune('=') { + goto l24 + } + position++ + break + case '"': + if buffer[position] != rune('"') { + goto l24 + } + position++ + break + case ')': + if buffer[position] != rune(')') { + goto l24 + } + position++ + break + case '(': + if buffer[position] != rune('(') { + goto l24 + } + position++ + break + case '\\': + if buffer[position] != rune('\\') { + goto l24 + } + position++ + break + case '\r': + if buffer[position] != rune('\r') { + goto l24 + } + position++ + break + case '\n': + if buffer[position] != rune('\n') { + goto l24 + } + position++ + break + case '\t': + if buffer[position] != rune('\t') { + goto l24 + } + position++ + break + default: + if buffer[position] != rune(' ') { + goto l24 + } + position++ + break + } + } + + goto l21 + l24: + position, tokenIndex = position24, tokenIndex24 + } + if !matchDot() { + goto l21 + } + goto l20 + l21: + position, tokenIndex = position21, tokenIndex21 + } + add(rulePegText, position19) + } + add(ruletag, position18) + } + l26: + { + position27, tokenIndex27 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l27 + } + position++ + goto l26 + l27: + position, tokenIndex = position27, tokenIndex27 + } + { + position28, tokenIndex28 := position, tokenIndex + { + position30 := position + if buffer[position] != rune('<') { + goto l29 + } + position++ + if buffer[position] != rune('=') { + goto l29 + } + position++ + add(rulele, position30) + } + l31: + { + position32, tokenIndex32 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l32 + } + position++ + goto l31 + l32: + position, tokenIndex = position32, tokenIndex32 + } + { + switch buffer[position] { + case 'D', 'd': + if !_rules[ruledate]() { + goto l29 + } + break + case 'T', 't': + if !_rules[ruletime]() { + goto l29 + } + break + default: + if !_rules[rulenumber]() { + goto l29 + } + break + } + } + + goto l28 + l29: + position, tokenIndex = position28, tokenIndex28 + { + position35 := position + if buffer[position] != rune('>') { + goto l34 + } + position++ + if buffer[position] != rune('=') { + goto l34 + } + position++ + add(rulege, position35) + } + l36: + { + position37, tokenIndex37 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l37 + } + position++ + goto l36 + l37: + position, tokenIndex = position37, tokenIndex37 + } + { + switch buffer[position] { + case 'D', 'd': + if !_rules[ruledate]() { + goto l34 + } + break + case 'T', 't': + if !_rules[ruletime]() { + goto l34 + } + break + default: + if !_rules[rulenumber]() { + goto l34 + } + break + } + } + + goto l28 + l34: + position, tokenIndex = position28, tokenIndex28 + { + switch buffer[position] { + case '=': + { + position40 := position + if buffer[position] != rune('=') { + goto l16 + } + position++ + add(ruleequal, position40) + } + l41: + { + position42, tokenIndex42 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l42 + } + position++ + goto l41 + l42: + position, tokenIndex = position42, tokenIndex42 + } + { + position43, tokenIndex43 := position, tokenIndex + if !_rules[rulenumber]() { + goto l44 + } + goto l43 + l44: + position, tokenIndex = position43, tokenIndex43 + if !_rules[ruletime]() { + goto l45 + } + goto l43 + l45: + position, tokenIndex = position43, tokenIndex43 + if !_rules[ruledate]() { + goto l46 + } + goto l43 + l46: + position, tokenIndex = position43, tokenIndex43 + if !_rules[rulevalue]() { + goto l16 + } + } + l43: + break + case '>': + { + position47 := position + if buffer[position] != rune('>') { + goto l16 + } + position++ + add(ruleg, position47) + } + l48: + { + position49, tokenIndex49 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l49 + } + position++ + goto l48 + l49: + position, tokenIndex = position49, tokenIndex49 + } + { + switch buffer[position] { + case 'D', 'd': + if !_rules[ruledate]() { + goto l16 + } + break + case 'T', 't': + if !_rules[ruletime]() { + goto l16 + } + break + default: + if !_rules[rulenumber]() { + goto l16 + } + break + } + } + + break + case '<': + { + position51 := position + if buffer[position] != rune('<') { + goto l16 + } + position++ + add(rulel, position51) + } + l52: + { + position53, tokenIndex53 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l53 + } + position++ + goto l52 + l53: + position, tokenIndex = position53, tokenIndex53 + } + { + switch buffer[position] { + case 'D', 'd': + if !_rules[ruledate]() { + goto l16 + } + break + case 'T', 't': + if !_rules[ruletime]() { + goto l16 + } + break + default: + if !_rules[rulenumber]() { + goto l16 + } + break + } + } + + break + default: + { + position55 := position + { + position56, tokenIndex56 := position, tokenIndex + if buffer[position] != rune('c') { + goto l57 + } + position++ + goto l56 + l57: + position, tokenIndex = position56, tokenIndex56 + if buffer[position] != rune('C') { + goto l16 + } + position++ + } + l56: + { + position58, tokenIndex58 := position, tokenIndex + if buffer[position] != rune('o') { + goto l59 + } + position++ + goto l58 + l59: + position, tokenIndex = position58, tokenIndex58 + if buffer[position] != rune('O') { + goto l16 + } + position++ + } + l58: + { + position60, tokenIndex60 := position, tokenIndex + if buffer[position] != rune('n') { + goto l61 + } + position++ + goto l60 + l61: + position, tokenIndex = position60, tokenIndex60 + if buffer[position] != rune('N') { + goto l16 + } + position++ + } + l60: + { + position62, tokenIndex62 := position, tokenIndex + if buffer[position] != rune('t') { + goto l63 + } + position++ + goto l62 + l63: + position, tokenIndex = position62, tokenIndex62 + if buffer[position] != rune('T') { + goto l16 + } + position++ + } + l62: + { + position64, tokenIndex64 := position, tokenIndex + if buffer[position] != rune('a') { + goto l65 + } + position++ + goto l64 + l65: + position, tokenIndex = position64, tokenIndex64 + if buffer[position] != rune('A') { + goto l16 + } + position++ + } + l64: + { + position66, tokenIndex66 := position, tokenIndex + if buffer[position] != rune('i') { + goto l67 + } + position++ + goto l66 + l67: + position, tokenIndex = position66, tokenIndex66 + if buffer[position] != rune('I') { + goto l16 + } + position++ + } + l66: + { + position68, tokenIndex68 := position, tokenIndex + if buffer[position] != rune('n') { + goto l69 + } + position++ + goto l68 + l69: + position, tokenIndex = position68, tokenIndex68 + if buffer[position] != rune('N') { + goto l16 + } + position++ + } + l68: + { + position70, tokenIndex70 := position, tokenIndex + if buffer[position] != rune('s') { + goto l71 + } + position++ + goto l70 + l71: + position, tokenIndex = position70, tokenIndex70 + if buffer[position] != rune('S') { + goto l16 + } + position++ + } + l70: + add(rulecontains, position55) + } + l72: + { + position73, tokenIndex73 := position, tokenIndex + if buffer[position] != rune(' ') { + goto l73 + } + position++ + goto l72 + l73: + position, tokenIndex = position73, tokenIndex73 + } + if !_rules[rulevalue]() { + goto l16 + } + break + } + } + + } + l28: + add(rulecondition, position17) + } + return true + l16: + position, tokenIndex = position16, tokenIndex16 + return false + }, + /* 2 tag <- <<(!((&('<') '<') | (&('>') '>') | (&('=') '=') | (&('"') '"') | (&(')') ')') | (&('(') '(') | (&('\\') '\\') | (&('\r') '\r') | (&('\n') '\n') | (&('\t') '\t') | (&(' ') ' ')) .)+>> */ + nil, + /* 3 value <- <<(!((&('<') '<') | (&('>') '>') | (&('=') '=') | (&('"') '"') | (&(')') ')') | (&('(') '(') | (&('\\') '\\') | (&('\r') '\r') | (&('\n') '\n') | (&('\t') '\t') | (&(' ') ' ')) .)+>> */ + func() bool { + position75, tokenIndex75 := position, tokenIndex + { + position76 := position + { + position77 := position + { + position80, tokenIndex80 := position, tokenIndex + { + switch buffer[position] { + case '<': + if buffer[position] != rune('<') { + goto l80 + } + position++ + break + case '>': + if buffer[position] != rune('>') { + goto l80 + } + position++ + break + case '=': + if buffer[position] != rune('=') { + goto l80 + } + position++ + break + case '"': + if buffer[position] != rune('"') { + goto l80 + } + position++ + break + case ')': + if buffer[position] != rune(')') { + goto l80 + } + position++ + break + case '(': + if buffer[position] != rune('(') { + goto l80 + } + position++ + break + case '\\': + if buffer[position] != rune('\\') { + goto l80 + } + position++ + break + case '\r': + if buffer[position] != rune('\r') { + goto l80 + } + position++ + break + case '\n': + if buffer[position] != rune('\n') { + goto l80 + } + position++ + break + case '\t': + if buffer[position] != rune('\t') { + goto l80 + } + position++ + break + default: + if buffer[position] != rune(' ') { + goto l80 + } + position++ + break + } + } + + goto l75 + l80: + position, tokenIndex = position80, tokenIndex80 + } + if !matchDot() { + goto l75 + } + l78: + { + position79, tokenIndex79 := position, tokenIndex + { + position82, tokenIndex82 := position, tokenIndex + { + switch buffer[position] { + case '<': + if buffer[position] != rune('<') { + goto l82 + } + position++ + break + case '>': + if buffer[position] != rune('>') { + goto l82 + } + position++ + break + case '=': + if buffer[position] != rune('=') { + goto l82 + } + position++ + break + case '"': + if buffer[position] != rune('"') { + goto l82 + } + position++ + break + case ')': + if buffer[position] != rune(')') { + goto l82 + } + position++ + break + case '(': + if buffer[position] != rune('(') { + goto l82 + } + position++ + break + case '\\': + if buffer[position] != rune('\\') { + goto l82 + } + position++ + break + case '\r': + if buffer[position] != rune('\r') { + goto l82 + } + position++ + break + case '\n': + if buffer[position] != rune('\n') { + goto l82 + } + position++ + break + case '\t': + if buffer[position] != rune('\t') { + goto l82 + } + position++ + break + default: + if buffer[position] != rune(' ') { + goto l82 + } + position++ + break + } + } + + goto l79 + l82: + position, tokenIndex = position82, tokenIndex82 + } + if !matchDot() { + goto l79 + } + goto l78 + l79: + position, tokenIndex = position79, tokenIndex79 + } + add(rulePegText, position77) + } + add(rulevalue, position76) + } + return true + l75: + position, tokenIndex = position75, tokenIndex75 + return false + }, + /* 4 number <- <<('0' / ([1-9] digit* ('.' digit*)?))>> */ + func() bool { + position84, tokenIndex84 := position, tokenIndex + { + position85 := position + { + position86 := position + { + position87, tokenIndex87 := position, tokenIndex + if buffer[position] != rune('0') { + goto l88 + } + position++ + goto l87 + l88: + position, tokenIndex = position87, tokenIndex87 + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l84 + } + position++ + l89: + { + position90, tokenIndex90 := position, tokenIndex + if !_rules[ruledigit]() { + goto l90 + } + goto l89 + l90: + position, tokenIndex = position90, tokenIndex90 + } + { + position91, tokenIndex91 := position, tokenIndex + if buffer[position] != rune('.') { + goto l91 + } + position++ + l93: + { + position94, tokenIndex94 := position, tokenIndex + if !_rules[ruledigit]() { + goto l94 + } + goto l93 + l94: + position, tokenIndex = position94, tokenIndex94 + } + goto l92 + l91: + position, tokenIndex = position91, tokenIndex91 + } + l92: + } + l87: + add(rulePegText, position86) + } + add(rulenumber, position85) + } + return true + l84: + position, tokenIndex = position84, tokenIndex84 + return false + }, + /* 5 digit <- <[0-9]> */ + func() bool { + position95, tokenIndex95 := position, tokenIndex + { + position96 := position + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l95 + } + position++ + add(ruledigit, position96) + } + return true + l95: + position, tokenIndex = position95, tokenIndex95 + return false + }, + /* 6 time <- <(('t' / 'T') ('i' / 'I') ('m' / 'M') ('e' / 'E') ' ' <(year '-' month '-' day 'T' digit digit ':' digit digit ':' digit digit ((('-' / '+') digit digit ':' digit digit) / 'Z'))>)> */ + func() bool { + position97, tokenIndex97 := position, tokenIndex + { + position98 := position + { + position99, tokenIndex99 := position, tokenIndex + if buffer[position] != rune('t') { + goto l100 + } + position++ + goto l99 + l100: + position, tokenIndex = position99, tokenIndex99 + if buffer[position] != rune('T') { + goto l97 + } + position++ + } + l99: + { + position101, tokenIndex101 := position, tokenIndex + if buffer[position] != rune('i') { + goto l102 + } + position++ + goto l101 + l102: + position, tokenIndex = position101, tokenIndex101 + if buffer[position] != rune('I') { + goto l97 + } + position++ + } + l101: + { + position103, tokenIndex103 := position, tokenIndex + if buffer[position] != rune('m') { + goto l104 + } + position++ + goto l103 + l104: + position, tokenIndex = position103, tokenIndex103 + if buffer[position] != rune('M') { + goto l97 + } + position++ + } + l103: + { + position105, tokenIndex105 := position, tokenIndex + if buffer[position] != rune('e') { + goto l106 + } + position++ + goto l105 + l106: + position, tokenIndex = position105, tokenIndex105 + if buffer[position] != rune('E') { + goto l97 + } + position++ + } + l105: + if buffer[position] != rune(' ') { + goto l97 + } + position++ + { + position107 := position + if !_rules[ruleyear]() { + goto l97 + } + if buffer[position] != rune('-') { + goto l97 + } + position++ + if !_rules[rulemonth]() { + goto l97 + } + if buffer[position] != rune('-') { + goto l97 + } + position++ + if !_rules[ruleday]() { + goto l97 + } + if buffer[position] != rune('T') { + goto l97 + } + position++ + if !_rules[ruledigit]() { + goto l97 + } + if !_rules[ruledigit]() { + goto l97 + } + if buffer[position] != rune(':') { + goto l97 + } + position++ + if !_rules[ruledigit]() { + goto l97 + } + if !_rules[ruledigit]() { + goto l97 + } + if buffer[position] != rune(':') { + goto l97 + } + position++ + if !_rules[ruledigit]() { + goto l97 + } + if !_rules[ruledigit]() { + goto l97 + } + { + position108, tokenIndex108 := position, tokenIndex + { + position110, tokenIndex110 := position, tokenIndex + if buffer[position] != rune('-') { + goto l111 + } + position++ + goto l110 + l111: + position, tokenIndex = position110, tokenIndex110 + if buffer[position] != rune('+') { + goto l109 + } + position++ + } + l110: + if !_rules[ruledigit]() { + goto l109 + } + if !_rules[ruledigit]() { + goto l109 + } + if buffer[position] != rune(':') { + goto l109 + } + position++ + if !_rules[ruledigit]() { + goto l109 + } + if !_rules[ruledigit]() { + goto l109 + } + goto l108 + l109: + position, tokenIndex = position108, tokenIndex108 + if buffer[position] != rune('Z') { + goto l97 + } + position++ + } + l108: + add(rulePegText, position107) + } + add(ruletime, position98) + } + return true + l97: + position, tokenIndex = position97, tokenIndex97 + return false + }, + /* 7 date <- <(('d' / 'D') ('a' / 'A') ('t' / 'T') ('e' / 'E') ' ' <(year '-' month '-' day)>)> */ + func() bool { + position112, tokenIndex112 := position, tokenIndex + { + position113 := position + { + position114, tokenIndex114 := position, tokenIndex + if buffer[position] != rune('d') { + goto l115 + } + position++ + goto l114 + l115: + position, tokenIndex = position114, tokenIndex114 + if buffer[position] != rune('D') { + goto l112 + } + position++ + } + l114: + { + position116, tokenIndex116 := position, tokenIndex + if buffer[position] != rune('a') { + goto l117 + } + position++ + goto l116 + l117: + position, tokenIndex = position116, tokenIndex116 + if buffer[position] != rune('A') { + goto l112 + } + position++ + } + l116: + { + position118, tokenIndex118 := position, tokenIndex + if buffer[position] != rune('t') { + goto l119 + } + position++ + goto l118 + l119: + position, tokenIndex = position118, tokenIndex118 + if buffer[position] != rune('T') { + goto l112 + } + position++ + } + l118: + { + position120, tokenIndex120 := position, tokenIndex + if buffer[position] != rune('e') { + goto l121 + } + position++ + goto l120 + l121: + position, tokenIndex = position120, tokenIndex120 + if buffer[position] != rune('E') { + goto l112 + } + position++ + } + l120: + if buffer[position] != rune(' ') { + goto l112 + } + position++ + { + position122 := position + if !_rules[ruleyear]() { + goto l112 + } + if buffer[position] != rune('-') { + goto l112 + } + position++ + if !_rules[rulemonth]() { + goto l112 + } + if buffer[position] != rune('-') { + goto l112 + } + position++ + if !_rules[ruleday]() { + goto l112 + } + add(rulePegText, position122) + } + add(ruledate, position113) + } + return true + l112: + position, tokenIndex = position112, tokenIndex112 + return false + }, + /* 8 year <- <(('1' / '2') digit digit digit)> */ + func() bool { + position123, tokenIndex123 := position, tokenIndex + { + position124 := position + { + position125, tokenIndex125 := position, tokenIndex + if buffer[position] != rune('1') { + goto l126 + } + position++ + goto l125 + l126: + position, tokenIndex = position125, tokenIndex125 + if buffer[position] != rune('2') { + goto l123 + } + position++ + } + l125: + if !_rules[ruledigit]() { + goto l123 + } + if !_rules[ruledigit]() { + goto l123 + } + if !_rules[ruledigit]() { + goto l123 + } + add(ruleyear, position124) + } + return true + l123: + position, tokenIndex = position123, tokenIndex123 + return false + }, + /* 9 month <- <(('0' / '1') digit)> */ + func() bool { + position127, tokenIndex127 := position, tokenIndex + { + position128 := position + { + position129, tokenIndex129 := position, tokenIndex + if buffer[position] != rune('0') { + goto l130 + } + position++ + goto l129 + l130: + position, tokenIndex = position129, tokenIndex129 + if buffer[position] != rune('1') { + goto l127 + } + position++ + } + l129: + if !_rules[ruledigit]() { + goto l127 + } + add(rulemonth, position128) + } + return true + l127: + position, tokenIndex = position127, tokenIndex127 + return false + }, + /* 10 day <- <(((&('3') '3') | (&('2') '2') | (&('1') '1') | (&('0') '0')) digit)> */ + func() bool { + position131, tokenIndex131 := position, tokenIndex + { + position132 := position + { + switch buffer[position] { + case '3': + if buffer[position] != rune('3') { + goto l131 + } + position++ + break + case '2': + if buffer[position] != rune('2') { + goto l131 + } + position++ + break + case '1': + if buffer[position] != rune('1') { + goto l131 + } + position++ + break + default: + if buffer[position] != rune('0') { + goto l131 + } + position++ + break + } + } + + if !_rules[ruledigit]() { + goto l131 + } + add(ruleday, position132) + } + return true + l131: + position, tokenIndex = position131, tokenIndex131 + return false + }, + /* 11 and <- <(('a' / 'A') ('n' / 'N') ('d' / 'D'))> */ + nil, + /* 12 equal <- <'='> */ + nil, + /* 13 contains <- <(('c' / 'C') ('o' / 'O') ('n' / 'N') ('t' / 'T') ('a' / 'A') ('i' / 'I') ('n' / 'N') ('s' / 'S'))> */ + nil, + /* 14 le <- <('<' '=')> */ + nil, + /* 15 ge <- <('>' '=')> */ + nil, + /* 16 l <- <'<'> */ + nil, + /* 17 g <- <'>'> */ + nil, + nil, + } + p.rules = _rules +} diff --git a/pubsub/query/query_test.go b/pubsub/query/query_test.go new file mode 100644 index 00000000..75d02ee4 --- /dev/null +++ b/pubsub/query/query_test.go @@ -0,0 +1,64 @@ +package query_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tendermint/tmlibs/pubsub/query" +) + +func TestMatches(t *testing.T) { + const shortForm = "2006-Jan-02" + txDate, err := time.Parse(shortForm, "2017-Jan-01") + require.NoError(t, err) + txTime, err := time.Parse(time.RFC3339, "2018-05-03T14:45:00Z") + require.NoError(t, err) + + testCases := []struct { + s string + tags map[string]interface{} + err bool + matches bool + }{ + {"tm.events.type=NewBlock", map[string]interface{}{"tm.events.type": "NewBlock"}, false, true}, + + {"tx.gas > 7", map[string]interface{}{"tx.gas": 8}, false, true}, + {"tx.gas > 7 AND tx.gas < 9", map[string]interface{}{"tx.gas": 8}, false, true}, + {"body.weight >= 3.5", map[string]interface{}{"body.weight": 3.5}, false, true}, + {"account.balance < 1000.0", map[string]interface{}{"account.balance": 900}, false, true}, + {"apples.kg <= 4", map[string]interface{}{"apples.kg": 4.0}, false, true}, + {"body.weight >= 4.5", map[string]interface{}{"body.weight": float32(4.5)}, false, true}, + {"oranges.kg < 4 AND watermellons.kg > 10", map[string]interface{}{"oranges.kg": 3, "watermellons.kg": 12}, false, true}, + {"peaches.kg < 4", map[string]interface{}{"peaches.kg": 5}, false, false}, + + {"tx.date > DATE 2017-01-01", map[string]interface{}{"tx.date": time.Now()}, false, true}, + {"tx.date = DATE 2017-01-01", map[string]interface{}{"tx.date": txDate}, false, true}, + {"tx.date = DATE 2018-01-01", map[string]interface{}{"tx.date": txDate}, false, false}, + + {"tx.time >= TIME 2013-05-03T14:45:00Z", map[string]interface{}{"tx.time": time.Now()}, false, true}, + {"tx.time = TIME 2013-05-03T14:45:00Z", map[string]interface{}{"tx.time": txTime}, false, false}, + + {"abci.owner.name CONTAINS Igor", map[string]interface{}{"abci.owner.name": "Igor,Ivan"}, false, true}, + {"abci.owner.name CONTAINS Igor", map[string]interface{}{"abci.owner.name": "Pavel,Ivan"}, false, false}, + } + + for _, tc := range testCases { + query, err := query.New(tc.s) + if !tc.err { + require.Nil(t, err) + } + + if tc.matches { + assert.True(t, query.Matches(tc.tags), "Query '%s' should match %v", tc.s, tc.tags) + } else { + assert.False(t, query.Matches(tc.tags), "Query '%s' should not match %v", tc.s, tc.tags) + } + } +} + +func TestMustParse(t *testing.T) { + assert.Panics(t, func() { query.MustParse("=") }) + assert.NotPanics(t, func() { query.MustParse("tm.events.type=NewBlock") }) +} From 8062ade7876d1d82b7385c09a3cdb6aa0b7ba490 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 12 Jul 2017 13:10:36 +0300 Subject: [PATCH 11/37] remove all clients (including closing all channels) on shutdown --- pubsub/pubsub.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go index 26484816..ae642a4f 100644 --- a/pubsub/pubsub.go +++ b/pubsub/pubsub.go @@ -216,7 +216,9 @@ loop: state.removeAll(cmd.clientID) } case shutdown: - state.reset() + for clientID, _ := range state.clients { + state.removeAll(clientID) + } break loop case sub: state.add(cmd.clientID, cmd.query, cmd.ch) @@ -286,11 +288,6 @@ func (state *state) removeAll(clientID string) { delete(state.clients, clientID) } -func (state *state) reset() { - state.queries = make(map[Query]map[string]chan<- interface{}) - state.clients = make(map[string]map[Query]struct{}) -} - func (state *state) send(msg interface{}, tags map[string]interface{}, slowClientStrategy overflowStrategy, logger log.Logger) { for q, clientToChannelMap := range state.queries { // NOTE we can use LRU cache to speed up common cases like query = " From e4f3f9d9bf327083eba26afa2b85ff09189856c3 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 12 Jul 2017 22:52:13 +0300 Subject: [PATCH 12/37] remove comment about LRU cache (see comments below) I've tried https://github.com/hashicorp/golang-lru/tree/master/simplelru today and here are the results: with LRU cache: ``` Benchmark10Clients-2 50000 29021 ns/op 3976 B/op 105 allocs/op Benchmark100Clients-2 3000 363432 ns/op 36382 B/op 1005 allocs/op Benchmark1000Clients-2 500 2473752 ns/op 360500 B/op 10009 allocs/op Benchmark10ClientsUsingTheSameQuery-2 300000 4059 ns/op 773 B/op 15 allocs/op Benchmark100ClientsUsingTheSameQuery-2 500000 4360 ns/op 773 B/op 15 allocs/op Benchmark1000ClientsUsingTheSameQuery-2 300000 4204 ns/op 773 B/op 15 allocs/op ``` without LRU cache: ``` Benchmark10Clients-2 200000 5267 ns/op 616 B/op 25 allocs/op Benchmark100Clients-2 30000 42134 ns/op 2776 B/op 205 allocs/op Benchmark1000Clients-2 3000 552648 ns/op 24376 B/op 2005 allocs/op Benchmark10ClientsOneQuery-2 1000000 2127 ns/op 462 B/op 9 allocs/op Benchmark100ClientsOneQuery-2 500000 2353 ns/op 462 B/op 9 allocs/op Benchmark1000ClientsOneQuery-2 500000 2339 ns/op 462 B/op 9 allocs/op ``` > How were you using the lru cache exactly? I was adding a KV pair each time there is a match plus checking if `lru.Contains(key)` before running the actual check (`q.Matches(tags)`). ``` key = fmt.Sprintf("%s/%v", query + tags) ``` --- pubsub/pubsub.go | 3 --- pubsub/pubsub_test.go | 32 +++++++++++++++++++++++++++++++- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go index ae642a4f..007f93f3 100644 --- a/pubsub/pubsub.go +++ b/pubsub/pubsub.go @@ -290,11 +290,8 @@ func (state *state) removeAll(clientID string) { func (state *state) send(msg interface{}, tags map[string]interface{}, slowClientStrategy overflowStrategy, logger log.Logger) { for q, clientToChannelMap := range state.queries { - // NOTE we can use LRU cache to speed up common cases like query = " - // tm.events.type=NewBlock" and tags = {"tm.events.type": "NewBlock"} if q.Matches(tags) { for clientID, ch := range clientToChannelMap { - logger.Info("Sending message to client", "msg", msg, "client", clientID) switch slowClientStrategy { case drop: select { diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index 570f76a8..3112ab5d 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -193,13 +193,21 @@ func Benchmark10Clients(b *testing.B) { benchmarkNClients(10, b) } func Benchmark100Clients(b *testing.B) { benchmarkNClients(100, b) } func Benchmark1000Clients(b *testing.B) { benchmarkNClients(1000, b) } +func Benchmark10ClientsOneQuery(b *testing.B) { benchmarkNClientsOneQuery(10, b) } +func Benchmark100ClientsOneQuery(b *testing.B) { benchmarkNClientsOneQuery(100, b) } +func Benchmark1000ClientsOneQuery(b *testing.B) { benchmarkNClientsOneQuery(1000, b) } + func benchmarkNClients(n int, b *testing.B) { - s := pubsub.NewServer(pubsub.BufferCapacity(b.N)) + s := pubsub.NewServer() s.Start() defer s.Stop() for i := 0; i < n; i++ { ch := make(chan interface{}) + go func() { + for range ch { + } + }() s.Subscribe(clientID, query.MustParse(fmt.Sprintf("abci.Account.Owner = Ivan AND abci.Invoices.Number = %d", i)), ch) } @@ -210,6 +218,28 @@ func benchmarkNClients(n int, b *testing.B) { } } +func benchmarkNClientsOneQuery(n int, b *testing.B) { + s := pubsub.NewServer() + s.Start() + defer s.Stop() + + q := query.MustParse("abci.Account.Owner = Ivan AND abci.Invoices.Number = 1") + for i := 0; i < n; i++ { + ch := make(chan interface{}) + go func() { + for range ch { + } + }() + s.Subscribe(clientID, q, ch) + } + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.PublishWithTags("Gamora", map[string]interface{}{"abci.Account.Owner": "Ivan", "abci.Invoices.Number": 1}) + } +} + /////////////////////////////////////////////////////////////////////////////// /// HELPERS /////////////////////////////////////////////////////////////////////////////// From 4aa024d843b081977304c0184e8c56c05d22c32f Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Wed, 12 Jul 2017 23:10:36 +0300 Subject: [PATCH 13/37] add more info to error messages --- pubsub/pubsub.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go index 007f93f3..aec60bcc 100644 --- a/pubsub/pubsub.go +++ b/pubsub/pubsub.go @@ -26,6 +26,7 @@ package pubsub import ( "errors" + "fmt" cmn "github.com/tendermint/tmlibs/common" "github.com/tendermint/tmlibs/log" @@ -174,7 +175,7 @@ func (s *Server) PublishWithTags(msg interface{}, tags map[string]interface{}) e select { case s.cmds <- pubCmd: default: - s.Logger.Error("Server overflowed, dropping message...", "msg", msg) + s.Logger.Error("Server overflowed, dropping message...", "msg", msg, "tags", fmt.Sprintf("%v", tags)) return ErrorOverflow } case wait: @@ -297,7 +298,7 @@ func (state *state) send(msg interface{}, tags map[string]interface{}, slowClien select { case ch <- msg: default: - logger.Error("Client is busy, skipping...", "clientID", clientID) + logger.Error("Wanted to send a message, but the client is busy", "msg", msg, "tags", fmt.Sprintf("%v", tags), "clientID", clientID) } case wait: ch <- msg From 13207a5927e21d96afb9a9520f5a1a7b42b323bb Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 14 Jul 2017 12:32:01 +0300 Subject: [PATCH 14/37] remove overflow options --- pubsub/pubsub.go | 91 ++++--------------------------------------- pubsub/pubsub_test.go | 53 ------------------------- 2 files changed, 8 insertions(+), 136 deletions(-) diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go index aec60bcc..2b1a569c 100644 --- a/pubsub/pubsub.go +++ b/pubsub/pubsub.go @@ -10,26 +10,13 @@ // match, this message will be pushed to all clients, subscribed to that query. // See query subpackage for our implementation. // -// Overflow strategies (incoming publish requests): -// -// 1) drop - drops publish requests when there are too many of them -// 2) wait - blocks until the server is ready to accept more publish requests (default) -// // Subscribe/Unsubscribe calls are always blocking. -// -// Overflow strategies (outgoing messages): -// -// 1) skip - do not send a message if the client is busy or slow (default) -// 2) wait - wait until the client is ready to accept new messages -// package pubsub import ( "errors" - "fmt" cmn "github.com/tendermint/tmlibs/common" - "github.com/tendermint/tmlibs/log" ) type operation int @@ -41,13 +28,6 @@ const ( shutdown ) -type overflowStrategy int - -const ( - drop overflowStrategy = iota - wait -) - var ( ErrorOverflow = errors.New("Server overflowed") ) @@ -72,20 +52,16 @@ type Server struct { cmn.BaseService cmds chan cmd - - overflowStrategy overflowStrategy - slowClientStrategy overflowStrategy } // Option sets a parameter for the server. type Option func(*Server) // NewServer returns a new server. See the commentary on the Option functions -// for a detailed description of how to configure buffering and overflow -// behavior. If no options are provided, the resulting server's queue is -// unbuffered and it blocks when overflowed. +// for a detailed description of how to configure buffering. If no options are +// provided, the resulting server's queue is unbuffered. func NewServer(options ...Option) *Server { - s := &Server{overflowStrategy: wait, slowClientStrategy: drop} + s := &Server{} s.BaseService = *cmn.NewBaseService(nil, "PubSub", s) for _, option := range options { @@ -111,38 +87,6 @@ func BufferCapacity(cap int) Option { } } -// OverflowStrategyDrop will tell the server to drop messages when it can't -// process more messages. -func OverflowStrategyDrop() Option { - return func(s *Server) { - s.overflowStrategy = drop - } -} - -// OverflowStrategyWait will tell the server to block and wait for some time -// for server to process other messages. Default strategy. -func OverflowStrategyWait() func(*Server) { - return func(s *Server) { - s.overflowStrategy = wait - } -} - -// WaitSlowClients will tell the server to block and wait until subscriber -// reads a messages even if it is fast enough to process them. -func WaitSlowClients() func(*Server) { - return func(s *Server) { - s.slowClientStrategy = wait - } -} - -// SkipSlowClients will tell the server to skip subscriber if it is busy -// processing previous message(s). Default strategy. -func SkipSlowClients() func(*Server) { - return func(s *Server) { - s.slowClientStrategy = drop - } -} - // Subscribe returns a channel on which messages matching the given query can // be received. If the subscription already exists old channel will be closed // and new one returned. @@ -170,17 +114,7 @@ func (s *Server) Publish(msg interface{}) error { // will be sent to a client. func (s *Server) PublishWithTags(msg interface{}, tags map[string]interface{}) error { pubCmd := cmd{op: pub, msg: msg, tags: tags} - switch s.overflowStrategy { - case drop: - select { - case s.cmds <- pubCmd: - default: - s.Logger.Error("Server overflowed, dropping message...", "msg", msg, "tags", fmt.Sprintf("%v", tags)) - return ErrorOverflow - } - case wait: - s.cmds <- pubCmd - } + s.cmds <- pubCmd return nil } @@ -224,7 +158,7 @@ loop: case sub: state.add(cmd.clientID, cmd.query, cmd.ch) case pub: - state.send(cmd.msg, cmd.tags, s.slowClientStrategy, s.Logger) + state.send(cmd.msg, cmd.tags) } } } @@ -289,20 +223,11 @@ func (state *state) removeAll(clientID string) { delete(state.clients, clientID) } -func (state *state) send(msg interface{}, tags map[string]interface{}, slowClientStrategy overflowStrategy, logger log.Logger) { +func (state *state) send(msg interface{}, tags map[string]interface{}) { for q, clientToChannelMap := range state.queries { if q.Matches(tags) { - for clientID, ch := range clientToChannelMap { - switch slowClientStrategy { - case drop: - select { - case ch <- msg: - default: - logger.Error("Wanted to send a message, but the client is busy", "msg", msg, "tags", fmt.Sprintf("%v", tags), "clientID", clientID) - } - case wait: - ch <- msg - } + for _, ch := range clientToChannelMap { + ch <- msg } } } diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index 3112ab5d..7cc4e599 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -126,30 +126,6 @@ func TestUnsubscribeAll(t *testing.T) { assert.False(t, ok) } -func TestOverflowStrategyDrop(t *testing.T) { - s := pubsub.NewServer(pubsub.OverflowStrategyDrop()) - s.SetLogger(log.TestingLogger()) - - err := s.Publish("Veda") - if assert.Error(t, err) { - assert.Equal(t, pubsub.ErrorOverflow, err) - } -} - -func TestOverflowStrategyWait(t *testing.T) { - s := pubsub.NewServer(pubsub.OverflowStrategyWait()) - s.SetLogger(log.TestingLogger()) - - go func() { - time.Sleep(1 * time.Second) - s.Start() - defer s.Stop() - }() - - err := s.Publish("Veda") - assert.NoError(t, err) -} - func TestBufferCapacity(t *testing.T) { s := pubsub.NewServer(pubsub.BufferCapacity(2)) s.SetLogger(log.TestingLogger()) @@ -160,35 +136,6 @@ func TestBufferCapacity(t *testing.T) { require.NoError(t, err) } -func TestWaitSlowClients(t *testing.T) { - s := pubsub.NewServer(pubsub.WaitSlowClients()) - s.SetLogger(log.TestingLogger()) - s.Start() - defer s.Stop() - - ch := make(chan interface{}) - s.Subscribe(clientID, query.Empty{}, ch) - err := s.Publish("Wonderwoman") - require.NoError(t, err) - - time.Sleep(1 * time.Second) - - assertReceive(t, "Wonderwoman", ch) -} - -func TestSkipSlowClients(t *testing.T) { - s := pubsub.NewServer(pubsub.SkipSlowClients()) - s.SetLogger(log.TestingLogger()) - s.Start() - defer s.Stop() - - ch := make(chan interface{}) - s.Subscribe(clientID, query.Empty{}, ch) - err := s.Publish("Cyclops") - require.NoError(t, err) - assert.Zero(t, len(ch)) -} - func Benchmark10Clients(b *testing.B) { benchmarkNClients(10, b) } func Benchmark100Clients(b *testing.B) { benchmarkNClients(100, b) } func Benchmark1000Clients(b *testing.B) { benchmarkNClients(1000, b) } From 0006bfc359e2b50a6a083ea750e2b1368477fcbc Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 14 Jul 2017 13:02:32 +0300 Subject: [PATCH 15/37] return ErrorOverflow on Subscribe if server is overflowed > why we need it? most of our subscribers will be RPC WS subscribers, so if there are too many, nothing wrong with rejecting to subscribe. however, consensus reactor must be the first to subscribe, since its work depends on the pubsub package. --- pubsub/example_test.go | 4 +-- pubsub/pubsub.go | 54 ++++++++++++++++++++++--------------- pubsub/pubsub_test.go | 61 ++++++++++++++++++++++++------------------ 3 files changed, 70 insertions(+), 49 deletions(-) diff --git a/pubsub/example_test.go b/pubsub/example_test.go index d64b96ea..38026ccd 100644 --- a/pubsub/example_test.go +++ b/pubsub/example_test.go @@ -17,8 +17,8 @@ func TestExample(t *testing.T) { defer s.Stop() ch := make(chan interface{}, 1) - s.Subscribe("example-client", query.MustParse("abci.account.name=John"), ch) - err := s.PublishWithTags("Tombstone", map[string]interface{}{"abci.account.name": "John"}) + err := s.Subscribe("example-client", query.MustParse("abci.account.name=John"), ch) require.NoError(t, err) + s.PublishWithTags("Tombstone", map[string]interface{}{"abci.account.name": "John"}) assertReceive(t, "Tombstone", ch) } diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go index 2b1a569c..9ac260c9 100644 --- a/pubsub/pubsub.go +++ b/pubsub/pubsub.go @@ -15,6 +15,7 @@ package pubsub import ( "errors" + "time" cmn "github.com/tendermint/tmlibs/common" ) @@ -28,8 +29,10 @@ const ( shutdown ) +const subscribeTimeout = 10 * time.Millisecond + var ( - ErrorOverflow = errors.New("Server overflowed") + ErrorOverflow = errors.New("server overflowed") ) type cmd struct { @@ -51,7 +54,8 @@ type Query interface { type Server struct { cmn.BaseService - cmds chan cmd + cmds chan cmd + cmdsCap int } // Option sets a parameter for the server. @@ -68,9 +72,8 @@ func NewServer(options ...Option) *Server { option(s) } - if s.cmds == nil { // if BufferCapacity was not set, create unbuffered channel - s.cmds = make(chan cmd) - } + // if BufferCapacity option was not set, the channel is unbuffered + s.cmds = make(chan cmd, s.cmdsCap) return s } @@ -82,40 +85,49 @@ func NewServer(options ...Option) *Server { func BufferCapacity(cap int) Option { return func(s *Server) { if cap > 0 { - s.cmds = make(chan cmd, cap) + s.cmdsCap = cap } } } -// Subscribe returns a channel on which messages matching the given query can -// be received. If the subscription already exists old channel will be closed -// and new one returned. -func (s *Server) Subscribe(clientID string, query Query, out chan<- interface{}) { - s.cmds <- cmd{op: sub, clientID: clientID, query: query, ch: out} +// Returns capacity of the internal server's queue. +func (s Server) BufferCapacity() int { + return s.cmdsCap } -// Unsubscribe unsubscribes the given client from the query. +// Subscribe returns a channel on which messages matching the given query can +// be received. If the subscription already exists old channel will be closed +// and new one returned. Error will be returned to the caller if the server is +// overflowed. +func (s *Server) Subscribe(clientID string, query Query, out chan<- interface{}) error { + select { + case s.cmds <- cmd{op: sub, clientID: clientID, query: query, ch: out}: + return nil + case <-time.After(subscribeTimeout): + return ErrorOverflow + } +} + +// Unsubscribe unsubscribes the given client from the query. Blocking. func (s *Server) Unsubscribe(clientID string, query Query) { s.cmds <- cmd{op: unsub, clientID: clientID, query: query} } -// Unsubscribe unsubscribes the given channel. +// Unsubscribe unsubscribes the given channel. Blocking. func (s *Server) UnsubscribeAll(clientID string) { s.cmds <- cmd{op: unsub, clientID: clientID} } -// Publish publishes the given message. -func (s *Server) Publish(msg interface{}) error { - return s.PublishWithTags(msg, make(map[string]interface{})) +// Publish publishes the given message. Blocking. +func (s *Server) Publish(msg interface{}) { + s.PublishWithTags(msg, make(map[string]interface{})) } // PublishWithTags publishes the given message with a set of tags. This set of // tags will be matched with client queries. If there is a match, the message -// will be sent to a client. -func (s *Server) PublishWithTags(msg interface{}, tags map[string]interface{}) error { - pubCmd := cmd{op: pub, msg: msg, tags: tags} - s.cmds <- pubCmd - return nil +// will be sent to a client. Blocking. +func (s *Server) PublishWithTags(msg interface{}, tags map[string]interface{}) { + s.cmds <- cmd{op: pub, msg: msg, tags: tags} } // OnStop implements Service.OnStop by shutting down the server. diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index 7cc4e599..fb15b348 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -25,13 +25,12 @@ func TestSubscribe(t *testing.T) { defer s.Stop() ch := make(chan interface{}, 1) - s.Subscribe(clientID, query.Empty{}, ch) - err := s.Publish("Ka-Zar") + err := s.Subscribe(clientID, query.Empty{}, ch) require.NoError(t, err) + s.Publish("Ka-Zar") assertReceive(t, "Ka-Zar", ch) - err = s.Publish("Quicksilver") - require.NoError(t, err) + s.Publish("Quicksilver") assertReceive(t, "Quicksilver", ch) } @@ -41,22 +40,22 @@ func TestDifferentClients(t *testing.T) { s.Start() defer s.Stop() ch1 := make(chan interface{}, 1) - s.Subscribe("client-1", query.MustParse("tm.events.type=NewBlock"), ch1) - err := s.PublishWithTags("Iceman", map[string]interface{}{"tm.events.type": "NewBlock"}) + err := s.Subscribe("client-1", query.MustParse("tm.events.type=NewBlock"), ch1) require.NoError(t, err) + s.PublishWithTags("Iceman", map[string]interface{}{"tm.events.type": "NewBlock"}) assertReceive(t, "Iceman", ch1) ch2 := make(chan interface{}, 1) - s.Subscribe("client-2", query.MustParse("tm.events.type=NewBlock AND abci.account.name=Igor"), ch2) - err = s.PublishWithTags("Ultimo", map[string]interface{}{"tm.events.type": "NewBlock", "abci.account.name": "Igor"}) + err = s.Subscribe("client-2", query.MustParse("tm.events.type=NewBlock AND abci.account.name=Igor"), ch2) require.NoError(t, err) + s.PublishWithTags("Ultimo", map[string]interface{}{"tm.events.type": "NewBlock", "abci.account.name": "Igor"}) assertReceive(t, "Ultimo", ch1) assertReceive(t, "Ultimo", ch2) ch3 := make(chan interface{}, 1) - s.Subscribe("client-3", query.MustParse("tm.events.type=NewRoundStep AND abci.account.name=Igor AND abci.invoice.number = 10"), ch3) - err = s.PublishWithTags("Valeria Richards", map[string]interface{}{"tm.events.type": "NewRoundStep"}) + err = s.Subscribe("client-3", query.MustParse("tm.events.type=NewRoundStep AND abci.account.name=Igor AND abci.invoice.number = 10"), ch3) require.NoError(t, err) + s.PublishWithTags("Valeria Richards", map[string]interface{}{"tm.events.type": "NewRoundStep"}) assert.Zero(t, len(ch3)) } @@ -69,19 +68,19 @@ func TestClientResubscribes(t *testing.T) { q := query.MustParse("tm.events.type=NewBlock") ch1 := make(chan interface{}, 1) - s.Subscribe(clientID, q, ch1) - err := s.PublishWithTags("Goblin Queen", map[string]interface{}{"tm.events.type": "NewBlock"}) + err := s.Subscribe(clientID, q, ch1) require.NoError(t, err) + s.PublishWithTags("Goblin Queen", map[string]interface{}{"tm.events.type": "NewBlock"}) assertReceive(t, "Goblin Queen", ch1) ch2 := make(chan interface{}, 1) - s.Subscribe(clientID, q, ch2) + err = s.Subscribe(clientID, q, ch2) + require.NoError(t, err) _, ok := <-ch1 assert.False(t, ok) - err = s.PublishWithTags("Spider-Man", map[string]interface{}{"tm.events.type": "NewBlock"}) - require.NoError(t, err) + s.PublishWithTags("Spider-Man", map[string]interface{}{"tm.events.type": "NewBlock"}) assertReceive(t, "Spider-Man", ch2) } @@ -92,11 +91,11 @@ func TestUnsubscribe(t *testing.T) { defer s.Stop() ch := make(chan interface{}) - s.Subscribe(clientID, query.Empty{}, ch) + err := s.Subscribe(clientID, query.Empty{}, ch) + require.NoError(t, err) s.Unsubscribe(clientID, query.Empty{}) - err := s.Publish("Nick Fury") - require.NoError(t, err) + s.Publish("Nick Fury") assert.Zero(t, len(ch), "Should not receive anything after Unsubscribe") _, ok := <-ch @@ -110,13 +109,14 @@ func TestUnsubscribeAll(t *testing.T) { defer s.Stop() ch1, ch2 := make(chan interface{}, 1), make(chan interface{}, 1) - s.Subscribe(clientID, query.MustParse("tm.events.type=NewBlock"), ch1) - s.Subscribe(clientID, query.MustParse("tm.events.type=NewBlockHeader"), ch2) + err := s.Subscribe(clientID, query.MustParse("tm.events.type=NewBlock"), ch1) + require.NoError(t, err) + err = s.Subscribe(clientID, query.MustParse("tm.events.type=NewBlockHeader"), ch2) + require.NoError(t, err) s.UnsubscribeAll(clientID) - err := s.Publish("Nick Fury") - require.NoError(t, err) + s.Publish("Nick Fury") assert.Zero(t, len(ch1), "Should not receive anything after UnsubscribeAll") assert.Zero(t, len(ch2), "Should not receive anything after UnsubscribeAll") @@ -130,10 +130,19 @@ func TestBufferCapacity(t *testing.T) { s := pubsub.NewServer(pubsub.BufferCapacity(2)) s.SetLogger(log.TestingLogger()) - err := s.Publish("Nighthawk") - require.NoError(t, err) - err = s.Publish("Sage") - require.NoError(t, err) + s.Publish("Nighthawk") + s.Publish("Sage") +} + +func TestSubscribeReturnsErrorIfServerOverflowed(t *testing.T) { + s := pubsub.NewServer() + s.SetLogger(log.TestingLogger()) + + ch := make(chan interface{}, 1) + err := s.Subscribe(clientID, query.MustParse("tm.events.type=NewBlock"), ch) + if assert.Error(t, err) { + assert.Equal(t, pubsub.ErrorOverflow, err) + } } func Benchmark10Clients(b *testing.B) { benchmarkNClients(10, b) } From e664f9c68861299060174ba348ad64a6854551d3 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 14 Jul 2017 14:49:25 +0300 Subject: [PATCH 16/37] use context to provide timeouts! --- pubsub/example_test.go | 7 +++- pubsub/pubsub.go | 53 +++++++++++++++----------- pubsub/pubsub_test.go | 86 ++++++++++++++++++++++++------------------ 3 files changed, 85 insertions(+), 61 deletions(-) diff --git a/pubsub/example_test.go b/pubsub/example_test.go index 38026ccd..6597c858 100644 --- a/pubsub/example_test.go +++ b/pubsub/example_test.go @@ -1,6 +1,7 @@ package pubsub_test import ( + "context" "testing" "github.com/stretchr/testify/require" @@ -16,9 +17,11 @@ func TestExample(t *testing.T) { s.Start() defer s.Stop() + ctx := context.Background() ch := make(chan interface{}, 1) - err := s.Subscribe("example-client", query.MustParse("abci.account.name=John"), ch) + err := s.Subscribe(ctx, "example-client", query.MustParse("abci.account.name=John"), ch) + require.NoError(t, err) + err = s.PublishWithTags(ctx, "Tombstone", map[string]interface{}{"abci.account.name": "John"}) require.NoError(t, err) - s.PublishWithTags("Tombstone", map[string]interface{}{"abci.account.name": "John"}) assertReceive(t, "Tombstone", ch) } diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go index 9ac260c9..34df86a4 100644 --- a/pubsub/pubsub.go +++ b/pubsub/pubsub.go @@ -14,8 +14,7 @@ package pubsub import ( - "errors" - "time" + "context" cmn "github.com/tendermint/tmlibs/common" ) @@ -29,12 +28,6 @@ const ( shutdown ) -const subscribeTimeout = 10 * time.Millisecond - -var ( - ErrorOverflow = errors.New("server overflowed") -) - type cmd struct { op operation query Query @@ -97,37 +90,53 @@ func (s Server) BufferCapacity() int { // Subscribe returns a channel on which messages matching the given query can // be received. If the subscription already exists old channel will be closed -// and new one returned. Error will be returned to the caller if the server is -// overflowed. -func (s *Server) Subscribe(clientID string, query Query, out chan<- interface{}) error { +// and new one returned. Error will be returned to the caller if the context is +// cancelled. +func (s *Server) Subscribe(ctx context.Context, clientID string, query Query, out chan<- interface{}) error { select { case s.cmds <- cmd{op: sub, clientID: clientID, query: query, ch: out}: return nil - case <-time.After(subscribeTimeout): - return ErrorOverflow + case <-ctx.Done(): + return ctx.Err() } } -// Unsubscribe unsubscribes the given client from the query. Blocking. -func (s *Server) Unsubscribe(clientID string, query Query) { - s.cmds <- cmd{op: unsub, clientID: clientID, query: query} +// Unsubscribe unsubscribes the given client from the query. Error will be +// returned to the caller if the context is cancelled. +func (s *Server) Unsubscribe(ctx context.Context, clientID string, query Query) error { + select { + case s.cmds <- cmd{op: unsub, clientID: clientID, query: query}: + return nil + case <-ctx.Done(): + return ctx.Err() + } } // Unsubscribe unsubscribes the given channel. Blocking. -func (s *Server) UnsubscribeAll(clientID string) { - s.cmds <- cmd{op: unsub, clientID: clientID} +func (s *Server) UnsubscribeAll(ctx context.Context, clientID string) error { + select { + case s.cmds <- cmd{op: unsub, clientID: clientID}: + return nil + case <-ctx.Done(): + return ctx.Err() + } } // Publish publishes the given message. Blocking. -func (s *Server) Publish(msg interface{}) { - s.PublishWithTags(msg, make(map[string]interface{})) +func (s *Server) Publish(ctx context.Context, msg interface{}) error { + return s.PublishWithTags(ctx, msg, make(map[string]interface{})) } // PublishWithTags publishes the given message with a set of tags. This set of // tags will be matched with client queries. If there is a match, the message // will be sent to a client. Blocking. -func (s *Server) PublishWithTags(msg interface{}, tags map[string]interface{}) { - s.cmds <- cmd{op: pub, msg: msg, tags: tags} +func (s *Server) PublishWithTags(ctx context.Context, msg interface{}, tags map[string]interface{}) error { + select { + case s.cmds <- cmd{op: pub, msg: msg, tags: tags}: + return nil + case <-ctx.Done(): + return ctx.Err() + } } // OnStop implements Service.OnStop by shutting down the server. diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index fb15b348..9c984144 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -1,6 +1,7 @@ package pubsub_test import ( + "context" "fmt" "runtime/debug" "testing" @@ -24,13 +25,16 @@ func TestSubscribe(t *testing.T) { s.Start() defer s.Stop() + ctx := context.Background() ch := make(chan interface{}, 1) - err := s.Subscribe(clientID, query.Empty{}, ch) + err := s.Subscribe(ctx, clientID, query.Empty{}, ch) + require.NoError(t, err) + err = s.Publish(ctx, "Ka-Zar") require.NoError(t, err) - s.Publish("Ka-Zar") assertReceive(t, "Ka-Zar", ch) - s.Publish("Quicksilver") + err = s.Publish(ctx, "Quicksilver") + require.NoError(t, err) assertReceive(t, "Quicksilver", ch) } @@ -39,23 +43,28 @@ func TestDifferentClients(t *testing.T) { s.SetLogger(log.TestingLogger()) s.Start() defer s.Stop() + + ctx := context.Background() ch1 := make(chan interface{}, 1) - err := s.Subscribe("client-1", query.MustParse("tm.events.type=NewBlock"), ch1) + err := s.Subscribe(ctx, "client-1", query.MustParse("tm.events.type=NewBlock"), ch1) + require.NoError(t, err) + err = s.PublishWithTags(ctx, "Iceman", map[string]interface{}{"tm.events.type": "NewBlock"}) require.NoError(t, err) - s.PublishWithTags("Iceman", map[string]interface{}{"tm.events.type": "NewBlock"}) assertReceive(t, "Iceman", ch1) ch2 := make(chan interface{}, 1) - err = s.Subscribe("client-2", query.MustParse("tm.events.type=NewBlock AND abci.account.name=Igor"), ch2) + err = s.Subscribe(ctx, "client-2", query.MustParse("tm.events.type=NewBlock AND abci.account.name=Igor"), ch2) + require.NoError(t, err) + err = s.PublishWithTags(ctx, "Ultimo", map[string]interface{}{"tm.events.type": "NewBlock", "abci.account.name": "Igor"}) require.NoError(t, err) - s.PublishWithTags("Ultimo", map[string]interface{}{"tm.events.type": "NewBlock", "abci.account.name": "Igor"}) assertReceive(t, "Ultimo", ch1) assertReceive(t, "Ultimo", ch2) ch3 := make(chan interface{}, 1) - err = s.Subscribe("client-3", query.MustParse("tm.events.type=NewRoundStep AND abci.account.name=Igor AND abci.invoice.number = 10"), ch3) + err = s.Subscribe(ctx, "client-3", query.MustParse("tm.events.type=NewRoundStep AND abci.account.name=Igor AND abci.invoice.number = 10"), ch3) + require.NoError(t, err) + err = s.PublishWithTags(ctx, "Valeria Richards", map[string]interface{}{"tm.events.type": "NewRoundStep"}) require.NoError(t, err) - s.PublishWithTags("Valeria Richards", map[string]interface{}{"tm.events.type": "NewRoundStep"}) assert.Zero(t, len(ch3)) } @@ -65,22 +74,25 @@ func TestClientResubscribes(t *testing.T) { s.Start() defer s.Stop() + ctx := context.Background() q := query.MustParse("tm.events.type=NewBlock") ch1 := make(chan interface{}, 1) - err := s.Subscribe(clientID, q, ch1) + err := s.Subscribe(ctx, clientID, q, ch1) + require.NoError(t, err) + err = s.PublishWithTags(ctx, "Goblin Queen", map[string]interface{}{"tm.events.type": "NewBlock"}) require.NoError(t, err) - s.PublishWithTags("Goblin Queen", map[string]interface{}{"tm.events.type": "NewBlock"}) assertReceive(t, "Goblin Queen", ch1) ch2 := make(chan interface{}, 1) - err = s.Subscribe(clientID, q, ch2) + err = s.Subscribe(ctx, clientID, q, ch2) require.NoError(t, err) _, ok := <-ch1 assert.False(t, ok) - s.PublishWithTags("Spider-Man", map[string]interface{}{"tm.events.type": "NewBlock"}) + err = s.PublishWithTags(ctx, "Spider-Man", map[string]interface{}{"tm.events.type": "NewBlock"}) + require.NoError(t, err) assertReceive(t, "Spider-Man", ch2) } @@ -90,12 +102,15 @@ func TestUnsubscribe(t *testing.T) { s.Start() defer s.Stop() + ctx := context.Background() ch := make(chan interface{}) - err := s.Subscribe(clientID, query.Empty{}, ch) + err := s.Subscribe(ctx, clientID, query.Empty{}, ch) + require.NoError(t, err) + err = s.Unsubscribe(ctx, clientID, query.Empty{}) require.NoError(t, err) - s.Unsubscribe(clientID, query.Empty{}) - s.Publish("Nick Fury") + err = s.Publish(ctx, "Nick Fury") + require.NoError(t, err) assert.Zero(t, len(ch), "Should not receive anything after Unsubscribe") _, ok := <-ch @@ -108,15 +123,18 @@ func TestUnsubscribeAll(t *testing.T) { s.Start() defer s.Stop() + ctx := context.Background() ch1, ch2 := make(chan interface{}, 1), make(chan interface{}, 1) - err := s.Subscribe(clientID, query.MustParse("tm.events.type=NewBlock"), ch1) + err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type=NewBlock"), ch1) require.NoError(t, err) - err = s.Subscribe(clientID, query.MustParse("tm.events.type=NewBlockHeader"), ch2) + err = s.Subscribe(ctx, clientID, query.MustParse("tm.events.type=NewBlockHeader"), ch2) require.NoError(t, err) - s.UnsubscribeAll(clientID) + err = s.UnsubscribeAll(ctx, clientID) + require.NoError(t, err) - s.Publish("Nick Fury") + err = s.Publish(ctx, "Nick Fury") + require.NoError(t, err) assert.Zero(t, len(ch1), "Should not receive anything after UnsubscribeAll") assert.Zero(t, len(ch2), "Should not receive anything after UnsubscribeAll") @@ -130,19 +148,11 @@ func TestBufferCapacity(t *testing.T) { s := pubsub.NewServer(pubsub.BufferCapacity(2)) s.SetLogger(log.TestingLogger()) - s.Publish("Nighthawk") - s.Publish("Sage") -} - -func TestSubscribeReturnsErrorIfServerOverflowed(t *testing.T) { - s := pubsub.NewServer() - s.SetLogger(log.TestingLogger()) - - ch := make(chan interface{}, 1) - err := s.Subscribe(clientID, query.MustParse("tm.events.type=NewBlock"), ch) - if assert.Error(t, err) { - assert.Equal(t, pubsub.ErrorOverflow, err) - } + ctx := context.Background() + err := s.Publish(ctx, "Nighthawk") + require.NoError(t, err) + err = s.Publish(ctx, "Sage") + require.NoError(t, err) } func Benchmark10Clients(b *testing.B) { benchmarkNClients(10, b) } @@ -158,19 +168,20 @@ func benchmarkNClients(n int, b *testing.B) { s.Start() defer s.Stop() + ctx := context.Background() for i := 0; i < n; i++ { ch := make(chan interface{}) go func() { for range ch { } }() - s.Subscribe(clientID, query.MustParse(fmt.Sprintf("abci.Account.Owner = Ivan AND abci.Invoices.Number = %d", i)), ch) + s.Subscribe(ctx, clientID, query.MustParse(fmt.Sprintf("abci.Account.Owner = Ivan AND abci.Invoices.Number = %d", i)), ch) } b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - s.PublishWithTags("Gamora", map[string]interface{}{"abci.Account.Owner": "Ivan", "abci.Invoices.Number": i}) + s.PublishWithTags(ctx, "Gamora", map[string]interface{}{"abci.Account.Owner": "Ivan", "abci.Invoices.Number": i}) } } @@ -179,6 +190,7 @@ func benchmarkNClientsOneQuery(n int, b *testing.B) { s.Start() defer s.Stop() + ctx := context.Background() q := query.MustParse("abci.Account.Owner = Ivan AND abci.Invoices.Number = 1") for i := 0; i < n; i++ { ch := make(chan interface{}) @@ -186,13 +198,13 @@ func benchmarkNClientsOneQuery(n int, b *testing.B) { for range ch { } }() - s.Subscribe(clientID, q, ch) + s.Subscribe(ctx, clientID, q, ch) } b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - s.PublishWithTags("Gamora", map[string]interface{}{"abci.Account.Owner": "Ivan", "abci.Invoices.Number": 1}) + s.PublishWithTags(ctx, "Gamora", map[string]interface{}{"abci.Account.Owner": "Ivan", "abci.Invoices.Number": 1}) } } From 17d6091ef42305e0a32a119e815f961924b64de2 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Sat, 15 Jul 2017 13:33:47 +0300 Subject: [PATCH 17/37] updates as per Bucky's comments --- pubsub/pubsub.go | 38 ++++++++++++++++++-------------------- pubsub/pubsub_test.go | 7 +++++++ 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go index 34df86a4..f5df418a 100644 --- a/pubsub/pubsub.go +++ b/pubsub/pubsub.go @@ -9,8 +9,6 @@ // When some message is published, we match it with all queries. If there is a // match, this message will be pushed to all clients, subscribed to that query. // See query subpackage for our implementation. -// -// Subscribe/Unsubscribe calls are always blocking. package pubsub import ( @@ -42,7 +40,7 @@ type Query interface { Matches(tags map[string]interface{}) bool } -// Server allows clients to subscribe/unsubscribe for messages, pubsling +// Server allows clients to subscribe/unsubscribe for messages, publishing // messages with or without tags, and manages internal state. type Server struct { cmn.BaseService @@ -83,15 +81,15 @@ func BufferCapacity(cap int) Option { } } -// Returns capacity of the internal server's queue. +// BufferCapacity returns capacity of the internal server's queue. func (s Server) BufferCapacity() int { return s.cmdsCap } -// Subscribe returns a channel on which messages matching the given query can -// be received. If the subscription already exists old channel will be closed -// and new one returned. Error will be returned to the caller if the context is -// cancelled. +// Subscribe creates a subscription for the given client. It accepts a channel +// on which messages matching the given query can be received. If the +// subscription already exists, the old channel will be closed. An error will +// be returned to the caller if the context is canceled. func (s *Server) Subscribe(ctx context.Context, clientID string, query Query, out chan<- interface{}) error { select { case s.cmds <- cmd{op: sub, clientID: clientID, query: query, ch: out}: @@ -101,8 +99,8 @@ func (s *Server) Subscribe(ctx context.Context, clientID string, query Query, ou } } -// Unsubscribe unsubscribes the given client from the query. Error will be -// returned to the caller if the context is cancelled. +// Unsubscribe removes the subscription on the given query. An error will be +// returned to the caller if the context is canceled. func (s *Server) Unsubscribe(ctx context.Context, clientID string, query Query) error { select { case s.cmds <- cmd{op: unsub, clientID: clientID, query: query}: @@ -112,7 +110,8 @@ func (s *Server) Unsubscribe(ctx context.Context, clientID string, query Query) } } -// Unsubscribe unsubscribes the given channel. Blocking. +// Unsubscribe removes all client subscriptions. An error will be returned to +// the caller if the context is canceled. func (s *Server) UnsubscribeAll(ctx context.Context, clientID string) error { select { case s.cmds <- cmd{op: unsub, clientID: clientID}: @@ -122,14 +121,15 @@ func (s *Server) UnsubscribeAll(ctx context.Context, clientID string) error { } } -// Publish publishes the given message. Blocking. +// Publish publishes the given message. An error will be returned to the caller +// if the context is canceled. func (s *Server) Publish(ctx context.Context, msg interface{}) error { return s.PublishWithTags(ctx, msg, make(map[string]interface{})) } -// PublishWithTags publishes the given message with a set of tags. This set of -// tags will be matched with client queries. If there is a match, the message -// will be sent to a client. Blocking. +// PublishWithTags publishes the given message with the set of tags. The set is +// matched with clients queries. If there is a match, the message is sent to +// the client. func (s *Server) PublishWithTags(ctx context.Context, msg interface{}, tags map[string]interface{}) error { select { case s.cmds <- cmd{op: pub, msg: msg, tags: tags}: @@ -152,7 +152,7 @@ type state struct { clients map[string]map[Query]struct{} } -// OnStart implements Service.OnStart by creating a main loop. +// OnStart implements Service.OnStart by starting the server. func (s *Server) OnStart() error { go s.loop(state{ queries: make(map[Query]map[string]chan<- interface{}), @@ -194,6 +194,8 @@ func (state *state) add(clientID string, q Query, ch chan<- interface{}) { close(oldCh) } } + + // create subscription state.queries[q][clientID] = ch // add client if needed @@ -201,10 +203,6 @@ func (state *state) add(clientID string, q Query, ch chan<- interface{}) { state.clients[clientID] = make(map[Query]struct{}) } state.clients[clientID][q] = struct{}{} - - // create subscription - clientToChannelMap := state.queries[q] - clientToChannelMap[clientID] = ch } func (state *state) remove(clientID string, q Query) { diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index 9c984144..9d003cff 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -153,6 +153,13 @@ func TestBufferCapacity(t *testing.T) { require.NoError(t, err) err = s.Publish(ctx, "Sage") require.NoError(t, err) + + ctx, cancel := context.WithTimeout(ctx, 10*time.Millisecond) + defer cancel() + err = s.Publish(ctx, "Ironclad") + if assert.Error(t, err) { + assert.Equal(t, context.DeadlineExceeded, err) + } } func Benchmark10Clients(b *testing.B) { benchmarkNClients(10, b) } From 992c54253f14b1d76766403bb2ca25ede71d5022 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Tue, 18 Jul 2017 11:47:30 +0300 Subject: [PATCH 18/37] fixes from gometalinter review --- pubsub/pubsub.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pubsub/pubsub.go b/pubsub/pubsub.go index f5df418a..52b8361f 100644 --- a/pubsub/pubsub.go +++ b/pubsub/pubsub.go @@ -110,8 +110,8 @@ func (s *Server) Unsubscribe(ctx context.Context, clientID string, query Query) } } -// Unsubscribe removes all client subscriptions. An error will be returned to -// the caller if the context is canceled. +// UnsubscribeAll removes all client subscriptions. An error will be returned +// to the caller if the context is canceled. func (s *Server) UnsubscribeAll(ctx context.Context, clientID string) error { select { case s.cmds <- cmd{op: unsub, clientID: clientID}: @@ -172,7 +172,7 @@ loop: state.removeAll(cmd.clientID) } case shutdown: - for clientID, _ := range state.clients { + for clientID := range state.clients { state.removeAll(clientID) } break loop @@ -232,7 +232,7 @@ func (state *state) removeAll(clientID string) { return } - for q, _ := range queryMap { + for q := range queryMap { ch := state.queries[q][clientID] close(ch) From 3c6c1b7d334f54b70eb44c39a69e95a4b10ad1dc Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 19 Jul 2017 15:02:04 -0400 Subject: [PATCH 19/37] common: ProtocolAndAddress --- common/net.go | 18 +++++++++++++++--- common/net_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 common/net_test.go diff --git a/common/net.go b/common/net.go index 2f9c9c8c..bdbe38f7 100644 --- a/common/net.go +++ b/common/net.go @@ -5,10 +5,22 @@ import ( "strings" ) -// protoAddr: e.g. "tcp://127.0.0.1:8080" or "unix:///tmp/test.sock" +// Connect dials the given address and returns a net.Conn. The protoAddr argument should be prefixed with the protocol, +// eg. "tcp://127.0.0.1:8080" or "unix:///tmp/test.sock" func Connect(protoAddr string) (net.Conn, error) { - parts := strings.SplitN(protoAddr, "://", 2) - proto, address := parts[0], parts[1] + proto, address := ProtocolAndAddress(protoAddr) conn, err := net.Dial(proto, address) return conn, err } + +// ProtocolAndAddress splits an address into the protocol and address components. +// For instance, "tcp://127.0.0.1:8080" will be split into "tcp" and "127.0.0.1:8080". +// If the address has no protocol prefix, the default is "tcp". +func ProtocolAndAddress(listenAddr string) (string, string) { + protocol, address := "tcp", listenAddr + parts := strings.SplitN(address, "://", 2) + if len(parts) == 2 { + protocol, address = parts[0], parts[1] + } + return protocol, address +} diff --git a/common/net_test.go b/common/net_test.go new file mode 100644 index 00000000..38d2ae82 --- /dev/null +++ b/common/net_test.go @@ -0,0 +1,38 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestProtocolAndAddress(t *testing.T) { + + cases := []struct { + fullAddr string + proto string + addr string + }{ + { + "tcp://mydomain:80", + "tcp", + "mydomain:80", + }, + { + "mydomain:80", + "tcp", + "mydomain:80", + }, + { + "unix://mydomain:80", + "unix", + "mydomain:80", + }, + } + + for _, c := range cases { + proto, addr := ProtocolAndAddress(c.fullAddr) + assert.Equal(t, proto, c.proto) + assert.Equal(t, addr, c.addr) + } +} From 77f6febb034e36424791a92848385bd420930a9b Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 20 Jul 2017 11:46:22 +0300 Subject: [PATCH 20/37] rename TestClientResubscribes to TestClientSubscribesTwice test UnsubscribeAll properly test BufferCapacity getter --- pubsub/pubsub_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index 9d003cff..85b4b1e4 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -68,7 +68,7 @@ func TestDifferentClients(t *testing.T) { assert.Zero(t, len(ch3)) } -func TestClientResubscribes(t *testing.T) { +func TestClientSubscribesTwice(t *testing.T) { s := pubsub.NewServer() s.SetLogger(log.TestingLogger()) s.Start() @@ -125,9 +125,9 @@ func TestUnsubscribeAll(t *testing.T) { ctx := context.Background() ch1, ch2 := make(chan interface{}, 1), make(chan interface{}, 1) - err := s.Subscribe(ctx, clientID, query.MustParse("tm.events.type=NewBlock"), ch1) + err := s.Subscribe(ctx, clientID, query.Empty{}, ch1) require.NoError(t, err) - err = s.Subscribe(ctx, clientID, query.MustParse("tm.events.type=NewBlockHeader"), ch2) + err = s.Subscribe(ctx, clientID, query.Empty{}, ch2) require.NoError(t, err) err = s.UnsubscribeAll(ctx, clientID) @@ -148,6 +148,8 @@ func TestBufferCapacity(t *testing.T) { s := pubsub.NewServer(pubsub.BufferCapacity(2)) s.SetLogger(log.TestingLogger()) + assert.Equal(t, 2, s.BufferCapacity()) + ctx := context.Background() err := s.Publish(ctx, "Nighthawk") require.NoError(t, err) From a6a06f820fa45aba2bac5f6648d0c8172d1a353a Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 21 Jul 2017 13:09:58 +0300 Subject: [PATCH 21/37] [pubsub/query] quote values using single quotes This fixes the problem with base-16 encoded values which may start with digits: 015AB.... In such cases, the parser recognizes them as numbers but fails to parse because of the follow-up characters (AB). ``` failed to parse tm.events.type=Tx AND hash=136E18F7E4C348B780CF873A0BF43922E5BAFA63: parse error near digit (line 1 symbol 31 - line 1 symbol 32): "6" ``` So, from now on we should quote any values. This seems to be the way Postgresql has chosen. --- pubsub/example_test.go | 2 +- pubsub/pubsub_test.go | 12 +- pubsub/query/parser_test.go | 33 +- pubsub/query/query.go | 5 +- pubsub/query/query.peg | 4 +- pubsub/query/query.peg.go | 832 ++++++++++++++++-------------------- pubsub/query/query_test.go | 8 +- 7 files changed, 395 insertions(+), 501 deletions(-) diff --git a/pubsub/example_test.go b/pubsub/example_test.go index 6597c858..3eda7d32 100644 --- a/pubsub/example_test.go +++ b/pubsub/example_test.go @@ -19,7 +19,7 @@ func TestExample(t *testing.T) { ctx := context.Background() ch := make(chan interface{}, 1) - err := s.Subscribe(ctx, "example-client", query.MustParse("abci.account.name=John"), ch) + err := s.Subscribe(ctx, "example-client", query.MustParse("abci.account.name='John'"), ch) require.NoError(t, err) err = s.PublishWithTags(ctx, "Tombstone", map[string]interface{}{"abci.account.name": "John"}) require.NoError(t, err) diff --git a/pubsub/pubsub_test.go b/pubsub/pubsub_test.go index 85b4b1e4..7bf7b41f 100644 --- a/pubsub/pubsub_test.go +++ b/pubsub/pubsub_test.go @@ -46,14 +46,14 @@ func TestDifferentClients(t *testing.T) { ctx := context.Background() ch1 := make(chan interface{}, 1) - err := s.Subscribe(ctx, "client-1", query.MustParse("tm.events.type=NewBlock"), ch1) + err := s.Subscribe(ctx, "client-1", query.MustParse("tm.events.type='NewBlock'"), ch1) require.NoError(t, err) err = s.PublishWithTags(ctx, "Iceman", map[string]interface{}{"tm.events.type": "NewBlock"}) require.NoError(t, err) assertReceive(t, "Iceman", ch1) ch2 := make(chan interface{}, 1) - err = s.Subscribe(ctx, "client-2", query.MustParse("tm.events.type=NewBlock AND abci.account.name=Igor"), ch2) + err = s.Subscribe(ctx, "client-2", query.MustParse("tm.events.type='NewBlock' AND abci.account.name='Igor'"), ch2) require.NoError(t, err) err = s.PublishWithTags(ctx, "Ultimo", map[string]interface{}{"tm.events.type": "NewBlock", "abci.account.name": "Igor"}) require.NoError(t, err) @@ -61,7 +61,7 @@ func TestDifferentClients(t *testing.T) { assertReceive(t, "Ultimo", ch2) ch3 := make(chan interface{}, 1) - err = s.Subscribe(ctx, "client-3", query.MustParse("tm.events.type=NewRoundStep AND abci.account.name=Igor AND abci.invoice.number = 10"), ch3) + err = s.Subscribe(ctx, "client-3", query.MustParse("tm.events.type='NewRoundStep' AND abci.account.name='Igor' AND abci.invoice.number = 10"), ch3) require.NoError(t, err) err = s.PublishWithTags(ctx, "Valeria Richards", map[string]interface{}{"tm.events.type": "NewRoundStep"}) require.NoError(t, err) @@ -75,7 +75,7 @@ func TestClientSubscribesTwice(t *testing.T) { defer s.Stop() ctx := context.Background() - q := query.MustParse("tm.events.type=NewBlock") + q := query.MustParse("tm.events.type='NewBlock'") ch1 := make(chan interface{}, 1) err := s.Subscribe(ctx, clientID, q, ch1) @@ -184,7 +184,7 @@ func benchmarkNClients(n int, b *testing.B) { for range ch { } }() - s.Subscribe(ctx, clientID, query.MustParse(fmt.Sprintf("abci.Account.Owner = Ivan AND abci.Invoices.Number = %d", i)), ch) + s.Subscribe(ctx, clientID, query.MustParse(fmt.Sprintf("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = %d", i)), ch) } b.ReportAllocs() @@ -200,7 +200,7 @@ func benchmarkNClientsOneQuery(n int, b *testing.B) { defer s.Stop() ctx := context.Background() - q := query.MustParse("abci.Account.Owner = Ivan AND abci.Invoices.Number = 1") + q := query.MustParse("abci.Account.Owner = 'Ivan' AND abci.Invoices.Number = 1") for i := 0; i < n; i++ { ch := make(chan interface{}) go func() { diff --git a/pubsub/query/parser_test.go b/pubsub/query/parser_test.go index 19496666..d96ac082 100644 --- a/pubsub/query/parser_test.go +++ b/pubsub/query/parser_test.go @@ -13,30 +13,37 @@ func TestParser(t *testing.T) { query string valid bool }{ - {"tm.events.type=NewBlock", true}, - {"tm.events.type = NewBlock", true}, - {"tm.events.type=TIME", true}, - {"tm.events.type=DATE", true}, + {"tm.events.type='NewBlock'", true}, + {"tm.events.type = 'NewBlock'", true}, + {"tm.events.name = ''", true}, + {"tm.events.type='TIME'", true}, + {"tm.events.type='DATE'", true}, + {"tm.events.type='='", true}, + {"tm.events.type='TIME", false}, + {"tm.events.type=TIME'", false}, {"tm.events.type==", false}, + {"tm.events.type=NewBlock", false}, {">==", false}, - {"tm.events.type NewBlock =", false}, - {"tm.events.type>NewBlock", false}, + {"tm.events.type 'NewBlock' =", false}, + {"tm.events.type>'NewBlock'", false}, {"", false}, {"=", false}, - {"=NewBlock", false}, + {"='NewBlock'", false}, {"tm.events.type=", false}, {"tm.events.typeNewBlock", false}, + {"tm.events.type'NewBlock'", false}, + {"'NewBlock'", false}, {"NewBlock", false}, {"", false}, - {"tm.events.type=NewBlock AND abci.account.name=Igor", true}, - {"tm.events.type=NewBlock AND", false}, - {"tm.events.type=NewBlock AN", false}, - {"tm.events.type=NewBlock AN tm.events.type=NewBlockHeader", false}, - {"AND tm.events.type=NewBlock ", false}, + {"tm.events.type='NewBlock' AND abci.account.name='Igor'", true}, + {"tm.events.type='NewBlock' AND", false}, + {"tm.events.type='NewBlock' AN", false}, + {"tm.events.type='NewBlock' AN tm.events.type='NewBlockHeader'", false}, + {"AND tm.events.type='NewBlock' ", false}, - {"abci.account.name CONTAINS Igor", true}, + {"abci.account.name CONTAINS 'Igor'", true}, {"tx.date > DATE 2013-05-03", true}, {"tx.date < DATE 2013-05-03", true}, diff --git a/pubsub/query/query.go b/pubsub/query/query.go index f084a3f9..fdfb87d7 100644 --- a/pubsub/query/query.go +++ b/pubsub/query/query.go @@ -94,9 +94,12 @@ func (q *Query) Matches(tags map[string]interface{}) bool { case rulecontains: op = opContains case rulevalue: + // strip single quotes from value (i.e. "'NewBlock'" -> "NewBlock") + valueWithoutSingleQuotes := buffer[begin+1 : end-1] + // see if the triplet (tag, operator, operand) matches any tag // "tx.gas", "=", "7", { "tx.gas": 7, "tx.ID": "4AE393495334" } - if !match(tag, op, reflect.ValueOf(buffer[begin:end]), tags) { + if !match(tag, op, reflect.ValueOf(valueWithoutSingleQuotes), tags) { return false } case rulenumber: diff --git a/pubsub/query/query.peg b/pubsub/query/query.peg index 9654289c..739892e4 100644 --- a/pubsub/query/query.peg +++ b/pubsub/query/query.peg @@ -13,8 +13,8 @@ condition <- tag ' '* (le ' '* (number / time / date) / contains ' '* value ) -tag <- < (![ \t\n\r\\()"=><] .)+ > -value <- < (![ \t\n\r\\()"=><] .)+ > +tag <- < (![ \t\n\r\\()"'=><] .)+ > +value <- < '\'' (!["'] .)* '\''> number <- < ('0' / [1-9] digit* ('.' digit*)?) > digit <- [0-9] diff --git a/pubsub/query/query.peg.go b/pubsub/query/query.peg.go index 5cd0a9e3..37ce75cd 100644 --- a/pubsub/query/query.peg.go +++ b/pubsub/query/query.peg.go @@ -442,7 +442,7 @@ func (p *QueryParser) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 condition <- <(tag ' '* ((le ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number))) / (ge ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number))) / ((&('=') (equal ' '* (number / time / date / value))) | (&('>') (g ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('<') (l ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('C' | 'c') (contains ' '* value)))))> */ + /* 1 condition <- <(tag ' '* ((le ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number))) / (ge ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number))) / ((&('=') (equal ' '* ((&('\'') value) | (&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('>') (g ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('<') (l ' '* ((&('D' | 'd') date) | (&('T' | 't') time) | (&('0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9') number)))) | (&('C' | 'c') (contains ' '* value)))))> */ func() bool { position16, tokenIndex16 := position, tokenIndex { @@ -473,6 +473,12 @@ func (p *QueryParser) Init() { } position++ break + case '\'': + if buffer[position] != rune('\'') { + goto l22 + } + position++ + break case '"': if buffer[position] != rune('"') { goto l22 @@ -556,6 +562,12 @@ func (p *QueryParser) Init() { } position++ break + case '\'': + if buffer[position] != rune('\'') { + goto l24 + } + position++ + break case '"': if buffer[position] != rune('"') { goto l24 @@ -750,50 +762,50 @@ func (p *QueryParser) Init() { position, tokenIndex = position42, tokenIndex42 } { - position43, tokenIndex43 := position, tokenIndex - if !_rules[rulenumber]() { - goto l44 - } - goto l43 - l44: - position, tokenIndex = position43, tokenIndex43 - if !_rules[ruletime]() { - goto l45 - } - goto l43 - l45: - position, tokenIndex = position43, tokenIndex43 - if !_rules[ruledate]() { - goto l46 - } - goto l43 - l46: - position, tokenIndex = position43, tokenIndex43 - if !_rules[rulevalue]() { - goto l16 + switch buffer[position] { + case '\'': + if !_rules[rulevalue]() { + goto l16 + } + break + case 'D', 'd': + if !_rules[ruledate]() { + goto l16 + } + break + case 'T', 't': + if !_rules[ruletime]() { + goto l16 + } + break + default: + if !_rules[rulenumber]() { + goto l16 + } + break } } - l43: + break case '>': { - position47 := position + position44 := position if buffer[position] != rune('>') { goto l16 } position++ - add(ruleg, position47) + add(ruleg, position44) } - l48: + l45: { - position49, tokenIndex49 := position, tokenIndex + position46, tokenIndex46 := position, tokenIndex if buffer[position] != rune(' ') { - goto l49 + goto l46 } position++ - goto l48 - l49: - position, tokenIndex = position49, tokenIndex49 + goto l45 + l46: + position, tokenIndex = position46, tokenIndex46 } { switch buffer[position] { @@ -818,23 +830,23 @@ func (p *QueryParser) Init() { break case '<': { - position51 := position + position48 := position if buffer[position] != rune('<') { goto l16 } position++ - add(rulel, position51) + add(rulel, position48) } - l52: + l49: { - position53, tokenIndex53 := position, tokenIndex + position50, tokenIndex50 := position, tokenIndex if buffer[position] != rune(' ') { - goto l53 + goto l50 } position++ - goto l52 - l53: - position, tokenIndex = position53, tokenIndex53 + goto l49 + l50: + position, tokenIndex = position50, tokenIndex50 } { switch buffer[position] { @@ -859,139 +871,139 @@ func (p *QueryParser) Init() { break default: { - position55 := position + position52 := position { - position56, tokenIndex56 := position, tokenIndex + position53, tokenIndex53 := position, tokenIndex if buffer[position] != rune('c') { - goto l57 + goto l54 } position++ - goto l56 - l57: - position, tokenIndex = position56, tokenIndex56 + goto l53 + l54: + position, tokenIndex = position53, tokenIndex53 if buffer[position] != rune('C') { goto l16 } position++ } - l56: + l53: { - position58, tokenIndex58 := position, tokenIndex + position55, tokenIndex55 := position, tokenIndex if buffer[position] != rune('o') { - goto l59 + goto l56 } position++ - goto l58 - l59: - position, tokenIndex = position58, tokenIndex58 + goto l55 + l56: + position, tokenIndex = position55, tokenIndex55 if buffer[position] != rune('O') { goto l16 } position++ } - l58: + l55: { - position60, tokenIndex60 := position, tokenIndex + position57, tokenIndex57 := position, tokenIndex if buffer[position] != rune('n') { - goto l61 + goto l58 } position++ - goto l60 - l61: - position, tokenIndex = position60, tokenIndex60 + goto l57 + l58: + position, tokenIndex = position57, tokenIndex57 if buffer[position] != rune('N') { goto l16 } position++ } - l60: + l57: { - position62, tokenIndex62 := position, tokenIndex + position59, tokenIndex59 := position, tokenIndex if buffer[position] != rune('t') { - goto l63 + goto l60 } position++ - goto l62 - l63: - position, tokenIndex = position62, tokenIndex62 + goto l59 + l60: + position, tokenIndex = position59, tokenIndex59 if buffer[position] != rune('T') { goto l16 } position++ } - l62: + l59: { - position64, tokenIndex64 := position, tokenIndex + position61, tokenIndex61 := position, tokenIndex if buffer[position] != rune('a') { - goto l65 + goto l62 } position++ - goto l64 - l65: - position, tokenIndex = position64, tokenIndex64 + goto l61 + l62: + position, tokenIndex = position61, tokenIndex61 if buffer[position] != rune('A') { goto l16 } position++ } - l64: + l61: { - position66, tokenIndex66 := position, tokenIndex + position63, tokenIndex63 := position, tokenIndex if buffer[position] != rune('i') { - goto l67 + goto l64 } position++ - goto l66 - l67: - position, tokenIndex = position66, tokenIndex66 + goto l63 + l64: + position, tokenIndex = position63, tokenIndex63 if buffer[position] != rune('I') { goto l16 } position++ } - l66: + l63: { - position68, tokenIndex68 := position, tokenIndex + position65, tokenIndex65 := position, tokenIndex if buffer[position] != rune('n') { - goto l69 + goto l66 } position++ - goto l68 - l69: - position, tokenIndex = position68, tokenIndex68 + goto l65 + l66: + position, tokenIndex = position65, tokenIndex65 if buffer[position] != rune('N') { goto l16 } position++ } - l68: + l65: { - position70, tokenIndex70 := position, tokenIndex + position67, tokenIndex67 := position, tokenIndex if buffer[position] != rune('s') { - goto l71 + goto l68 } position++ - goto l70 - l71: - position, tokenIndex = position70, tokenIndex70 + goto l67 + l68: + position, tokenIndex = position67, tokenIndex67 if buffer[position] != rune('S') { goto l16 } position++ } - l70: - add(rulecontains, position55) + l67: + add(rulecontains, position52) } - l72: + l69: { - position73, tokenIndex73 := position, tokenIndex + position70, tokenIndex70 := position, tokenIndex if buffer[position] != rune(' ') { - goto l73 + goto l70 } position++ - goto l72 - l73: - position, tokenIndex = position73, tokenIndex73 + goto l69 + l70: + position, tokenIndex = position70, tokenIndex70 } if !_rules[rulevalue]() { goto l16 @@ -1009,550 +1021,458 @@ func (p *QueryParser) Init() { position, tokenIndex = position16, tokenIndex16 return false }, - /* 2 tag <- <<(!((&('<') '<') | (&('>') '>') | (&('=') '=') | (&('"') '"') | (&(')') ')') | (&('(') '(') | (&('\\') '\\') | (&('\r') '\r') | (&('\n') '\n') | (&('\t') '\t') | (&(' ') ' ')) .)+>> */ + /* 2 tag <- <<(!((&('<') '<') | (&('>') '>') | (&('=') '=') | (&('\'') '\'') | (&('"') '"') | (&(')') ')') | (&('(') '(') | (&('\\') '\\') | (&('\r') '\r') | (&('\n') '\n') | (&('\t') '\t') | (&(' ') ' ')) .)+>> */ nil, - /* 3 value <- <<(!((&('<') '<') | (&('>') '>') | (&('=') '=') | (&('"') '"') | (&(')') ')') | (&('(') '(') | (&('\\') '\\') | (&('\r') '\r') | (&('\n') '\n') | (&('\t') '\t') | (&(' ') ' ')) .)+>> */ + /* 3 value <- <<('\'' (!('"' / '\'') .)* '\'')>> */ func() bool { - position75, tokenIndex75 := position, tokenIndex + position72, tokenIndex72 := position, tokenIndex { - position76 := position + position73 := position { - position77 := position - { - position80, tokenIndex80 := position, tokenIndex - { - switch buffer[position] { - case '<': - if buffer[position] != rune('<') { - goto l80 - } - position++ - break - case '>': - if buffer[position] != rune('>') { - goto l80 - } - position++ - break - case '=': - if buffer[position] != rune('=') { - goto l80 - } - position++ - break - case '"': - if buffer[position] != rune('"') { - goto l80 - } - position++ - break - case ')': - if buffer[position] != rune(')') { - goto l80 - } - position++ - break - case '(': - if buffer[position] != rune('(') { - goto l80 - } - position++ - break - case '\\': - if buffer[position] != rune('\\') { - goto l80 - } - position++ - break - case '\r': - if buffer[position] != rune('\r') { - goto l80 - } - position++ - break - case '\n': - if buffer[position] != rune('\n') { - goto l80 - } - position++ - break - case '\t': - if buffer[position] != rune('\t') { - goto l80 - } - position++ - break - default: - if buffer[position] != rune(' ') { - goto l80 - } - position++ - break - } - } - - goto l75 - l80: - position, tokenIndex = position80, tokenIndex80 + position74 := position + if buffer[position] != rune('\'') { + goto l72 } - if !matchDot() { - goto l75 - } - l78: + position++ + l75: { - position79, tokenIndex79 := position, tokenIndex + position76, tokenIndex76 := position, tokenIndex { - position82, tokenIndex82 := position, tokenIndex + position77, tokenIndex77 := position, tokenIndex { - switch buffer[position] { - case '<': - if buffer[position] != rune('<') { - goto l82 - } - position++ - break - case '>': - if buffer[position] != rune('>') { - goto l82 - } - position++ - break - case '=': - if buffer[position] != rune('=') { - goto l82 - } - position++ - break - case '"': - if buffer[position] != rune('"') { - goto l82 - } - position++ - break - case ')': - if buffer[position] != rune(')') { - goto l82 - } - position++ - break - case '(': - if buffer[position] != rune('(') { - goto l82 - } - position++ - break - case '\\': - if buffer[position] != rune('\\') { - goto l82 - } - position++ - break - case '\r': - if buffer[position] != rune('\r') { - goto l82 - } - position++ - break - case '\n': - if buffer[position] != rune('\n') { - goto l82 - } - position++ - break - case '\t': - if buffer[position] != rune('\t') { - goto l82 - } - position++ - break - default: - if buffer[position] != rune(' ') { - goto l82 - } - position++ - break + position78, tokenIndex78 := position, tokenIndex + if buffer[position] != rune('"') { + goto l79 } + position++ + goto l78 + l79: + position, tokenIndex = position78, tokenIndex78 + if buffer[position] != rune('\'') { + goto l77 + } + position++ } - - goto l79 - l82: - position, tokenIndex = position82, tokenIndex82 + l78: + goto l76 + l77: + position, tokenIndex = position77, tokenIndex77 } if !matchDot() { - goto l79 + goto l76 } - goto l78 - l79: - position, tokenIndex = position79, tokenIndex79 + goto l75 + l76: + position, tokenIndex = position76, tokenIndex76 } - add(rulePegText, position77) + if buffer[position] != rune('\'') { + goto l72 + } + position++ + add(rulePegText, position74) } - add(rulevalue, position76) + add(rulevalue, position73) } return true - l75: - position, tokenIndex = position75, tokenIndex75 + l72: + position, tokenIndex = position72, tokenIndex72 return false }, /* 4 number <- <<('0' / ([1-9] digit* ('.' digit*)?))>> */ func() bool { - position84, tokenIndex84 := position, tokenIndex + position80, tokenIndex80 := position, tokenIndex { - position85 := position + position81 := position { - position86 := position + position82 := position { - position87, tokenIndex87 := position, tokenIndex + position83, tokenIndex83 := position, tokenIndex if buffer[position] != rune('0') { - goto l88 - } - position++ - goto l87 - l88: - position, tokenIndex = position87, tokenIndex87 - if c := buffer[position]; c < rune('1') || c > rune('9') { goto l84 } position++ - l89: + goto l83 + l84: + position, tokenIndex = position83, tokenIndex83 + if c := buffer[position]; c < rune('1') || c > rune('9') { + goto l80 + } + position++ + l85: { - position90, tokenIndex90 := position, tokenIndex + position86, tokenIndex86 := position, tokenIndex if !_rules[ruledigit]() { - goto l90 + goto l86 } - goto l89 - l90: - position, tokenIndex = position90, tokenIndex90 + goto l85 + l86: + position, tokenIndex = position86, tokenIndex86 } { - position91, tokenIndex91 := position, tokenIndex + position87, tokenIndex87 := position, tokenIndex if buffer[position] != rune('.') { - goto l91 + goto l87 } position++ - l93: + l89: { - position94, tokenIndex94 := position, tokenIndex + position90, tokenIndex90 := position, tokenIndex if !_rules[ruledigit]() { - goto l94 + goto l90 } - goto l93 - l94: - position, tokenIndex = position94, tokenIndex94 + goto l89 + l90: + position, tokenIndex = position90, tokenIndex90 } - goto l92 - l91: - position, tokenIndex = position91, tokenIndex91 + goto l88 + l87: + position, tokenIndex = position87, tokenIndex87 } - l92: + l88: } - l87: - add(rulePegText, position86) + l83: + add(rulePegText, position82) } - add(rulenumber, position85) + add(rulenumber, position81) } return true - l84: - position, tokenIndex = position84, tokenIndex84 + l80: + position, tokenIndex = position80, tokenIndex80 return false }, /* 5 digit <- <[0-9]> */ func() bool { - position95, tokenIndex95 := position, tokenIndex + position91, tokenIndex91 := position, tokenIndex { - position96 := position + position92 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l95 + goto l91 } position++ - add(ruledigit, position96) + add(ruledigit, position92) } return true - l95: - position, tokenIndex = position95, tokenIndex95 + l91: + position, tokenIndex = position91, tokenIndex91 return false }, /* 6 time <- <(('t' / 'T') ('i' / 'I') ('m' / 'M') ('e' / 'E') ' ' <(year '-' month '-' day 'T' digit digit ':' digit digit ':' digit digit ((('-' / '+') digit digit ':' digit digit) / 'Z'))>)> */ func() bool { - position97, tokenIndex97 := position, tokenIndex + position93, tokenIndex93 := position, tokenIndex { - position98 := position + position94 := position + { + position95, tokenIndex95 := position, tokenIndex + if buffer[position] != rune('t') { + goto l96 + } + position++ + goto l95 + l96: + position, tokenIndex = position95, tokenIndex95 + if buffer[position] != rune('T') { + goto l93 + } + position++ + } + l95: + { + position97, tokenIndex97 := position, tokenIndex + if buffer[position] != rune('i') { + goto l98 + } + position++ + goto l97 + l98: + position, tokenIndex = position97, tokenIndex97 + if buffer[position] != rune('I') { + goto l93 + } + position++ + } + l97: { position99, tokenIndex99 := position, tokenIndex - if buffer[position] != rune('t') { + if buffer[position] != rune('m') { goto l100 } position++ goto l99 l100: position, tokenIndex = position99, tokenIndex99 - if buffer[position] != rune('T') { - goto l97 + if buffer[position] != rune('M') { + goto l93 } position++ } l99: { position101, tokenIndex101 := position, tokenIndex - if buffer[position] != rune('i') { + if buffer[position] != rune('e') { goto l102 } position++ goto l101 l102: position, tokenIndex = position101, tokenIndex101 - if buffer[position] != rune('I') { - goto l97 + if buffer[position] != rune('E') { + goto l93 } position++ } l101: - { - position103, tokenIndex103 := position, tokenIndex - if buffer[position] != rune('m') { - goto l104 - } - position++ - goto l103 - l104: - position, tokenIndex = position103, tokenIndex103 - if buffer[position] != rune('M') { - goto l97 - } - position++ - } - l103: - { - position105, tokenIndex105 := position, tokenIndex - if buffer[position] != rune('e') { - goto l106 - } - position++ - goto l105 - l106: - position, tokenIndex = position105, tokenIndex105 - if buffer[position] != rune('E') { - goto l97 - } - position++ - } - l105: if buffer[position] != rune(' ') { - goto l97 + goto l93 } position++ { - position107 := position + position103 := position if !_rules[ruleyear]() { - goto l97 + goto l93 } if buffer[position] != rune('-') { - goto l97 + goto l93 } position++ if !_rules[rulemonth]() { - goto l97 + goto l93 } if buffer[position] != rune('-') { - goto l97 + goto l93 } position++ if !_rules[ruleday]() { - goto l97 + goto l93 } if buffer[position] != rune('T') { - goto l97 + goto l93 } position++ if !_rules[ruledigit]() { - goto l97 + goto l93 } if !_rules[ruledigit]() { - goto l97 + goto l93 } if buffer[position] != rune(':') { - goto l97 + goto l93 } position++ if !_rules[ruledigit]() { - goto l97 + goto l93 } if !_rules[ruledigit]() { - goto l97 + goto l93 } if buffer[position] != rune(':') { - goto l97 + goto l93 } position++ if !_rules[ruledigit]() { - goto l97 + goto l93 } if !_rules[ruledigit]() { - goto l97 + goto l93 } { - position108, tokenIndex108 := position, tokenIndex + position104, tokenIndex104 := position, tokenIndex { - position110, tokenIndex110 := position, tokenIndex + position106, tokenIndex106 := position, tokenIndex if buffer[position] != rune('-') { - goto l111 + goto l107 } position++ - goto l110 - l111: - position, tokenIndex = position110, tokenIndex110 + goto l106 + l107: + position, tokenIndex = position106, tokenIndex106 if buffer[position] != rune('+') { - goto l109 + goto l105 } position++ } - l110: + l106: if !_rules[ruledigit]() { - goto l109 + goto l105 } if !_rules[ruledigit]() { - goto l109 + goto l105 } if buffer[position] != rune(':') { - goto l109 + goto l105 } position++ if !_rules[ruledigit]() { - goto l109 + goto l105 } if !_rules[ruledigit]() { - goto l109 + goto l105 } - goto l108 - l109: - position, tokenIndex = position108, tokenIndex108 + goto l104 + l105: + position, tokenIndex = position104, tokenIndex104 if buffer[position] != rune('Z') { - goto l97 + goto l93 } position++ } - l108: - add(rulePegText, position107) + l104: + add(rulePegText, position103) } - add(ruletime, position98) + add(ruletime, position94) } return true - l97: - position, tokenIndex = position97, tokenIndex97 + l93: + position, tokenIndex = position93, tokenIndex93 return false }, /* 7 date <- <(('d' / 'D') ('a' / 'A') ('t' / 'T') ('e' / 'E') ' ' <(year '-' month '-' day)>)> */ func() bool { - position112, tokenIndex112 := position, tokenIndex + position108, tokenIndex108 := position, tokenIndex { - position113 := position + position109 := position + { + position110, tokenIndex110 := position, tokenIndex + if buffer[position] != rune('d') { + goto l111 + } + position++ + goto l110 + l111: + position, tokenIndex = position110, tokenIndex110 + if buffer[position] != rune('D') { + goto l108 + } + position++ + } + l110: + { + position112, tokenIndex112 := position, tokenIndex + if buffer[position] != rune('a') { + goto l113 + } + position++ + goto l112 + l113: + position, tokenIndex = position112, tokenIndex112 + if buffer[position] != rune('A') { + goto l108 + } + position++ + } + l112: { position114, tokenIndex114 := position, tokenIndex - if buffer[position] != rune('d') { + if buffer[position] != rune('t') { goto l115 } position++ goto l114 l115: position, tokenIndex = position114, tokenIndex114 - if buffer[position] != rune('D') { - goto l112 + if buffer[position] != rune('T') { + goto l108 } position++ } l114: { position116, tokenIndex116 := position, tokenIndex - if buffer[position] != rune('a') { + if buffer[position] != rune('e') { goto l117 } position++ goto l116 l117: position, tokenIndex = position116, tokenIndex116 - if buffer[position] != rune('A') { - goto l112 + if buffer[position] != rune('E') { + goto l108 } position++ } l116: - { - position118, tokenIndex118 := position, tokenIndex - if buffer[position] != rune('t') { - goto l119 - } - position++ - goto l118 - l119: - position, tokenIndex = position118, tokenIndex118 - if buffer[position] != rune('T') { - goto l112 - } - position++ - } - l118: - { - position120, tokenIndex120 := position, tokenIndex - if buffer[position] != rune('e') { - goto l121 - } - position++ - goto l120 - l121: - position, tokenIndex = position120, tokenIndex120 - if buffer[position] != rune('E') { - goto l112 - } - position++ - } - l120: if buffer[position] != rune(' ') { - goto l112 + goto l108 } position++ { - position122 := position + position118 := position if !_rules[ruleyear]() { - goto l112 + goto l108 } if buffer[position] != rune('-') { - goto l112 + goto l108 } position++ if !_rules[rulemonth]() { - goto l112 + goto l108 } if buffer[position] != rune('-') { - goto l112 + goto l108 } position++ if !_rules[ruleday]() { - goto l112 + goto l108 } - add(rulePegText, position122) + add(rulePegText, position118) } - add(ruledate, position113) + add(ruledate, position109) } return true - l112: - position, tokenIndex = position112, tokenIndex112 + l108: + position, tokenIndex = position108, tokenIndex108 return false }, /* 8 year <- <(('1' / '2') digit digit digit)> */ + func() bool { + position119, tokenIndex119 := position, tokenIndex + { + position120 := position + { + position121, tokenIndex121 := position, tokenIndex + if buffer[position] != rune('1') { + goto l122 + } + position++ + goto l121 + l122: + position, tokenIndex = position121, tokenIndex121 + if buffer[position] != rune('2') { + goto l119 + } + position++ + } + l121: + if !_rules[ruledigit]() { + goto l119 + } + if !_rules[ruledigit]() { + goto l119 + } + if !_rules[ruledigit]() { + goto l119 + } + add(ruleyear, position120) + } + return true + l119: + position, tokenIndex = position119, tokenIndex119 + return false + }, + /* 9 month <- <(('0' / '1') digit)> */ func() bool { position123, tokenIndex123 := position, tokenIndex { position124 := position { position125, tokenIndex125 := position, tokenIndex - if buffer[position] != rune('1') { + if buffer[position] != rune('0') { goto l126 } position++ goto l125 l126: position, tokenIndex = position125, tokenIndex125 - if buffer[position] != rune('2') { + if buffer[position] != rune('1') { goto l123 } position++ @@ -1561,77 +1481,41 @@ func (p *QueryParser) Init() { if !_rules[ruledigit]() { goto l123 } - if !_rules[ruledigit]() { - goto l123 - } - if !_rules[ruledigit]() { - goto l123 - } - add(ruleyear, position124) + add(rulemonth, position124) } return true l123: position, tokenIndex = position123, tokenIndex123 return false }, - /* 9 month <- <(('0' / '1') digit)> */ + /* 10 day <- <(((&('3') '3') | (&('2') '2') | (&('1') '1') | (&('0') '0')) digit)> */ func() bool { position127, tokenIndex127 := position, tokenIndex { position128 := position - { - position129, tokenIndex129 := position, tokenIndex - if buffer[position] != rune('0') { - goto l130 - } - position++ - goto l129 - l130: - position, tokenIndex = position129, tokenIndex129 - if buffer[position] != rune('1') { - goto l127 - } - position++ - } - l129: - if !_rules[ruledigit]() { - goto l127 - } - add(rulemonth, position128) - } - return true - l127: - position, tokenIndex = position127, tokenIndex127 - return false - }, - /* 10 day <- <(((&('3') '3') | (&('2') '2') | (&('1') '1') | (&('0') '0')) digit)> */ - func() bool { - position131, tokenIndex131 := position, tokenIndex - { - position132 := position { switch buffer[position] { case '3': if buffer[position] != rune('3') { - goto l131 + goto l127 } position++ break case '2': if buffer[position] != rune('2') { - goto l131 + goto l127 } position++ break case '1': if buffer[position] != rune('1') { - goto l131 + goto l127 } position++ break default: if buffer[position] != rune('0') { - goto l131 + goto l127 } position++ break @@ -1639,13 +1523,13 @@ func (p *QueryParser) Init() { } if !_rules[ruledigit]() { - goto l131 + goto l127 } - add(ruleday, position132) + add(ruleday, position128) } return true - l131: - position, tokenIndex = position131, tokenIndex131 + l127: + position, tokenIndex = position127, tokenIndex127 return false }, /* 11 and <- <(('a' / 'A') ('n' / 'N') ('d' / 'D'))> */ diff --git a/pubsub/query/query_test.go b/pubsub/query/query_test.go index 75d02ee4..431ae1fe 100644 --- a/pubsub/query/query_test.go +++ b/pubsub/query/query_test.go @@ -22,7 +22,7 @@ func TestMatches(t *testing.T) { err bool matches bool }{ - {"tm.events.type=NewBlock", map[string]interface{}{"tm.events.type": "NewBlock"}, false, true}, + {"tm.events.type='NewBlock'", map[string]interface{}{"tm.events.type": "NewBlock"}, false, true}, {"tx.gas > 7", map[string]interface{}{"tx.gas": 8}, false, true}, {"tx.gas > 7 AND tx.gas < 9", map[string]interface{}{"tx.gas": 8}, false, true}, @@ -40,8 +40,8 @@ func TestMatches(t *testing.T) { {"tx.time >= TIME 2013-05-03T14:45:00Z", map[string]interface{}{"tx.time": time.Now()}, false, true}, {"tx.time = TIME 2013-05-03T14:45:00Z", map[string]interface{}{"tx.time": txTime}, false, false}, - {"abci.owner.name CONTAINS Igor", map[string]interface{}{"abci.owner.name": "Igor,Ivan"}, false, true}, - {"abci.owner.name CONTAINS Igor", map[string]interface{}{"abci.owner.name": "Pavel,Ivan"}, false, false}, + {"abci.owner.name CONTAINS 'Igor'", map[string]interface{}{"abci.owner.name": "Igor,Ivan"}, false, true}, + {"abci.owner.name CONTAINS 'Igor'", map[string]interface{}{"abci.owner.name": "Pavel,Ivan"}, false, false}, } for _, tc := range testCases { @@ -60,5 +60,5 @@ func TestMatches(t *testing.T) { func TestMustParse(t *testing.T) { assert.Panics(t, func() { query.MustParse("=") }) - assert.NotPanics(t, func() { query.MustParse("tm.events.type=NewBlock") }) + assert.NotPanics(t, func() { query.MustParse("tm.events.type='NewBlock'") }) } From 8a51210efca6ecd881d3336c8d7b4f6ef0ff30e2 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 28 Jul 2017 11:22:48 -0400 Subject: [PATCH 22/37] [common] use temp intead of {filePath}.new The problem with {filePath}.new is that it is not safe for concurrent use! Calling this function with the same params results in the following error: ``` panic: Panicked on a Crisis: rename /root/.tendermint_test/consensus_replay_test/priv_validator.json.new /root/.tendermint_test/consensus_replay_test/priv_validator.json: no such file or directory goroutine 47860 [running]: github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common.PanicCrisis(0xcba800, 0xc42152d640) /go/src/github.com/tendermint/tendermint/vendor/github.com/tendermint/tmlibs/common/errors.go:33 +0x10f github.com/tendermint/tendermint/types.(*PrivValidator).save(0xc42235f2c0) /go/src/github.com/tendermint/tendermint/types/priv_validator.go:165 +0x159 github.com/tendermint/tendermint/types.(*PrivValidator).signBytesHRS(0xc42235f2c0, 0x6, 0x0, 0xc424e88f03, 0xc429908580, 0xca, 0x155, 0x80, 0xc424e88f00, 0x7f4ecafc88d0, ...) /go/src/github.com/tendermint/tendermint/types/priv_validator.go:249 +0x2bb github.com/tendermint/tendermint/types.(*PrivValidator).SignVote(0xc42235f2c0, 0xc4228c7460, 0xf, 0xc424e88f00, 0x0, 0x0) /go/src/github.com/tendermint/tendermint/types/priv_validator.go:186 +0x1a2 github.com/tendermint/tendermint/consensus.(*ConsensusState).signVote(0xc424efd520, 0xc400000002, 0xc422d5e3c0, 0x14, 0x20, 0x1, 0xc4247b6560, 0x14, 0x20, 0x0, ...) github.com/tendermint/tendermint/consensus/_test/_obj_test/state.go:1556 +0x35e github.com/tendermint/tendermint/consensus.(*ConsensusState).signAddVote(0xc424efd520, 0x2, 0xc422d5e3c0, 0x14, 0x20, 0x1, 0xc4247b6560, 0x14, 0x20, 0xc42001b300) github.com/tendermint/tendermint/consensus/_test/_obj_test/state.go:1568 +0x200 github.com/tendermint/tendermint/consensus.(*ConsensusState).enterPrecommit(0xc424efd520, 0x6, 0x0) github.com/tendermint/tendermint/consensus/_test/_obj_test/state.go:1082 +0x13a4 github.com/tendermint/tendermint/consensus.(*ConsensusState).addVote(0xc424efd520, 0xc424e88780, 0x0, 0x0, 0x39, 0x1dc, 0x28c) github.com/tendermint/tendermint/consensus/_test/_obj_test/state.go:1477 +0x1be5 github.com/tendermint/tendermint/consensus.(*ConsensusState).tryAddVote(0xc424efd520, 0xc424e88780, 0x0, 0x0, 0xd7fb00, 0xc42152ce00) github.com/tendermint/tendermint/consensus/_test/_obj_test/state.go:1382 +0x93 github.com/tendermint/tendermint/consensus.(*ConsensusState).handleMsg(0xc424efd520, 0xcb58e0, 0xc42547feb8, 0x0, 0x0, 0x6, 0x0, 0x4, 0xed10ca07e, 0x3077bfea, ...) github.com/tendermint/tendermint/consensus/_test/_obj_test/state.go:660 +0x9fc github.com/tendermint/tendermint/consensus.(*ConsensusState).receiveRoutine(0xc424efd520, 0x0) github.com/tendermint/tendermint/consensus/_test/_obj_test/state.go:615 +0x5f5 created by github.com/tendermint/tendermint/consensus.(*ConsensusState).OnStart github.com/tendermint/tendermint/consensus/_test/_obj_test/state.go:332 +0x4a7 exit status 2 FAIL github.com/tendermint/tendermint/consensus 76.644s make: *** [test_integrations] Error 1 ``` See https://github.com/tendermint/tendermint/pull/568 --- common/os.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/common/os.go b/common/os.go index 9dc81c57..ae2ed087 100644 --- a/common/os.go +++ b/common/os.go @@ -93,9 +93,8 @@ func MustWriteFile(filePath string, contents []byte, mode os.FileMode) { } } -// Writes to newBytes to filePath. -// Guaranteed not to lose *both* oldBytes and newBytes, -// (assuming that the OS is perfect) +// WriteFileAtomic writes newBytes to temp and atomically moves to filePath +// when everything else succeeds. func WriteFileAtomic(filePath string, newBytes []byte, mode os.FileMode) error { // If a file already exists there, copy to filePath+".bak" (overwrite anything) if _, err := os.Stat(filePath); !os.IsNotExist(err) { @@ -108,13 +107,27 @@ func WriteFileAtomic(filePath string, newBytes []byte, mode os.FileMode) error { return fmt.Errorf("Could not write file %v. %v", filePath+".bak", err) } } - // Write newBytes to filePath.new - err := ioutil.WriteFile(filePath+".new", newBytes, mode) + f, err := ioutil.TempFile("", "") if err != nil { - return fmt.Errorf("Could not write file %v. %v", filePath+".new", err) + return err + } + _, err = f.Write(newBytes) + if err == nil { + err = f.Sync() + } + if closeErr := f.Close(); err == nil { + err = closeErr + } + if permErr := os.Chmod(f.Name(), mode); err == nil { + err = permErr + } + if err == nil { + err = os.Rename(f.Name(), filePath) + } + // any err should result in full cleanup + if err != nil { + os.Remove(f.Name()) } - // Move filePath.new to filePath - err = os.Rename(filePath+".new", filePath) return err } From b25aa3b472f67638710954460ef4c77e28dd9e8f Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 28 Jul 2017 11:26:04 -0400 Subject: [PATCH 23/37] [common] do not create {filePath}.bak in WriteFileAtomic We use WriteFileAtomic in two places: ``` p2p/addrbook.go 338: err = cmn.WriteFileAtomic(filePath, jsonBytes, 0644) types/priv_validator.go 162: err = WriteFileAtomic(privVal.filePath, jsonBytes, 0600) ``` and we don't need .bak in any of the above. We save priv_validator every 10ms and addrbook every 2 min. --- common/os.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/common/os.go b/common/os.go index ae2ed087..b1e77897 100644 --- a/common/os.go +++ b/common/os.go @@ -96,17 +96,6 @@ func MustWriteFile(filePath string, contents []byte, mode os.FileMode) { // WriteFileAtomic writes newBytes to temp and atomically moves to filePath // when everything else succeeds. func WriteFileAtomic(filePath string, newBytes []byte, mode os.FileMode) error { - // If a file already exists there, copy to filePath+".bak" (overwrite anything) - if _, err := os.Stat(filePath); !os.IsNotExist(err) { - fileBytes, err := ioutil.ReadFile(filePath) - if err != nil { - return fmt.Errorf("Could not read file %v. %v", filePath, err) - } - err = ioutil.WriteFile(filePath+".bak", fileBytes, mode) - if err != nil { - return fmt.Errorf("Could not write file %v. %v", filePath+".bak", err) - } - } f, err := ioutil.TempFile("", "") if err != nil { return err From d1ca2c6f838fe7428b7da6f62ebf293d4d91b44b Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 28 Jul 2017 11:31:50 -0400 Subject: [PATCH 24/37] [common] add a test for WriteFileAtomic --- common/os_test.go | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 common/os_test.go diff --git a/common/os_test.go b/common/os_test.go new file mode 100644 index 00000000..05359e36 --- /dev/null +++ b/common/os_test.go @@ -0,0 +1,29 @@ +package common + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "testing" + "time" +) + +func TestWriteFileAtomic(t *testing.T) { + data := []byte("Becatron") + fname := fmt.Sprintf("/tmp/write-file-atomic-test-%v.txt", time.Now().UnixNano()) + err := WriteFileAtomic(fname, data, 0664) + if err != nil { + t.Fatal(err) + } + rData, err := ioutil.ReadFile(fname) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(data, rData) { + t.Fatalf("data mismatch: %v != %v", data, rData) + } + if err := os.Remove(fname); err != nil { + t.Fatal(err) + } +} From d67a62171542ef848f97eecf898b5ecf64fe83b5 Mon Sep 17 00:00:00 2001 From: Emmanuel Odeke Date: Mon, 31 Jul 2017 15:11:15 -0600 Subject: [PATCH 25/37] http: http-utils added after extraction Found common http utils that were being multiply duplicated across many libraries and since am moving things in basecoin/unstable to add for more functionality, it's better to put them in one place. Utilities and tests added: - [X] FparseJSON - [X] FparseAndValidateJSON - [X] ParseRequestJSON - [X] ParseAndValidateRequestJSON - [X] WriteCode - [X] WriteError - [X] WriteSuccess - [X] ErrorResponse During review from @ethanfrey, made updates: * Removed tt.want since it was a distraction/artifact that made the reviewer think the tests weren't testing for both failed and passed results. * Added ErrorWithCode as WithCode is a common options pattern in Go that could cause confusion: ErrorWithCode(error, int) ErrorResponse * Using json.NewDecoder(io.Reader) error instead of ioutil.ReadAll(io.Reader) to slurp all the bytes. * Added more test scenarios to achieve 100% coverage of http.go --- common/http.go | 153 +++++++++++++++++++++++++++ common/http_test.go | 250 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 403 insertions(+) create mode 100644 common/http.go create mode 100644 common/http_test.go diff --git a/common/http.go b/common/http.go new file mode 100644 index 00000000..56b5b6c6 --- /dev/null +++ b/common/http.go @@ -0,0 +1,153 @@ +package common + +import ( + "encoding/json" + "io" + "net/http" + + "gopkg.in/go-playground/validator.v9" + + "github.com/pkg/errors" +) + +type ErrorResponse struct { + Success bool `json:"success,omitempty"` + + // Err is the error message if Success is false + Err string `json:"error,omitempty"` + + // Code is set if Success is false + Code int `json:"code,omitempty"` +} + +// ErrorWithCode makes an ErrorResponse with the +// provided err's Error() content, and status code. +// It panics if err is nil. +func ErrorWithCode(err error, code int) *ErrorResponse { + return &ErrorResponse{ + Err: err.Error(), + Code: code, + } +} + +// Ensure that ErrorResponse implements error +var _ error = (*ErrorResponse)(nil) + +func (er *ErrorResponse) Error() string { + return er.Err +} + +// Ensure that ErrorResponse implements httpCoder +var _ httpCoder = (*ErrorResponse)(nil) + +func (er *ErrorResponse) HTTPCode() int { + return er.Code +} + +var errNilBody = errors.Errorf("expecting a non-nil body") + +// FparseJSON unmarshals into save, the body of the provided reader. +// Since it uses json.Unmarshal, save must be of a pointer type +// or compatible with json.Unmarshal. +func FparseJSON(r io.Reader, save interface{}) error { + if r == nil { + return errors.Wrap(errNilBody, "Reader") + } + + dec := json.NewDecoder(r) + if err := dec.Decode(save); err != nil { + return errors.Wrap(err, "Decode/Unmarshal") + } + return nil +} + +// ParseRequestJSON unmarshals into save, the body of the +// request. It closes the body of the request after parsing. +// Since it uses json.Unmarshal, save must be of a pointer type +// or compatible with json.Unmarshal. +func ParseRequestJSON(r *http.Request, save interface{}) error { + if r == nil || r.Body == nil { + return errNilBody + } + defer r.Body.Close() + + return FparseJSON(r.Body, save) +} + +// ParseRequestAndValidateJSON unmarshals into save, the body of the +// request and invokes a validator on the saved content. To ensure +// validation, make sure to set tags "validate" on your struct as +// per https://godoc.org/gopkg.in/go-playground/validator.v9. +// It closes the body of the request after parsing. +// Since it uses json.Unmarshal, save must be of a pointer type +// or compatible with json.Unmarshal. +func ParseRequestAndValidateJSON(r *http.Request, save interface{}) error { + if r == nil || r.Body == nil { + return errNilBody + } + defer r.Body.Close() + + return FparseAndValidateJSON(r.Body, save) +} + +// FparseAndValidateJSON like FparseJSON unmarshals into save, +// the body of the provided reader. However, it invokes the validator +// to check the set validators on your struct fields as per +// per https://godoc.org/gopkg.in/go-playground/validator.v9. +// Since it uses json.Unmarshal, save must be of a pointer type +// or compatible with json.Unmarshal. +func FparseAndValidateJSON(r io.Reader, save interface{}) error { + if err := FparseJSON(r, save); err != nil { + return err + } + return validate(save) +} + +var theValidator = validator.New() + +func validate(obj interface{}) error { + return errors.Wrap(theValidator.Struct(obj), "Validate") +} + +// WriteSuccess JSON marshals the content provided, to an HTTP +// response, setting the provided status code and setting header +// "Content-Type" to "application/json". +func WriteSuccess(w http.ResponseWriter, data interface{}) { + WriteCode(w, data, 200) +} + +// WriteCode JSON marshals content, to an HTTP response, +// setting the provided status code, and setting header +// "Content-Type" to "application/json". If JSON marshalling fails +// with an error, WriteCode instead writes out the error invoking +// WriteError. +func WriteCode(w http.ResponseWriter, out interface{}, code int) { + blob, err := json.MarshalIndent(out, "", " ") + if err != nil { + WriteError(w, err) + } else { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + w.Write(blob) + } +} + +type httpCoder interface { + HTTPCode() int +} + +// WriteError is a convenience function to write out an +// error to an http.ResponseWriter, to send out an error +// that's structured as JSON i.e the form +// {"error": sss, "code": ddd} +// If err implements the interface HTTPCode() int, +// it will use that status code otherwise, it will +// set code to be http.StatusBadRequest +func WriteError(w http.ResponseWriter, err error) { + code := http.StatusBadRequest + if httpC, ok := err.(httpCoder); ok { + code = httpC.HTTPCode() + } + + WriteCode(w, ErrorWithCode(err, code), code) +} diff --git a/common/http_test.go b/common/http_test.go new file mode 100644 index 00000000..b207684b --- /dev/null +++ b/common/http_test.go @@ -0,0 +1,250 @@ +package common_test + +import ( + "bytes" + "encoding/json" + "errors" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/tendermint/tmlibs/common" +) + +func TestWriteSuccess(t *testing.T) { + w := httptest.NewRecorder() + common.WriteSuccess(w, "foo") + assert.Equal(t, w.Code, 200, "should get a 200") +} + +var blankErrResponse = new(common.ErrorResponse) + +func TestWriteError(t *testing.T) { + tests := [...]struct { + msg string + code int + }{ + 0: { + msg: "this is a message", + code: 419, + }, + } + + for i, tt := range tests { + w := httptest.NewRecorder() + msg := tt.msg + + // First check without a defined code, should send back a 400 + common.WriteError(w, errors.New(msg)) + assert.Equal(t, w.Code, http.StatusBadRequest, "#%d: should get a 400", i) + blob, err := ioutil.ReadAll(w.Body) + if err != nil { + assert.Failf(t, "expecting a successful ioutil.ReadAll", "#%d", i) + continue + } + + recv := new(common.ErrorResponse) + if err := json.Unmarshal(blob, recv); err != nil { + assert.Failf(t, "expecting a successful json.Unmarshal", "#%d", i) + continue + } + + assert.Equal(t, reflect.DeepEqual(recv, blankErrResponse), false, "expecting a non-blank error response") + + // Now test with an error that's .HTTPCode() int conforming + + // Reset w + w = httptest.NewRecorder() + + common.WriteError(w, common.ErrorWithCode(errors.New("foo"), tt.code)) + assert.Equal(t, w.Code, tt.code, "case #%d", i) + } +} + +type marshalFailer struct{} + +var errFooFailed = errors.New("foo failed here") + +func (mf *marshalFailer) MarshalJSON() ([]byte, error) { + return nil, errFooFailed +} + +func TestWriteCode(t *testing.T) { + codes := [...]int{ + 0: http.StatusOK, + 1: http.StatusBadRequest, + 2: http.StatusUnauthorized, + 3: http.StatusInternalServerError, + } + + for i, code := range codes { + w := httptest.NewRecorder() + common.WriteCode(w, "foo", code) + assert.Equal(t, w.Code, code, "#%d", i) + + // Then for the failed JSON marshaling + w = httptest.NewRecorder() + common.WriteCode(w, &marshalFailer{}, code) + wantCode := http.StatusBadRequest + assert.Equal(t, w.Code, wantCode, "#%d", i) + assert.True(t, strings.Contains(string(w.Body.Bytes()), errFooFailed.Error()), + "#%d: expected %q in the error message", i, errFooFailed) + } +} + +type saver struct { + Foo int `json:"foo" validate:"min=10"` + Bar string `json:"bar"` +} + +type rcloser struct { + closeOnce sync.Once + body *bytes.Buffer + closeChan chan bool +} + +var errAlreadyClosed = errors.New("already closed") + +func (rc *rcloser) Close() error { + var err = errAlreadyClosed + rc.closeOnce.Do(func() { + err = nil + rc.closeChan <- true + close(rc.closeChan) + }) + return err +} + +func (rc *rcloser) Read(b []byte) (int, error) { + return rc.body.Read(b) +} + +var _ io.ReadCloser = (*rcloser)(nil) + +func makeReq(strBody string) (*http.Request, <-chan bool) { + closeChan := make(chan bool, 1) + buf := new(bytes.Buffer) + buf.Write([]byte(strBody)) + req := &http.Request{ + Header: make(http.Header), + Body: &rcloser{body: buf, closeChan: closeChan}, + } + return req, closeChan +} + +func TestParseRequestJSON(t *testing.T) { + tests := [...]struct { + body string + wantErr bool + useNil bool + }{ + 0: {wantErr: true, body: ``}, + 1: {body: `{}`}, + 2: {body: `{"foo": 2}`}, // Not that the validate tags don't matter here since we are just parsing + 3: {body: `{"foo": "abcd"}`, wantErr: true}, + 4: {useNil: true, wantErr: true}, + } + + for i, tt := range tests { + req, closeChan := makeReq(tt.body) + if tt.useNil { + req.Body = nil + } + sav := new(saver) + err := common.ParseRequestJSON(req, sav) + if tt.wantErr { + assert.NotEqual(t, err, nil, "#%d: want non-nil error", i) + continue + } + assert.Equal(t, err, nil, "#%d: want nil error", i) + wasClosed := <-closeChan + assert.Equal(t, wasClosed, true, "#%d: should have invoked close", i) + } +} + +func TestFparseJSON(t *testing.T) { + r1 := strings.NewReader(`{"foo": 1}`) + sav := new(saver) + require.Equal(t, common.FparseJSON(r1, sav), nil, "expecting successful parsing") + r2 := strings.NewReader(`{"bar": "blockchain"}`) + require.Equal(t, common.FparseJSON(r2, sav), nil, "expecting successful parsing") + require.Equal(t, reflect.DeepEqual(sav, &saver{Foo: 1, Bar: "blockchain"}), true, "should have parsed both") + + // Now with a nil body + require.NotEqual(t, nil, common.FparseJSON(nil, sav), "expecting a nil error report") +} + +func TestFparseAndValidateJSON(t *testing.T) { + r1 := strings.NewReader(`{"foo": 1}`) + sav := new(saver) + require.NotEqual(t, common.FparseAndValidateJSON(r1, sav), nil, "expecting validation to fail") + r1 = strings.NewReader(`{"foo": 100}`) + require.Equal(t, common.FparseJSON(r1, sav), nil, "expecting successful parsing") + r2 := strings.NewReader(`{"bar": "blockchain"}`) + require.Equal(t, common.FparseAndValidateJSON(r2, sav), nil, "expecting successful parsing") + require.Equal(t, reflect.DeepEqual(sav, &saver{Foo: 100, Bar: "blockchain"}), true, "should have parsed both") + + // Now with a nil body + require.NotEqual(t, nil, common.FparseJSON(nil, sav), "expecting a nil error report") +} + +var blankSaver = new(saver) + +func TestParseAndValidateRequestJSON(t *testing.T) { + tests := [...]struct { + body string + wantErr bool + useNil bool + }{ + 0: {wantErr: true, body: ``}, + 1: {body: `{}`, wantErr: true}, // Here it should fail since Foo doesn't meet the minimum value + 2: {body: `{"foo": 2}`, wantErr: true}, // Here validation should fail + 3: {body: `{"foo": "abcd"}`, wantErr: true}, + 4: {useNil: true, wantErr: true}, + 5: {body: `{"foo": 100}`}, // Must succeed + } + + for i, tt := range tests { + req, closeChan := makeReq(tt.body) + if tt.useNil { + req.Body = nil + } + sav := new(saver) + err := common.ParseRequestAndValidateJSON(req, sav) + if tt.wantErr { + assert.NotEqual(t, err, nil, "#%d: want non-nil error", i) + continue + } + + assert.Equal(t, err, nil, "#%d: want nil error", i) + assert.False(t, reflect.DeepEqual(blankSaver, sav), "#%d: expecting a set saver", i) + + wasClosed := <-closeChan + assert.Equal(t, wasClosed, true, "#%d: should have invoked close", i) + } +} + +func TestErrorWithCode(t *testing.T) { + tests := [...]struct { + code int + err error + }{ + 0: {code: 500, err: errors.New("funky")}, + 1: {code: 406, err: errors.New("purist")}, + } + + for i, tt := range tests { + errRes := common.ErrorWithCode(tt.err, tt.code) + assert.Equal(t, errRes.Error(), tt.err.Error(), "#%d: expecting the error values to be equal", i) + assert.Equal(t, errRes.Code, tt.code, "expecting the same status code", i) + assert.Equal(t, errRes.HTTPCode(), tt.code, "expecting the same status code", i) + } +} From b4a51871b91cdae3b1e1b742fcd92ef021de7b73 Mon Sep 17 00:00:00 2001 From: Emmanuel Odeke Date: Fri, 4 Aug 2017 02:03:46 -0600 Subject: [PATCH 26/37] common/IsDirEmpty: do not mask non-existance errors Currently IsDirEmpty returns true, err if it encounters any error after trying to os.Open the directory. I noticed this while studying the code and recalled a bug from an earlier project in which doing the exact same thing on code without permissions would trip out and falsely report that the directory was empty. Given demo.go in https://play.golang.org/p/vhTPU2RiCJ * Demo: ```shell $ mkdir -p sample-demo/1 && touch sample-demo/2 $ echo "1st round" && go run demo.go sample-demo $ sudo chown root sample-demo && sudo chmod 0700 sample-demo $ echo "2nd round" && go run demo.go sample-demo ``` That then prints out ```shell 1st round original:: empty: false err: updated:: empty: false err: 2nd round original:: empty: true err: open data/: permission denied updated:: empty: false err: open data/: permission denied ``` where in "2nd round", the original code falsely reports that the directory is empty but that's a permission error. I could write a code test for it, but that test requires me to change users and switch to root as a Go user so no point in complicating our tests, but otherwise it is a 1-to-1 translation between shell and Go. --- common/os.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/common/os.go b/common/os.go index 9dc81c57..c19322ba 100644 --- a/common/os.go +++ b/common/os.go @@ -48,7 +48,12 @@ func EnsureDir(dir string, mode os.FileMode) error { func IsDirEmpty(name string) (bool, error) { f, err := os.Open(name) if err != nil { - return true, err //folder is non-existent + if os.IsNotExist(err) { + return true, err + } + // Otherwise perhaps a permission + // error or some other error. + return false, err } defer f.Close() From fa990f08036b3f6bbb0d184c657b805e802d915c Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 10 Aug 2017 19:46:59 -0400 Subject: [PATCH 27/37] add test case for hex --- pubsub/query/parser_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pubsub/query/parser_test.go b/pubsub/query/parser_test.go index d96ac082..165ddda7 100644 --- a/pubsub/query/parser_test.go +++ b/pubsub/query/parser_test.go @@ -75,6 +75,9 @@ func TestParser(t *testing.T) { {"account.balance >= -300", false}, {"account.balance >>= 400", false}, {"account.balance=33.22.1", false}, + + {"hash='136E18F7E4C348B780CF873A0BF43922E5BAFA63'", true}, + {"hash=136E18F7E4C348B780CF873A0BF43922E5BAFA63", false}, } for _, c := range cases { From 956966e6587aa6b8dd3a375d35c3744291c38d60 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Fri, 11 Aug 2017 16:36:26 -0400 Subject: [PATCH 28/37] add missing validator package to glide.yaml --- common/http_test.go | 4 ++-- glide.lock | 16 ++++++++++++---- glide.yaml | 1 + 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/common/http_test.go b/common/http_test.go index b207684b..73761fb1 100644 --- a/common/http_test.go +++ b/common/http_test.go @@ -47,13 +47,13 @@ func TestWriteError(t *testing.T) { assert.Equal(t, w.Code, http.StatusBadRequest, "#%d: should get a 400", i) blob, err := ioutil.ReadAll(w.Body) if err != nil { - assert.Failf(t, "expecting a successful ioutil.ReadAll", "#%d", i) + assert.Fail(t, "expecting a successful ioutil.ReadAll", "#%d", i) continue } recv := new(common.ErrorResponse) if err := json.Unmarshal(blob, recv); err != nil { - assert.Failf(t, "expecting a successful json.Unmarshal", "#%d", i) + assert.Fail(t, "expecting a successful json.Unmarshal", "#%d", i) continue } diff --git a/glide.lock b/glide.lock index b30f538a..b0b3ff3c 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 69359a39dbb6957c9f09167520317ad72d4bfa75f37a614b347e2510768c8a42 -updated: 2017-05-05T17:46:34.975369143Z +hash: 6efda1f3891a7211fc3dc1499c0079267868ced9739b781928af8e225420f867 +updated: 2017-08-11T20:28:34.550901198Z imports: - name: github.com/fsnotify/fsnotify version: 4da3e2cfbabc9f751898f250b49f2439785783a1 @@ -11,6 +11,12 @@ imports: - log/term - name: github.com/go-logfmt/logfmt version: 390ab7935ee28ec6b286364bba9b4dd6410cb3d5 +- name: github.com/go-playground/locales + version: 1e5f1161c6416a5ff48840eb8724a394e48cc534 + subpackages: + - currency +- name: github.com/go-playground/universal-translator + version: 71201497bace774495daed26a3874fd339e0b538 - name: github.com/go-stack/stack version: 7a2f19628aabfe68f0766b59e74d6315f8347d22 - name: github.com/golang/snappy @@ -97,11 +103,13 @@ imports: subpackages: - transform - unicode/norm +- name: gopkg.in/go-playground/validator.v9 + version: d529ee1b0f30352444f507cc6cdac96bfd12decc - name: gopkg.in/yaml.v2 version: cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b testImports: - name: github.com/davecgh/go-spew - version: 04cdfd42973bb9c8589fd6a731800cf222fde1a9 + version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9 subpackages: - spew - name: github.com/pmezard/go-difflib @@ -109,7 +117,7 @@ testImports: subpackages: - difflib - name: github.com/stretchr/testify - version: 4d4bfba8f1d1027c4fdbe371823030df51419987 + version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 subpackages: - assert - require diff --git a/glide.yaml b/glide.yaml index d8bdd587..22825a27 100644 --- a/glide.yaml +++ b/glide.yaml @@ -23,6 +23,7 @@ import: - package: golang.org/x/crypto subpackages: - ripemd160 +- package: gopkg.in/go-playground/validator.v9 testImport: - package: github.com/stretchr/testify version: ^1.1.4 From 8f1dea89f5a7b2c2d883a0310c5ef94d8980bcdb Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 25 Aug 2017 16:35:37 -0400 Subject: [PATCH 29/37] db: fix memdb iterator --- db/mem_db.go | 2 +- db/mem_db_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 db/mem_db_test.go diff --git a/db/mem_db.go b/db/mem_db.go index 28662429..db40227e 100644 --- a/db/mem_db.go +++ b/db/mem_db.go @@ -82,7 +82,7 @@ func newMemDBIterator() *memDBIterator { } func (it *memDBIterator) Next() bool { - if it.last >= len(it.keys) { + if it.last >= len(it.keys)-1 { return false } it.last++ diff --git a/db/mem_db_test.go b/db/mem_db_test.go new file mode 100644 index 00000000..a76e10dc --- /dev/null +++ b/db/mem_db_test.go @@ -0,0 +1,28 @@ +package db + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMemDbIterator(t *testing.T) { + db := NewMemDB() + keys := make([][]byte, 100) + for i := 0; i < 100; i++ { + keys[i] = []byte{byte(i)} + } + + value := []byte{5} + for _, k := range keys { + db.Set(k, value) + } + + iter := db.Iterator() + i := 0 + for iter.Next() { + assert.Equal(t, db.Get(iter.Key()), iter.Value(), "values dont match for key") + i += 1 + } + assert.Equal(t, i, len(db.db), "iterator didnt cover whole db") +} From bdfd978b688ef12c0b8a9f1eef5c515298b67654 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 25 Aug 2017 16:58:37 -0400 Subject: [PATCH 30/37] update changelog --- CHANGELOG.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4fad695f..ea7410d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,24 @@ # Changelog -## Develop-Branch changes (unreleased) +## 0.3.0 (August 25, 2017) BREAKING CHANGES: -- [run] NewBaseService takes the new logger +- [common] NewBaseService takes the new logger - [cli] RunCaptureWithArgs now captures stderr and stdout - +func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (stdout, stderr string, err error) - -func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (output string, err error) FEATURES: +- [common] various common HTTP functionality - [common] Date range parsing from string (ex. "2015-12-31:2017-12-31") +- [pubsub] New package for publish-subscribe with more advanced filtering + +BUG FIXES: + +- [common] fix atomicity of WriteFileAtomic by calling fsync +- [db] fix memDb iteration index out of range ## 0.2.2 (June 16, 2017) From bfec1ff1cd7fda9f5b2d8b570e3bec163e5f9149 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 25 Aug 2017 16:58:59 -0400 Subject: [PATCH 31/37] bump version to 0.3.0 --- version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version/version.go b/version/version.go index 42af8ff7..ee59a7ca 100644 --- a/version/version.go +++ b/version/version.go @@ -1,3 +1,3 @@ package version -const Version = "0.2.2" +const Version = "0.3.0" From 3a36776d4a99aeedaef571df0c99e0daa65f7fa8 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Sun, 10 Sep 2017 18:45:20 -0700 Subject: [PATCH 32/37] Reorder file for grokability --- log/filter.go | 76 ++++++++++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/log/filter.go b/log/filter.go index 45108883..f6198b2b 100644 --- a/log/filter.go +++ b/log/filter.go @@ -2,6 +2,26 @@ package log import "fmt" +type level byte + +const ( + levelDebug level = 1 << iota + levelInfo + levelError +) + +type filter struct { + next Logger + allowed level // XOR'd levels for default case + allowedKeyvals map[keyval]level // When key-value match, use this level + errNotAllowed error +} + +type keyval struct { + key interface{} + value interface{} +} + // NewFilter wraps next and implements filtering. See the commentary on the // Option functions for a detailed description of how to configure levels. If // no options are provided, all leveled log events created with Debug, Info or @@ -17,35 +37,6 @@ func NewFilter(next Logger, options ...Option) Logger { return l } -// AllowLevel returns an option for the given level or error if no option exist -// for such level. -func AllowLevel(lvl string) (Option, error) { - switch lvl { - case "debug": - return AllowDebug(), nil - case "info": - return AllowInfo(), nil - case "error": - return AllowError(), nil - case "none": - return AllowNone(), nil - default: - return nil, fmt.Errorf("Expected either \"info\", \"debug\", \"error\" or \"none\" level, given %s", lvl) - } -} - -type filter struct { - next Logger - allowed level - allowedKeyvals map[keyval]level - errNotAllowed error -} - -type keyval struct { - key interface{} - value interface{} -} - func (l *filter) Info(msg string, keyvals ...interface{}) error { levelAllowed := l.allowed&levelInfo != 0 if !levelAllowed { @@ -96,9 +87,28 @@ func (l *filter) With(keyvals ...interface{}) Logger { return &filter{next: l.next.With(keyvals...), allowed: l.allowed, errNotAllowed: l.errNotAllowed, allowedKeyvals: l.allowedKeyvals} } +//-------------------------------------------------------------------------------- + // Option sets a parameter for the filter. type Option func(*filter) +// AllowLevel returns an option for the given level or error if no option exist +// for such level. +func AllowLevel(lvl string) (Option, error) { + switch lvl { + case "debug": + return AllowDebug(), nil + case "info": + return AllowInfo(), nil + case "error": + return AllowError(), nil + case "none": + return AllowNone(), nil + default: + return nil, fmt.Errorf("Expected either \"info\", \"debug\", \"error\" or \"none\" level, given %s", lvl) + } +} + // AllowAll is an alias for AllowDebug. func AllowAll() Option { return AllowDebug() @@ -155,11 +165,3 @@ func AllowErrorWith(key interface{}, value interface{}) Option { func AllowNoneWith(key interface{}, value interface{}) Option { return func(l *filter) { l.allowedKeyvals[keyval{key, value}] = 0 } } - -type level byte - -const ( - levelDebug level = 1 << iota - levelInfo - levelError -) From 9a2438e0dcd35318f839d23debf539193ff50a44 Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Wed, 20 Sep 2017 02:48:04 -0400 Subject: [PATCH 33/37] common: Fingerprint comment --- common/byteslice.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/common/byteslice.go b/common/byteslice.go index be828f06..ceaf06bd 100644 --- a/common/byteslice.go +++ b/common/byteslice.go @@ -4,6 +4,9 @@ import ( "bytes" ) +// Fingerprint returns the first 6 bytes of a byte slice. +// If the slice is less than 6 bytes, the fingerprint +// contains trailing zeroes. func Fingerprint(slice []byte) []byte { fingerprint := make([]byte, 6) copy(fingerprint, slice) From 65a07b80a33196e063601cb97b51e45dcbd7d66c Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 18 Sep 2017 18:01:14 -0700 Subject: [PATCH 34/37] change logger interface to not return errors (Refs #50) See https://github.com/go-kit/kit/issues/164 for discussion of why kitlog returns an error. ``` Package log is designed to be used for more than simple application info/warning/error logging; it's suitable for log-structured data in an e.g. Lambda architecture, where each invocation is important. I agree with you that if we were doing only application logging the error would be more noise than signal. But the scope of the package is larger than that. ``` Since we are doing only application logging and we're not checking errors, it is safe to get rid them. --- log/filter.go | 31 +++++++++++-------------------- log/filter_test.go | 18 ------------------ log/logger.go | 6 +++--- log/nop_logger.go | 14 +++----------- log/nop_logger_test.go | 18 ------------------ log/tm_logger.go | 12 ++++++------ log/tm_logger_test.go | 11 ----------- log/tracing_logger.go | 12 ++++++------ 8 files changed, 29 insertions(+), 93 deletions(-) delete mode 100644 log/nop_logger_test.go diff --git a/log/filter.go b/log/filter.go index f6198b2b..768c09b8 100644 --- a/log/filter.go +++ b/log/filter.go @@ -14,7 +14,6 @@ type filter struct { next Logger allowed level // XOR'd levels for default case allowedKeyvals map[keyval]level // When key-value match, use this level - errNotAllowed error } type keyval struct { @@ -37,28 +36,28 @@ func NewFilter(next Logger, options ...Option) Logger { return l } -func (l *filter) Info(msg string, keyvals ...interface{}) error { +func (l *filter) Info(msg string, keyvals ...interface{}) { levelAllowed := l.allowed&levelInfo != 0 if !levelAllowed { - return l.errNotAllowed + return } - return l.next.Info(msg, keyvals...) + l.next.Info(msg, keyvals...) } -func (l *filter) Debug(msg string, keyvals ...interface{}) error { +func (l *filter) Debug(msg string, keyvals ...interface{}) { levelAllowed := l.allowed&levelDebug != 0 if !levelAllowed { - return l.errNotAllowed + return } - return l.next.Debug(msg, keyvals...) + l.next.Debug(msg, keyvals...) } -func (l *filter) Error(msg string, keyvals ...interface{}) error { +func (l *filter) Error(msg string, keyvals ...interface{}) { levelAllowed := l.allowed&levelError != 0 if !levelAllowed { - return l.errNotAllowed + return } - return l.next.Error(msg, keyvals...) + l.next.Error(msg, keyvals...) } // With implements Logger by constructing a new filter with a keyvals appended @@ -80,11 +79,11 @@ func (l *filter) With(keyvals ...interface{}) Logger { for i := len(keyvals) - 2; i >= 0; i -= 2 { for kv, allowed := range l.allowedKeyvals { if keyvals[i] == kv.key && keyvals[i+1] == kv.value { - return &filter{next: l.next.With(keyvals...), allowed: allowed, errNotAllowed: l.errNotAllowed, allowedKeyvals: l.allowedKeyvals} + return &filter{next: l.next.With(keyvals...), allowed: allowed, allowedKeyvals: l.allowedKeyvals} } } } - return &filter{next: l.next.With(keyvals...), allowed: l.allowed, errNotAllowed: l.errNotAllowed, allowedKeyvals: l.allowedKeyvals} + return &filter{next: l.next.With(keyvals...), allowed: l.allowed, allowedKeyvals: l.allowedKeyvals} } //-------------------------------------------------------------------------------- @@ -138,14 +137,6 @@ func allowed(allowed level) Option { return func(l *filter) { l.allowed = allowed } } -// ErrNotAllowed sets the error to return from Log when it squelches a log -// event disallowed by the configured Allow[Level] option. By default, -// ErrNotAllowed is nil; in this case the log event is squelched with no -// error. -func ErrNotAllowed(err error) Option { - return func(l *filter) { l.errNotAllowed = err } -} - // AllowDebugWith allows error, info and debug level log events to pass for a specific key value pair. func AllowDebugWith(key interface{}, value interface{}) Option { return func(l *filter) { l.allowedKeyvals[keyval{key, value}] = levelError | levelInfo | levelDebug } diff --git a/log/filter_test.go b/log/filter_test.go index 4665db3d..fafafacb 100644 --- a/log/filter_test.go +++ b/log/filter_test.go @@ -2,7 +2,6 @@ package log_test import ( "bytes" - "errors" "strings" "testing" @@ -71,23 +70,6 @@ func TestVariousLevels(t *testing.T) { } } -func TestErrNotAllowed(t *testing.T) { - myError := errors.New("squelched!") - opts := []log.Option{ - log.AllowError(), - log.ErrNotAllowed(myError), - } - logger := log.NewFilter(log.NewNopLogger(), opts...) - - if want, have := myError, logger.Info("foo", "bar", "baz"); want != have { - t.Errorf("want %#+v, have %#+v", want, have) - } - - if want, have := error(nil), logger.Error("foo", "bar", "baz"); want != have { - t.Errorf("want %#+v, have %#+v", want, have) - } -} - func TestLevelContext(t *testing.T) { var buf bytes.Buffer diff --git a/log/logger.go b/log/logger.go index be273f48..ddb187bc 100644 --- a/log/logger.go +++ b/log/logger.go @@ -8,9 +8,9 @@ import ( // Logger is what any Tendermint library should take. type Logger interface { - Debug(msg string, keyvals ...interface{}) error - Info(msg string, keyvals ...interface{}) error - Error(msg string, keyvals ...interface{}) error + Debug(msg string, keyvals ...interface{}) + Info(msg string, keyvals ...interface{}) + Error(msg string, keyvals ...interface{}) With(keyvals ...interface{}) Logger } diff --git a/log/nop_logger.go b/log/nop_logger.go index 306a8405..12d75abe 100644 --- a/log/nop_logger.go +++ b/log/nop_logger.go @@ -8,17 +8,9 @@ var _ Logger = (*nopLogger)(nil) // NewNopLogger returns a logger that doesn't do anything. func NewNopLogger() Logger { return &nopLogger{} } -func (nopLogger) Info(string, ...interface{}) error { - return nil -} - -func (nopLogger) Debug(string, ...interface{}) error { - return nil -} - -func (nopLogger) Error(string, ...interface{}) error { - return nil -} +func (nopLogger) Info(string, ...interface{}) {} +func (nopLogger) Debug(string, ...interface{}) {} +func (nopLogger) Error(string, ...interface{}) {} func (l *nopLogger) With(...interface{}) Logger { return l diff --git a/log/nop_logger_test.go b/log/nop_logger_test.go deleted file mode 100644 index d2009fdf..00000000 --- a/log/nop_logger_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package log_test - -import ( - "testing" - - "github.com/tendermint/tmlibs/log" -) - -func TestNopLogger(t *testing.T) { - t.Parallel() - logger := log.NewNopLogger() - if err := logger.Info("Hello", "abc", 123); err != nil { - t.Error(err) - } - if err := logger.With("def", "ghi").Debug(""); err != nil { - t.Error(err) - } -} diff --git a/log/tm_logger.go b/log/tm_logger.go index a903dbe8..dc6932dd 100644 --- a/log/tm_logger.go +++ b/log/tm_logger.go @@ -50,21 +50,21 @@ func NewTMLoggerWithColorFn(w io.Writer, colorFn func(keyvals ...interface{}) te } // Info logs a message at level Info. -func (l *tmLogger) Info(msg string, keyvals ...interface{}) error { +func (l *tmLogger) Info(msg string, keyvals ...interface{}) { lWithLevel := kitlevel.Info(l.srcLogger) - return kitlog.With(lWithLevel, msgKey, msg).Log(keyvals...) + kitlog.With(lWithLevel, msgKey, msg).Log(keyvals...) } // Debug logs a message at level Debug. -func (l *tmLogger) Debug(msg string, keyvals ...interface{}) error { +func (l *tmLogger) Debug(msg string, keyvals ...interface{}) { lWithLevel := kitlevel.Debug(l.srcLogger) - return kitlog.With(lWithLevel, msgKey, msg).Log(keyvals...) + kitlog.With(lWithLevel, msgKey, msg).Log(keyvals...) } // Error logs a message at level Error. -func (l *tmLogger) Error(msg string, keyvals ...interface{}) error { +func (l *tmLogger) Error(msg string, keyvals ...interface{}) { lWithLevel := kitlevel.Error(l.srcLogger) - return kitlog.With(lWithLevel, msgKey, msg).Log(keyvals...) + kitlog.With(lWithLevel, msgKey, msg).Log(keyvals...) } // With returns a new contextual logger with keyvals prepended to those passed diff --git a/log/tm_logger_test.go b/log/tm_logger_test.go index 15c940ce..8cd2f827 100644 --- a/log/tm_logger_test.go +++ b/log/tm_logger_test.go @@ -7,17 +7,6 @@ import ( "github.com/tendermint/tmlibs/log" ) -func TestTMLogger(t *testing.T) { - t.Parallel() - logger := log.NewTMLogger(ioutil.Discard) - if err := logger.Info("Hello", "abc", 123); err != nil { - t.Error(err) - } - if err := logger.With("def", "ghi").Debug(""); err != nil { - t.Error(err) - } -} - func BenchmarkTMLoggerSimple(b *testing.B) { benchmarkRunner(b, log.NewTMLogger(ioutil.Discard), baseInfoMessage) } diff --git a/log/tracing_logger.go b/log/tracing_logger.go index 794bdaeb..d2a6ff44 100644 --- a/log/tracing_logger.go +++ b/log/tracing_logger.go @@ -28,16 +28,16 @@ type tracingLogger struct { next Logger } -func (l *tracingLogger) Info(msg string, keyvals ...interface{}) error { - return l.next.Info(msg, formatErrors(keyvals)...) +func (l *tracingLogger) Info(msg string, keyvals ...interface{}) { + l.next.Info(msg, formatErrors(keyvals)...) } -func (l *tracingLogger) Debug(msg string, keyvals ...interface{}) error { - return l.next.Debug(msg, formatErrors(keyvals)...) +func (l *tracingLogger) Debug(msg string, keyvals ...interface{}) { + l.next.Debug(msg, formatErrors(keyvals)...) } -func (l *tracingLogger) Error(msg string, keyvals ...interface{}) error { - return l.next.Error(msg, formatErrors(keyvals)...) +func (l *tracingLogger) Error(msg string, keyvals ...interface{}) { + l.next.Error(msg, formatErrors(keyvals)...) } func (l *tracingLogger) With(keyvals ...interface{}) Logger { From 246082368a06552c217803f12aa04c41eed64c0d Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Mon, 18 Sep 2017 18:06:10 -0700 Subject: [PATCH 35/37] add changelog entry [ci skip] --- CHANGELOG.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea7410d7..02278c1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,19 @@ # Changelog -## 0.3.0 (August 25, 2017) +## 0.3.0 (September 21, 2017_ BREAKING CHANGES: +- [log] logger functions no longer returns an error - [common] NewBaseService takes the new logger - [cli] RunCaptureWithArgs now captures stderr and stdout - +func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (stdout, stderr string, err error) - - -func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (output string, err error) + - -func RunCaptureWithArgs(cmd Executable, args []string, env map[string]string) (output string, err error) FEATURES: - [common] various common HTTP functionality -- [common] Date range parsing from string (ex. "2015-12-31:2017-12-31") +- [common] Date range parsing from string (ex. "2015-12-31:2017-12-31") - [pubsub] New package for publish-subscribe with more advanced filtering BUG FIXES: @@ -30,7 +31,7 @@ FEATURES: IMPROVEMENTS: - [cli] Error handling for tests -- [cli] Support dashes in ENV variables +- [cli] Support dashes in ENV variables BUG FIXES: From d71d1394ec48f89fab50bf8e592e3de6200b5b94 Mon Sep 17 00:00:00 2001 From: Anton Kaliaev Date: Thu, 21 Sep 2017 16:11:28 -0700 Subject: [PATCH 36/37] call fsync after flush (Refs #573) short: flushing the bufio buffer is not enough to ensure data consistency. long: Saving an entry to the WAL calls writeLine to append data to the autofile group backing the WAL, then calls group.Flush() to flush that data to persistent storage. group.Flush() in turn proxies to headBuf.flush(), flushing the active bufio.BufferedWriter. However, BufferedWriter wraps a Writer, not another BufferedWriter, and the way it flushes is by calling io.Writer.Write() to clear the BufferedWriter's buffer. The io.Writer we're wrapping here is AutoFile, whose Write method calls os.File.Write(), performing an unbuffered write to the operating system, where, I assume, it sits in the OS buffers awaiting sync. This means that Wal.Save does not, in fact, ensure the saved operation is synced to disk before returning. --- autofile/group.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/autofile/group.go b/autofile/group.go index 39f274e0..ce3e3000 100644 --- a/autofile/group.go +++ b/autofile/group.go @@ -153,7 +153,11 @@ func (g *Group) WriteLine(line string) error { func (g *Group) Flush() error { g.mtx.Lock() defer g.mtx.Unlock() - return g.headBuf.Flush() + err := g.headBuf.Flush() + if err == nil { + err = g.Head.Sync() + } + return err } func (g *Group) processTicks() { From bffe6744ec277d60f707ab442e25513617842f8e Mon Sep 17 00:00:00 2001 From: Ethan Buchman Date: Fri, 22 Sep 2017 09:38:58 -0400 Subject: [PATCH 37/37] changelog --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02278c1e..bf39e544 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## 0.3.0 (September 21, 2017_ +## 0.3.0 (September 22, 2017) BREAKING CHANGES: @@ -14,12 +14,14 @@ FEATURES: - [common] various common HTTP functionality - [common] Date range parsing from string (ex. "2015-12-31:2017-12-31") +- [common] ProtocolAndAddress function - [pubsub] New package for publish-subscribe with more advanced filtering BUG FIXES: - [common] fix atomicity of WriteFileAtomic by calling fsync - [db] fix memDb iteration index out of range +- [autofile] fix Flush by calling fsync ## 0.2.2 (June 16, 2017)