Merge branch 'varint' into develop

binary/binary.go

@@ -9,11 +9,11 @@ import (
 func ReadBinary(o interface{}, r io.Reader, n *int64, err *error) interface{} {
 	rv, rt := reflect.ValueOf(o), reflect.TypeOf(o)
 	if rv.Kind() == reflect.Ptr {
-		readReflect(rv.Elem(), rt.Elem(), r, n, err)
+		readReflectBinary(rv.Elem(), rt.Elem(), Options{}, r, n, err)
 		return o
 	} else {
 		ptrRv := reflect.New(rt)
-		readReflect(ptrRv.Elem(), rt, r, n, err)
+		readReflectBinary(ptrRv.Elem(), rt, Options{}, r, n, err)
 		return ptrRv.Elem().Interface()
 	}
 }
@@ -24,7 +24,7 @@ func WriteBinary(o interface{}, w io.Writer, n *int64, err *error) {
 	if rv.Kind() == reflect.Ptr {
 		rv, rt = rv.Elem(), rt.Elem()
 	}
-	writeReflect(rv, rt, w, n, err)
+	writeReflectBinary(rv, rt, Options{}, w, n, err)
 }
 
 func ReadJSON(o interface{}, bytes []byte, err *error) interface{} {

binary/byteslice.go

@@ -46,12 +46,12 @@ func WriteByteSlices(bzz [][]byte, w io.Writer, n *int64, err *error) {
 }
 
 func ReadByteSlices(r io.Reader, n *int64, err *error) [][]byte {
-	length := ReadUvarint(r, n, err)
+	length := int(ReadUvarint(r, n, err))
 	if *err != nil {
 		return nil
 	}
 	bzz := make([][]byte, length)
-	for i := uint(0); i < length; i++ {
+	for i := 0; i < length; i++ {
 		bz := ReadByteSlice(r, n, err)
 		if *err != nil {
 			return nil

binary/codec.go

@@ -36,51 +36,51 @@ const (
 )
 
 func BasicCodecEncoder(o interface{}, w io.Writer, n *int64, err *error) {
-	switch o.(type) {
+	switch o := o.(type) {
 	case nil:
 		panic("nil type unsupported")
 	case byte:
 		WriteByte(typeByte, w, n, err)
-		WriteByte(o.(byte), w, n, err)
+		WriteByte(o, w, n, err)
 	case int8:
 		WriteByte(typeInt8, w, n, err)
-		WriteInt8(o.(int8), w, n, err)
+		WriteInt8(o, w, n, err)
 	//case uint8:
 	//	WriteByte( typeUint8, w, n, err)
-	//	WriteUint8( o.(uint8), w, n, err)
+	//	WriteUint8( o, w, n, err)
 	case int16:
 		WriteByte(typeInt16, w, n, err)
-		WriteInt16(o.(int16), w, n, err)
+		WriteInt16(o, w, n, err)
 	case uint16:
 		WriteByte(typeUint16, w, n, err)
-		WriteUint16(o.(uint16), w, n, err)
+		WriteUint16(o, w, n, err)
 	case int32:
 		WriteByte(typeInt32, w, n, err)
-		WriteInt32(o.(int32), w, n, err)
+		WriteInt32(o, w, n, err)
 	case uint32:
 		WriteByte(typeUint32, w, n, err)
-		WriteUint32(o.(uint32), w, n, err)
+		WriteUint32(o, w, n, err)
 	case int64:
 		WriteByte(typeInt64, w, n, err)
-		WriteInt64(o.(int64), w, n, err)
+		WriteInt64(o, w, n, err)
 	case uint64:
 		WriteByte(typeUint64, w, n, err)
-		WriteUint64(o.(uint64), w, n, err)
+		WriteUint64(o, w, n, err)
 	case int:
 		WriteByte(typeVarint, w, n, err)
-		WriteVarint(o.(int), w, n, err)
+		WriteVarint(o, w, n, err)
 	case uint:
 		WriteByte(typeUvarint, w, n, err)
-		WriteUvarint(o.(uint), w, n, err)
+		WriteUvarint(o, w, n, err)
 	case string:
 		WriteByte(typeString, w, n, err)
-		WriteString(o.(string), w, n, err)
+		WriteString(o, w, n, err)
 	case []byte:
 		WriteByte(typeByteSlice, w, n, err)
-		WriteByteSlice(o.([]byte), w, n, err)
+		WriteByteSlice(o, w, n, err)
 	case time.Time:
 		WriteByte(typeTime, w, n, err)
-		WriteTime(o.(time.Time), w, n, err)
+		WriteTime(o, w, n, err)
 	default:
 		panic(fmt.Sprintf("Unsupported type: %v", reflect.TypeOf(o)))
 	}

binary/int.go

@@ -158,7 +158,8 @@ func ReadUint64(r io.Reader, n *int64, err *error) uint64 {
 
 // Varint
 
-func uvarIntSize(i uint) int {
+func uvarintSize(i_ uint) int {
+	i := uint64(i_)
 	if i < 1<<8 {
 		return 1
 	}
@@ -189,7 +190,7 @@ func WriteVarint(i int, w io.Writer, n *int64, err *error) {
 		negate = true
 		i = -i
 	}
-	var size = uvarIntSize(uint(i))
+	var size = uvarintSize(uint(i))
 	if negate {
 		// e.g. 0xF1 for a single negative byte
 		WriteUint8(uint8(size+0xF0), w, n, err)
@@ -231,7 +232,7 @@ func ReadVarint(r io.Reader, n *int64, err *error) int {
 // Uvarint
 
 func WriteUvarint(i uint, w io.Writer, n *int64, err *error) {
-	var size = uvarIntSize(i)
+	var size = uvarintSize(i)
 	WriteUint8(uint8(size), w, n, err)
 	buf := make([]byte, 8)
 	binary.BigEndian.PutUint64(buf, uint64(i))

binary/reflect.go

@@ -31,14 +31,39 @@ type TypeInfo struct {
 	Fields []StructFieldInfo
 }
 
-type StructFieldInfo struct {
+type Options struct {
-	Index    int          // Struct field index
+	JSONName string // (JSON) Corresponding JSON field name. (override with `json=""`)
-	JSONName string       // Corresponding JSON field name. (override with `json=""`)
+	Varint   bool   // (Binary) Use length-prefixed encoding for (u)int*
-	Type     reflect.Type // Struct field type
 }
 
-func (info StructFieldInfo) unpack() (int, string, reflect.Type) {
+func getOptionsFromField(field reflect.StructField) (skip bool, opts Options) {
-	return info.Index, info.JSONName, info.Type
+	jsonName := field.Tag.Get("json")
+	if jsonName == "-" {
+		skip = true
+		return
+	} else if jsonName == "" {
+		jsonName = field.Name
+	}
+	varint := false
+	binTag := field.Tag.Get("binary")
+	if binTag == "varint" { // TODO: extend
+		varint = true
+	}
+	opts = Options{
+		JSONName: jsonName,
+		Varint:   varint,
+	}
+	return
+}
+
+type StructFieldInfo struct {
+	Index   int          // Struct field index
+	Type    reflect.Type // Struct field type
+	Options              // Encoding options
+}
+
+func (info StructFieldInfo) unpack() (int, reflect.Type, Options) {
+	return info.Index, info.Type, info.Options
 }
 
 // e.g. If o is struct{Foo}{}, return is the Foo interface type.
@@ -136,16 +161,14 @@ func MakeTypeInfo(rt reflect.Type) *TypeInfo {
 			if field.PkgPath != "" {
 				continue
 			}
-			jsonName := field.Tag.Get("json")
+			skip, opts := getOptionsFromField(field)
-			if jsonName == "-" {
+			if skip {
 				continue
-			} else if jsonName == "" {
-				jsonName = field.Name
 			}
 			structFields = append(structFields, StructFieldInfo{
-				Index:    i,
+				Index:   i,
-				JSONName: jsonName,
+				Type:    field.Type,
-				Type:     field.Type,
+				Options: opts,
 			})
 		}
 		info.Fields = structFields
@@ -154,7 +177,7 @@ func MakeTypeInfo(rt reflect.Type) *TypeInfo {
 	return info
 }
 
-func readReflect(rv reflect.Value, rt reflect.Type, r io.Reader, n *int64, err *error) {
+func readReflectBinary(rv reflect.Value, rt reflect.Type, opts Options, r io.Reader, n *int64, err *error) {
 
 	// Get typeInfo
 	typeInfo := GetTypeInfo(rt)
@@ -179,7 +202,7 @@ func readReflect(rv reflect.Value, rt reflect.Type, r io.Reader, n *int64, err *
 		}
 		crv := reflect.New(crt).Elem()
 		r = NewPrefixedReader([]byte{typeByte}, r)
-		readReflect(crv, crt, r, n, err)
+		readReflectBinary(crv, crt, opts, r, n, err)
 		rv.Set(crv) // NOTE: orig rv is ignored.
 		return
 	}
@@ -236,7 +259,7 @@ func readReflect(rv reflect.Value, rt reflect.Type, r io.Reader, n *int64, err *
 				tmpSliceRv := reflect.MakeSlice(rt, l, l)
 				for j := 0; j < l; j++ {
 					elemRv := tmpSliceRv.Index(j)
-					readReflect(elemRv, elemRt, r, n, err)
+					readReflectBinary(elemRv, elemRt, opts, r, n, err)
 					if *err != nil {
 						return
 					}
@@ -254,14 +277,10 @@ func readReflect(rv reflect.Value, rt reflect.Type, r io.Reader, n *int64, err *
 			log.Debug(Fmt("Read time: %v", t))
 			rv.Set(reflect.ValueOf(t))
 		} else {
-			numFields := rt.NumField()
+			for _, fieldInfo := range typeInfo.Fields {
-			for i := 0; i < numFields; i++ {
+				i, fieldType, opts := fieldInfo.unpack()
-				field := rt.Field(i)
-				if field.PkgPath != "" {
-					continue
-				}
 				fieldRv := rv.Field(i)
-				readReflect(fieldRv, field.Type, r, n, err)
+				readReflectBinary(fieldRv, fieldType, opts, r, n, err)
 			}
 		}
 
@@ -271,9 +290,15 @@ func readReflect(rv reflect.Value, rt reflect.Type, r io.Reader, n *int64, err *
 		rv.SetString(str)
 
 	case reflect.Int64:
-		num := ReadUint64(r, n, err)
+		if opts.Varint {
-		log.Debug(Fmt("Read num: %v", num))
+			num := ReadVarint(r, n, err)
-		rv.SetInt(int64(num))
+			log.Debug(Fmt("Read num: %v", num))
+			rv.SetInt(int64(num))
+		} else {
+			num := ReadInt64(r, n, err)
+			log.Debug(Fmt("Read num: %v", num))
+			rv.SetInt(int64(num))
+		}
 
 	case reflect.Int32:
 		num := ReadUint32(r, n, err)
@@ -291,14 +316,20 @@ func readReflect(rv reflect.Value, rt reflect.Type, r io.Reader, n *int64, err *
 		rv.SetInt(int64(num))
 
 	case reflect.Int:
-		num := ReadUvarint(r, n, err)
+		num := ReadVarint(r, n, err)
 		log.Debug(Fmt("Read num: %v", num))
 		rv.SetInt(int64(num))
 
 	case reflect.Uint64:
-		num := ReadUint64(r, n, err)
+		if opts.Varint {
-		log.Debug(Fmt("Read num: %v", num))
+			num := ReadUvarint(r, n, err)
-		rv.SetUint(uint64(num))
+			log.Debug(Fmt("Read num: %v", num))
+			rv.SetUint(uint64(num))
+		} else {
+			num := ReadUint64(r, n, err)
+			log.Debug(Fmt("Read num: %v", num))
+			rv.SetUint(uint64(num))
+		}
 
 	case reflect.Uint32:
 		num := ReadUint32(r, n, err)
@@ -332,7 +363,7 @@ func readReflect(rv reflect.Value, rt reflect.Type, r io.Reader, n *int64, err *
 
 // rv: the reflection value of the thing to write
 // rt: the type of rv as declared in the container, not necessarily rv.Type().
-func writeReflect(rv reflect.Value, rt reflect.Type, w io.Writer, n *int64, err *error) {
+func writeReflectBinary(rv reflect.Value, rt reflect.Type, opts Options, w io.Writer, n *int64, err *error) {
 
 	// Get typeInfo
 	typeInfo := GetTypeInfo(rt)
@@ -364,8 +395,8 @@ func writeReflect(rv reflect.Value, rt reflect.Type, w io.Writer, n *int64, err
 			// We support writing unsafely for convenience.
 		}
 		// We don't have to write the typeByte here,
-		// the writeReflect() call below will write it.
+		// the writeReflectBinary() call below will write it.
-		writeReflect(crv, crt, w, n, err)
+		writeReflectBinary(crv, crt, opts, w, n, err)
 		return
 	}
 
@@ -405,7 +436,7 @@ func writeReflect(rv reflect.Value, rt reflect.Type, w io.Writer, n *int64, err
 			// Write elems
 			for i := 0; i < length; i++ {
 				elemRv := rv.Index(i)
-				writeReflect(elemRv, elemRt, w, n, err)
+				writeReflectBinary(elemRv, elemRt, opts, w, n, err)
 			}
 		}
 
@@ -414,14 +445,10 @@ func writeReflect(rv reflect.Value, rt reflect.Type, w io.Writer, n *int64, err
 			// Special case: time.Time
 			WriteTime(rv.Interface().(time.Time), w, n, err)
 		} else {
-			numFields := rt.NumField()
+			for _, fieldInfo := range typeInfo.Fields {
-			for i := 0; i < numFields; i++ {
+				i, fieldType, opts := fieldInfo.unpack()
-				field := rt.Field(i)
-				if field.PkgPath != "" {
-					continue
-				}
 				fieldRv := rv.Field(i)
-				writeReflect(fieldRv, field.Type, w, n, err)
+				writeReflectBinary(fieldRv, fieldType, opts, w, n, err)
 			}
 		}
 
@@ -429,7 +456,11 @@ func writeReflect(rv reflect.Value, rt reflect.Type, w io.Writer, n *int64, err
 		WriteString(rv.String(), w, n, err)
 
 	case reflect.Int64:
-		WriteInt64(rv.Int(), w, n, err)
+		if opts.Varint {
+			WriteVarint(int(rv.Int()), w, n, err)
+		} else {
+			WriteInt64(rv.Int(), w, n, err)
+		}
 
 	case reflect.Int32:
 		WriteInt32(int32(rv.Int()), w, n, err)
@@ -444,7 +475,11 @@ func writeReflect(rv reflect.Value, rt reflect.Type, w io.Writer, n *int64, err
 		WriteVarint(int(rv.Int()), w, n, err)
 
 	case reflect.Uint64:
-		WriteUint64(rv.Uint(), w, n, err)
+		if opts.Varint {
+			WriteUvarint(uint(rv.Uint()), w, n, err)
+		} else {
+			WriteUint64(rv.Uint(), w, n, err)
+		}
 
 	case reflect.Uint32:
 		WriteUint32(uint32(rv.Uint()), w, n, err)
@@ -607,8 +642,8 @@ func readReflectJSON(rv reflect.Value, rt reflect.Type, o interface{}, err *erro
 			// TODO: ensure that all fields are set?
 			// TODO: disallow unknown oMap fields?
 			for _, fieldInfo := range typeInfo.Fields {
-				i, jsonName, fieldType := fieldInfo.unpack()
+				i, fieldType, opts := fieldInfo.unpack()
-				value, ok := oMap[jsonName]
+				value, ok := oMap[opts.JSONName]
 				if !ok {
 					continue // Skip missing fields.
 				}
@@ -755,14 +790,14 @@ func writeReflectJSON(rv reflect.Value, rt reflect.Type, w io.Writer, n *int64,
 			WriteTo([]byte("{"), w, n, err)
 			wroteField := false
 			for _, fieldInfo := range typeInfo.Fields {
-				i, jsonName, fieldType := fieldInfo.unpack()
+				i, fieldType, opts := fieldInfo.unpack()
 				fieldRv := rv.Field(i)
 				if wroteField {
 					WriteTo([]byte(","), w, n, err)
 				} else {
 					wroteField = true
 				}
-				WriteTo([]byte(Fmt("\"%v\":", jsonName)), w, n, err)
+				WriteTo([]byte(Fmt("\"%v\":", opts.JSONName)), w, n, err)
 				writeReflectJSON(fieldRv, fieldType, w, n, err)
 			}
 			WriteTo([]byte("}"), w, n, err)

common/bit_array.go

@@ -9,8 +9,8 @@ import (
 
 type BitArray struct {
 	mtx   sync.Mutex
-	Bits  uint     // NOTE: persisted via reflect, must be exported
+	Bits  uint     `json:"bits"`  // NOTE: persisted via reflect, must be exported
-	Elems []uint64 // NOTE: persisted via reflect, must be exported
+	Elems []uint64 `json:"elems"` // NOTE: persisted via reflect, must be exported
 }
 
 func NewBitArray(bits uint) *BitArray {