From d0e593add389795ad1b3c5ddfff35da56eddaca1 Mon Sep 17 00:00:00 2001 From: Nicholas Wiersma Date: Thu, 7 Mar 2019 16:56:10 +0100 Subject: [PATCH] Added Tests (#4) --- README.md | 1 - codec_skip_internal_test.go | 15 ++++ config_internal_test.go | 53 ++++++++++++ container/container.go | 78 +++++++----------- container/container_test.go | 124 ++++++++++++++++++++++++++++ container/example_test.go | 3 + decoder_test.go | 13 +++ decoder_union_test.go | 13 +++ encoder_test.go | 13 +++ internal/bytesx/reset.go | 37 +++++++++ internal/bytesx/reset_test.go | 37 +++++++++ reader_generic_test.go | 9 +++ registry/client.go | 3 - registry/client_test.go | 4 +- registry/example_test.go | 3 + schema.go | 26 +----- schema_canonical_test.go | 2 +- schema_test.go | 148 +++++++++++++++++++++------------- types_test.go | 3 + 19 files changed, 450 insertions(+), 135 deletions(-) create mode 100644 codec_skip_internal_test.go create mode 100644 config_internal_test.go create mode 100644 internal/bytesx/reset.go create mode 100644 internal/bytesx/reset_test.go diff --git a/README.md b/README.md index 8e106957..432641af 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,6 @@ Always benchmark with your own workload. The result depends heavily on the data ## TODO -* Improve test coverage, docs and examples * Logical Types * Schema * Refactor parsing to be cleaner diff --git a/codec_skip_internal_test.go b/codec_skip_internal_test.go new file mode 100644 index 00000000..3ad5c802 --- /dev/null +++ b/codec_skip_internal_test.go @@ -0,0 +1,15 @@ +package avro + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCreateSkipDecoder_UnsupportedType(t *testing.T) { + schema := NewPrimitiveSchema(Type("test")) + + dec := createSkipDecoder(schema) + + assert.IsType(t, &errorDecoder{}, dec) +} diff --git a/config_internal_test.go b/config_internal_test.go new file mode 100644 index 00000000..ead5f8a1 --- /dev/null +++ b/config_internal_test.go @@ -0,0 +1,53 @@ +package avro + +import ( + "testing" + + "github.com/modern-go/reflect2" + "github.com/stretchr/testify/assert" +) + +func TestConfig_Freeze(t *testing.T) { + api := Config{ + TagKey: "test", + BlockLength: 2, + }.Freeze() + cfg := api.(*frozenConfig) + + assert.Equal(t, "test", cfg.getTagKey()) + assert.Equal(t, 2, cfg.getBlockLength()) +} + +func TestConfig_ReusesDecoders(t *testing.T) { + api := Config{ + TagKey: "test", + BlockLength: 2, + }.Freeze() + cfg := api.(*frozenConfig) + + schema := MustParse(`"long"`) + var long int64 + typ := reflect2.TypeOfPtr(&long) + + dec1 := cfg.DecoderOf(schema, typ) + dec2 := cfg.DecoderOf(schema, typ) + + assert.Equal(t, dec1, dec2) +} + +func TestConfig_ReusesEncoders(t *testing.T) { + api := Config{ + TagKey: "test", + BlockLength: 2, + }.Freeze() + cfg := api.(*frozenConfig) + + schema := MustParse(`"long"`) + var long int64 + typ := reflect2.TypeOfPtr(long) + + enc1 := cfg.EncoderOf(schema, typ) + enc2 := cfg.EncoderOf(schema, typ) + + assert.Equal(t, enc1, enc2) +} diff --git a/container/container.go b/container/container.go index edcc6afd..c172bea5 100644 --- a/container/container.go +++ b/container/container.go @@ -14,6 +14,7 @@ import ( "io" "github.com/hamba/avro" + "github.com/hamba/avro/internal/bytesx" ) const ( @@ -44,7 +45,7 @@ type Header struct { // Decoder reads and decodes Avro values from a container file. type Decoder struct { reader *avro.Reader - resetReader *resetReader + resetReader *bytesx.ResetReader decoder *avro.Decoder sync [16]byte @@ -58,18 +59,18 @@ func NewDecoder(r io.Reader) (*Decoder, error) { var h Header reader.ReadVal(HeaderSchema, &h) if reader.Error != nil { - return nil, fmt.Errorf("file: unexpected error: %v", reader.Error) + return nil, fmt.Errorf("decoder: unexpected error: %v", reader.Error) } if h.Magic != magicBytes { - return nil, errors.New("file: invalid avro file") + return nil, errors.New("decoder: invalid avro file") } schema, err := avro.Parse(string(h.Meta[schemaKey])) if err != nil { return nil, err } - decReader := &resetReader{} + decReader := bytesx.NewResetReader([]byte{}) // TODO: File Codecs // codec, ok := codecs[string(h.Meta[codecKey])] @@ -88,7 +89,7 @@ func NewDecoder(r io.Reader) (*Decoder, error) { // HasNext determines if there is another value to read. func (d *Decoder) HasNext() bool { if d.count <= 0 { - count, _ := d.readBlock() // err handled in Error function + count := d.readBlock() d.count = count } @@ -103,11 +104,7 @@ func (d *Decoder) Decode(v interface{}) error { d.count-- - err := d.decoder.Decode(v) - if err == io.EOF { - return nil - } - return err + return d.decoder.Decode(v) } // Error returns the last reader error. @@ -119,7 +116,7 @@ func (d *Decoder) Error() error { return d.reader.Error } -func (d *Decoder) readBlock() (int64, error) { +func (d *Decoder) readBlock() int64 { count := d.reader.ReadLong() size := d.reader.ReadLong() @@ -130,13 +127,20 @@ func (d *Decoder) readBlock() (int64, error) { var sync [16]byte d.reader.Read(sync[:]) if d.sync != sync && d.reader.Error != io.EOF { - return count, errors.New("file: invalid block") + d.reader.Error = errors.New("decoder: invalid block") } - if d.reader.Error == io.EOF { - return count, nil + return count +} + +// EncoderFunc represents an configuration function for Encoder +type EncoderFunc func(e *Encoder) + +// WithBlockLength sets the block length on the encoder. +func WithBlockLength(length int) EncoderFunc { + return func(e *Encoder) { + e.blockLength = length } - return count, d.reader.Error } // Encoder writes Avro container file to an output stream. @@ -151,7 +155,7 @@ type Encoder struct { } // NewEncoder returns a new encoder that writes to w using schema s. -func NewEncoder(s string, w io.Writer) (*Encoder, error) { +func NewEncoder(s string, w io.Writer, opts ...EncoderFunc) (*Encoder, error) { schema, err := avro.Parse(s) if err != nil { return nil, err @@ -165,25 +169,24 @@ func NewEncoder(s string, w io.Writer) (*Encoder, error) { schemaKey: []byte(schema.String()), }, } - _, err = rand.Read(header.Sync[:]) - if err != nil { - return nil, err - } - + _, _ = rand.Read(header.Sync[:]) writer.WriteVal(HeaderSchema, header) - if writer.Error != nil { - return nil, writer.Error - } buf := &bytes.Buffer{} - return &Encoder{ + e := &Encoder{ writer: writer, buf: buf, encoder: avro.NewEncoderForSchema(schema, buf), sync: header.Sync, blockLength: 100, - }, nil + } + + for _, opt := range opts { + opt(e) + } + + return e, nil } // Encode writes the Avro encoding of v to the stream. @@ -224,26 +227,3 @@ func (e *Encoder) writerBlock() error { e.buf.Reset() return e.writer.Flush() } - -type resetReader struct { - buf []byte - head int - tail int -} - -func (r *resetReader) Read(p []byte) (int, error) { - if r.head == r.tail { - return 0, io.EOF - } - - n := copy(p, r.buf) - r.head += n - - return n, nil -} - -func (r *resetReader) Reset(buf []byte) { - r.buf = buf - r.head = 0 - r.tail = len(buf) -} diff --git a/container/container_test.go b/container/container_test.go index 005408e0..ecd387bb 100644 --- a/container/container_test.go +++ b/container/container_test.go @@ -2,6 +2,7 @@ package container_test import ( "bytes" + "errors" "os" "testing" @@ -54,6 +55,30 @@ type TestRecord struct { Bool bool `avro:"bool"` } +func TestNewDecoder_InvalidHeader(t *testing.T) { + data := []byte{'O', 'b', 'j'} + + _, err := container.NewDecoder(bytes.NewReader(data)) + + assert.Error(t, err) +} + +func TestNewDecoder_InvalidMagic(t *testing.T) { + data := []byte{'f', 'o', 'o', 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + + _, err := container.NewDecoder(bytes.NewReader(data)) + + assert.Error(t, err) +} + +func TestNewDecoder_InvalidSchema(t *testing.T) { + data := []byte{'O', 'b', 'j', 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + + _, err := container.NewDecoder(bytes.NewReader(data)) + + assert.Error(t, err) +} + func TestDecoder(t *testing.T) { unionStr := "union value" want := FullRecord{ @@ -106,6 +131,59 @@ func TestDecoder(t *testing.T) { assert.Equal(t, 1, count) } +func TestDecoder_DecodeAvroError(t *testing.T) { + data := []byte{'O', 'b', 'j', 0x01, 0x01, 0x26, 0x16, 'a', 'v', 'r', 'o', '.', 's', 'c', 'h', 'e', 'm', 'a', + 0x0c, '"', 'l', 'o', 'n', 'g', '"', 0x00, 0xfb, 0x2b, 0x0f, 0x1a, 0xdd, 0xfd, 0x90, 0x7d, 0x87, 0x12, + 0x15, 0x29, 0xd7, 0x1d, 0x1c, 0xdd, 0x02, 0x16, 0xe2, 0xa2, 0xf3, 0xad, 0xad, 0xad, 0xe2, 0xa2, 0xf3, + 0xad, 0xad, 0xfb, 0x2b, 0x0f, 0x1a, 0xdd, 0xfd, 0x90, 0x7d, 0x87, 0x12, 0x15, 0x29, 0xd7, 0x1d, 0x1c, 0xdd, + } + + dec, _ := container.NewDecoder(bytes.NewReader(data)) + _ = dec.HasNext() + + var l int64 + err := dec.Decode(&l) + + assert.Error(t, err) +} + +func TestDecoder_DecodeMustCallHasNext(t *testing.T) { + data := []byte{'O', 'b', 'j', 0x01, 0x01, 0x26, 0x16, 'a', 'v', 'r', 'o', '.', 's', 'c', 'h', 'e', 'm', 'a', + 0x0c, '"', 'l', 'o', 'n', 'g', '"', 0x00, 0xfb, 0x2b, 0x0f, 0x1a, 0xdd, 0xfd, 0x90, 0x7d, 0x87, 0x12, + 0x15, 0x29, 0xd7, 0x1d, 0x1c, 0xdd, 0x02, 0x02, 0x02, 0xfb, 0x2b, 0x0f, 0x1a, 0xdd, 0xfd, 0x90, 0x7d, + 0x87, 0x12, 0x15, 0x29, 0xd7, 0x1d, 0x1c, 0xdd, + } + + dec, _ := container.NewDecoder(bytes.NewReader(data)) + + var l int64 + err := dec.Decode(&l) + + assert.Error(t, err) +} + +func TestDecoder_InvalidBlock(t *testing.T) { + data := []byte{'O', 'b', 'j', 0x01, 0x01, 0x26, 0x16, 'a', 'v', 'r', 'o', '.', 's', 'c', 'h', 'e', 'm', 'a', + 0x0c, '"', 'l', 'o', 'n', 'g', '"', 0x00, 0xfa, 0x2b, 0x0f, 0x1a, 0xdd, 0xfd, 0x90, 0x7d, 0x87, 0x12, + 0x15, 0x29, 0xd7, 0x1d, 0x1c, 0xdd, 0x02, 0x02, 0x02, 0xfb, 0x2b, 0x0f, 0x1a, 0xdd, 0xfd, 0x90, 0x7d, + 0x87, 0x12, 0x15, 0x29, 0xd7, 0x1d, 0x1c, 0xdd, + } + + dec, _ := container.NewDecoder(bytes.NewReader(data)) + + dec.HasNext() + + assert.Error(t, dec.Error()) +} + +func TestNewEncoder_InvalidSchema(t *testing.T) { + buf := &bytes.Buffer{} + + _, err := container.NewEncoder(``, buf) + + assert.Error(t, err) +} + func TestEncoder(t *testing.T) { unionStr := "union value" record := FullRecord{ @@ -147,3 +225,49 @@ func TestEncoder(t *testing.T) { assert.NoError(t, err) } + +func TestEncoder_EncodeError(t *testing.T) { + buf := &bytes.Buffer{} + enc, _ := container.NewEncoder(`"long"`, buf) + + err := enc.Encode("test") + + assert.Error(t, err) +} + +func TestEncoder_EncodeWritesBlocks(t *testing.T) { + buf := &bytes.Buffer{} + enc, _ := container.NewEncoder(`"long"`, buf, container.WithBlockLength(1)) + defer enc.Close() + + err := enc.Encode(int64(1)) + + assert.NoError(t, err) + assert.Equal(t, 61, buf.Len()) +} + +func TestEncoder_EncodeHandlesWriteBlockError(t *testing.T) { + w := &errorWriter{} + enc, _ := container.NewEncoder(`"long"`, w, container.WithBlockLength(1)) + defer enc.Close() + + err := enc.Encode(int64(1)) + + assert.Error(t, err) +} + +func TestEncoder_CloseHandlesWriteBlockError(t *testing.T) { + w := &errorWriter{} + enc, _ := container.NewEncoder(`"long"`, w) + _ = enc.Encode(int64(1)) + + err := enc.Close() + + assert.Error(t, err) +} + +type errorWriter struct{} + +func (*errorWriter) Write(p []byte) (n int, err error) { + return 0, errors.New("test") +} diff --git a/container/example_test.go b/container/example_test.go index 7311a820..bf6c8355 100644 --- a/container/example_test.go +++ b/container/example_test.go @@ -68,6 +68,9 @@ func ExampleNewEncoder() { var record SimpleRecord err = enc.Encode(record) + if err != nil { + log.Fatal(err) + } if err := enc.Close(); err != nil { log.Fatal(err) diff --git a/decoder_test.go b/decoder_test.go index 194b2387..7275fc04 100644 --- a/decoder_test.go +++ b/decoder_test.go @@ -17,6 +17,19 @@ func TestNewDecoder_SchemaError(t *testing.T) { assert.Error(t, err) } +func TestDecoder_DecodeUnsupportedTypeError(t *testing.T) { + defer ConfigTeardown() + + data := []byte{0x01} + schema := avro.NewPrimitiveSchema(avro.Type("test")) + dec := avro.NewDecoderForSchema(schema, bytes.NewReader(data)) + + var b bool + err := dec.Decode(&b) + + assert.Error(t, err) +} + func TestDecoder_DecodeEmptyReader(t *testing.T) { defer ConfigTeardown() diff --git a/decoder_union_test.go b/decoder_union_test.go index 9b43faea..28691038 100644 --- a/decoder_union_test.go +++ b/decoder_union_test.go @@ -158,6 +158,19 @@ func TestDecoder_UnionTypedSetTypeError(t *testing.T) { assert.Error(t, err) } +func TestDecoder_UnionTypedSetTypeNilPtr(t *testing.T) { + defer ConfigTeardown() + + data := []byte{0x02, 0x06, 0x66, 0x6F, 0x6F} + schema := `["null", "string"]` + dec, _ := avro.NewDecoder(schema, bytes.NewReader(data)) + + got := &TestUnionType{} + err := dec.Decode(&got) + + assert.Error(t, err) +} + func TestDecoder_UnionTypedInvalidSchema(t *testing.T) { defer ConfigTeardown() diff --git a/encoder_test.go b/encoder_test.go index a4bb4e09..e14fc65e 100644 --- a/encoder_test.go +++ b/encoder_test.go @@ -1,6 +1,7 @@ package avro_test import ( + "bytes" "testing" "github.com/hamba/avro" @@ -16,6 +17,18 @@ func TestNewEncoder_SchemaError(t *testing.T) { assert.Error(t, err) } +func TestEncoder_EncodeUnsupportedType(t *testing.T) { + defer ConfigTeardown() + + schema := avro.NewPrimitiveSchema(avro.Type("test")) + buf := bytes.NewBuffer([]byte{}) + enc := avro.NewEncoderForSchema(schema, buf) + + err := enc.Encode(true) + + assert.Error(t, err) +} + func TestMarshal(t *testing.T) { defer ConfigTeardown() diff --git a/internal/bytesx/reset.go b/internal/bytesx/reset.go new file mode 100644 index 00000000..5c4c2415 --- /dev/null +++ b/internal/bytesx/reset.go @@ -0,0 +1,37 @@ +package bytesx + +import "io" + +// ResetReader implements the io.Reader reading from a resettable byte slice. +type ResetReader struct { + buf []byte + head int + tail int +} + +// NewResetReader returns a new Reader reading from b. +func NewResetReader(b []byte) *ResetReader { + r := &ResetReader{} + r.Reset(b) + + return r +} + +// Read reads bytes into p. +func (r *ResetReader) Read(p []byte) (int, error) { + if r.head == r.tail { + return 0, io.EOF + } + + n := copy(p, r.buf) + r.head += n + + return n, nil +} + +// Reset resets the byte slice being read from. +func (r *ResetReader) Reset(b []byte) { + r.buf = b + r.head = 0 + r.tail = len(b) +} diff --git a/internal/bytesx/reset_test.go b/internal/bytesx/reset_test.go new file mode 100644 index 00000000..4a0488ce --- /dev/null +++ b/internal/bytesx/reset_test.go @@ -0,0 +1,37 @@ +package bytesx_test + +import ( + "io" + "testing" + + "github.com/hamba/avro/internal/bytesx" + "github.com/stretchr/testify/assert" +) + +func TestNewResetReader(t *testing.T) { + r := bytesx.NewResetReader([]byte{}) + + assert.IsType(t, &bytesx.ResetReader{}, r) + assert.Implements(t, (*io.Reader)(nil), r) +} + +func TestResetReader_Read(t *testing.T) { + r := bytesx.NewResetReader([]byte("test")) + + b := make([]byte, 4) + n, err := r.Read(b) + + assert.NoError(t, err) + assert.Equal(t, 4, n) + assert.Equal(t, []byte("test"), b) +} + +func TestResetReader_ReadReturnsEOF(t *testing.T) { + r := bytesx.NewResetReader([]byte{}) + + b := make([]byte, 4) + n, err := r.Read(b) + + assert.Equal(t, io.EOF, err) + assert.Equal(t, 0, n) +} diff --git a/reader_generic_test.go b/reader_generic_test.go index 598d29b5..8940445d 100644 --- a/reader_generic_test.go +++ b/reader_generic_test.go @@ -162,3 +162,12 @@ func TestReader_ReadNext(t *testing.T) { }) } } + +func TestReader_ReadNextUnsupportedType(t *testing.T) { + schema := avro.NewPrimitiveSchema(avro.Type("test")) + r := avro.NewReader(bytes.NewReader([]byte{0x01}), 10) + + _ = r.ReadNext(schema) + + assert.Error(t, r.Error) +} diff --git a/registry/client.go b/registry/client.go index 40a0be5a..d5cac146 100644 --- a/registry/client.go +++ b/registry/client.go @@ -237,6 +237,3 @@ type Error struct { func (e Error) Error() string { return e.Message } - - - diff --git a/registry/client_test.go b/registry/client_test.go index f0ae7247..a643f5aa 100644 --- a/registry/client_test.go +++ b/registry/client_test.go @@ -395,8 +395,8 @@ func TestClient_HandlesServerError(t *testing.T) { func TestError_Error(t *testing.T) { err := registry.Error{ StatusCode: 404, - Code: 40403, - Message: "Schema not found", + Code: 40403, + Message: "Schema not found", } str := err.Error() diff --git a/registry/example_test.go b/registry/example_test.go index 88b561a5..77e55b8f 100644 --- a/registry/example_test.go +++ b/registry/example_test.go @@ -24,6 +24,9 @@ func Example() { id, schema, err := reg.IsRegistered("foobar", schemaRaw) if err != nil { id, schema, err = reg.CreateSchema("foobar", schemaRaw) + if err != nil { + log.Fatal(err) + } } fmt.Println("id: ", id) diff --git a/schema.go b/schema.go index 2a374867..4501e5ff 100644 --- a/schema.go +++ b/schema.go @@ -127,7 +127,6 @@ type RecordSchema struct { name string fields []*Field - canonical string fingerprint [32]byte } @@ -148,10 +147,6 @@ func (s *RecordSchema) Fields() []*Field { // String returns the canonical form of the schema. func (s *RecordSchema) String() string { - if s.canonical != "" { - return s.canonical - } - fields := "" for _, f := range s.fields { fields += f.String() + "," @@ -160,8 +155,7 @@ func (s *RecordSchema) String() string { fields = fields[:len(fields)-1] } - s.canonical = `{"name":"` + s.name + `","type":"record","fields":[` + fields + `]}` - return s.canonical + return `{"name":"` + s.name + `","type":"record","fields":[` + fields + `]}` } // Fingerprint returns the SHA256 fingerprint of the schema. @@ -323,7 +317,6 @@ func (s *MapSchema) String() string { type UnionSchema struct { types Schemas - canonical string fingerprint [32]byte } @@ -348,10 +341,6 @@ func (s *UnionSchema) Nullable() bool { // String returns the canonical form of the schema. func (s *UnionSchema) String() string { - if s.canonical != "" { - return s.canonical - } - types := "" for _, typ := range s.types { types += typ.String() + "," @@ -360,8 +349,7 @@ func (s *UnionSchema) String() string { types = types[:len(types)-1] } - s.canonical = `[` + types + `]` - return s.canonical + return `[` + types + `]` } // Fingerprint returns the SHA256 fingerprint of the schema. @@ -379,7 +367,6 @@ type FixedSchema struct { name string size int - canonical string fingerprint [32]byte } @@ -400,13 +387,8 @@ func (s *FixedSchema) Size() int { // String returns the canonical form of the schema. func (s *FixedSchema) String() string { - if s.canonical != "" { - return s.canonical - } - size := strconv.Itoa(s.size) - s.canonical = `{"name":"` + s.name + `","type":"fixed","size":` + size + `}` - return s.canonical + return `{"name":"` + s.name + `","type":"fixed","size":` + size + `}` } // Fingerprint returns the SHA256 fingerprint of the schema. @@ -434,7 +416,7 @@ func (s *NullSchema) String() string { // Fingerprint returns the SHA256 fingerprint of the schema. func (s *NullSchema) Fingerprint() [32]byte { - return [32]uint8{0xf0, 0x72, 0xcb, 0xec, 0x3b, 0xf8, 0x84, 0x18, 0x71, 0xd4, 0x28, 0x42, 0x30, 0xc5, 0xe9, 0x83, 0xdc, 0x21, 0x1a, 0x56, 0x83, 0x7a, 0xed, 0x86, 0x24, 0x87, 0x14, 0x8f, 0x94, 0x7d, 0x1a, 0x1f} + return [32]byte{0xf0, 0x72, 0xcb, 0xec, 0x3b, 0xf8, 0x84, 0x18, 0x71, 0xd4, 0x28, 0x42, 0x30, 0xc5, 0xe9, 0x83, 0xdc, 0x21, 0x1a, 0x56, 0x83, 0x7a, 0xed, 0x86, 0x24, 0x87, 0x14, 0x8f, 0x94, 0x7d, 0x1a, 0x1f} } // RefSchema is a reference to a named Avro schema. diff --git a/schema_canonical_test.go b/schema_canonical_test.go index 55fcaaf0..5459362c 100644 --- a/schema_canonical_test.go +++ b/schema_canonical_test.go @@ -10,7 +10,7 @@ import ( // Test cases are taken from the reference implementation here: // https://github.com/apache/avro/blob/master/share/test/data/schema-tests.txt -func TestSchema_CanonicalAndFingerprint(t *testing.T) { +func TestSchema_Canonical(t *testing.T) { tests := []struct { input string canonical string diff --git a/schema_test.go b/schema_test.go index 4ad85bb1..6669a7a1 100644 --- a/schema_test.go +++ b/schema_test.go @@ -58,73 +58,90 @@ func TestNullSchema(t *testing.T) { } for _, schm := range schemas { - s, err := avro.Parse(schm) + schema, err := avro.Parse(schm) assert.NoError(t, err) - assert.Equal(t, avro.Null, s.Type()) + assert.Equal(t, avro.Null, schema.Type()) + want := [32]byte{0xf0, 0x72, 0xcb, 0xec, 0x3b, 0xf8, 0x84, 0x18, 0x71, 0xd4, 0x28, 0x42, 0x30, 0xc5, 0xe9, 0x83, 0xdc, 0x21, 0x1a, 0x56, 0x83, 0x7a, 0xed, 0x86, 0x24, 0x87, 0x14, 0x8f, 0x94, 0x7d, 0x1a, 0x1f} + assert.Equal(t, want, schema.Fingerprint()) } } func TestPrimitiveSchema(t *testing.T) { tests := []struct { - schema string - want avro.Type + schema string + want avro.Type + wantFingerprint [32]byte }{ { - schema: "string", - want: avro.String, + schema: "string", + want: avro.String, + wantFingerprint: [32]byte{0xe9, 0xe5, 0xc1, 0xc9, 0xe4, 0xf6, 0x27, 0x73, 0x39, 0xd1, 0xbc, 0xde, 0x7, 0x33, 0xa5, 0x9b, 0xd4, 0x2f, 0x87, 0x31, 0xf4, 0x49, 0xda, 0x6d, 0xc1, 0x30, 0x10, 0xa9, 0x16, 0x93, 0xd, 0x48}, }, { - schema: `{"type":"string"}`, - want: avro.String, + schema: `{"type":"string"}`, + want: avro.String, + wantFingerprint: [32]byte{0xe9, 0xe5, 0xc1, 0xc9, 0xe4, 0xf6, 0x27, 0x73, 0x39, 0xd1, 0xbc, 0xde, 0x7, 0x33, 0xa5, 0x9b, 0xd4, 0x2f, 0x87, 0x31, 0xf4, 0x49, 0xda, 0x6d, 0xc1, 0x30, 0x10, 0xa9, 0x16, 0x93, 0xd, 0x48}, }, { - schema: "bytes", - want: avro.Bytes, + schema: "bytes", + want: avro.Bytes, + wantFingerprint: [32]byte{0x9a, 0xe5, 0x7, 0xa9, 0xdd, 0x39, 0xee, 0x5b, 0x7c, 0x7e, 0x28, 0x5d, 0xa2, 0xc0, 0x84, 0x65, 0x21, 0xc8, 0xae, 0x8d, 0x80, 0xfe, 0xea, 0xe5, 0x50, 0x4e, 0xc, 0x98, 0x1d, 0x53, 0xf5, 0xfa}, }, { - schema: `{"type":"bytes"}`, - want: avro.Bytes, + schema: `{"type":"bytes"}`, + want: avro.Bytes, + wantFingerprint: [32]byte{0x9a, 0xe5, 0x7, 0xa9, 0xdd, 0x39, 0xee, 0x5b, 0x7c, 0x7e, 0x28, 0x5d, 0xa2, 0xc0, 0x84, 0x65, 0x21, 0xc8, 0xae, 0x8d, 0x80, 0xfe, 0xea, 0xe5, 0x50, 0x4e, 0xc, 0x98, 0x1d, 0x53, 0xf5, 0xfa}, }, { - schema: "int", - want: avro.Int, + schema: "int", + want: avro.Int, + wantFingerprint: [32]byte{0x3f, 0x2b, 0x87, 0xa9, 0xfe, 0x7c, 0xc9, 0xb1, 0x38, 0x35, 0x59, 0x8c, 0x39, 0x81, 0xcd, 0x45, 0xe3, 0xe3, 0x55, 0x30, 0x9e, 0x50, 0x90, 0xaa, 0x9, 0x33, 0xd7, 0xbe, 0xcb, 0x6f, 0xba, 0x45}, }, { - schema: `{"type":"int"}`, - want: avro.Int, + schema: `{"type":"int"}`, + want: avro.Int, + wantFingerprint: [32]byte{0x3f, 0x2b, 0x87, 0xa9, 0xfe, 0x7c, 0xc9, 0xb1, 0x38, 0x35, 0x59, 0x8c, 0x39, 0x81, 0xcd, 0x45, 0xe3, 0xe3, 0x55, 0x30, 0x9e, 0x50, 0x90, 0xaa, 0x9, 0x33, 0xd7, 0xbe, 0xcb, 0x6f, 0xba, 0x45}, }, { - schema: "long", - want: avro.Long, + schema: "long", + want: avro.Long, + wantFingerprint: [32]byte{0xc3, 0x2c, 0x49, 0x7d, 0xf6, 0x73, 0xc, 0x97, 0xfa, 0x7, 0x36, 0x2a, 0xa5, 0x2, 0x3f, 0x37, 0xd4, 0x9a, 0x2, 0x7e, 0xc4, 0x52, 0x36, 0x7, 0x78, 0x11, 0x4c, 0xf4, 0x27, 0x96, 0x5a, 0xdd}, }, { - schema: `{"type":"long"}`, - want: avro.Long, + schema: `{"type":"long"}`, + want: avro.Long, + wantFingerprint: [32]byte{0xc3, 0x2c, 0x49, 0x7d, 0xf6, 0x73, 0xc, 0x97, 0xfa, 0x7, 0x36, 0x2a, 0xa5, 0x2, 0x3f, 0x37, 0xd4, 0x9a, 0x2, 0x7e, 0xc4, 0x52, 0x36, 0x7, 0x78, 0x11, 0x4c, 0xf4, 0x27, 0x96, 0x5a, 0xdd}, }, { - schema: "float", - want: avro.Float, + schema: "float", + want: avro.Float, + wantFingerprint: [32]byte{0x1e, 0x71, 0xf9, 0xec, 0x5, 0x1d, 0x66, 0x3f, 0x56, 0xb0, 0xd8, 0xe1, 0xfc, 0x84, 0xd7, 0x1a, 0xa5, 0x6c, 0xcf, 0xe9, 0xfa, 0x93, 0xaa, 0x20, 0xd1, 0x5, 0x47, 0xa7, 0xab, 0xeb, 0x5c, 0xc0}, }, { - schema: `{"type":"float"}`, - want: avro.Float, + schema: `{"type":"float"}`, + want: avro.Float, + wantFingerprint: [32]byte{0x1e, 0x71, 0xf9, 0xec, 0x5, 0x1d, 0x66, 0x3f, 0x56, 0xb0, 0xd8, 0xe1, 0xfc, 0x84, 0xd7, 0x1a, 0xa5, 0x6c, 0xcf, 0xe9, 0xfa, 0x93, 0xaa, 0x20, 0xd1, 0x5, 0x47, 0xa7, 0xab, 0xeb, 0x5c, 0xc0}, }, { - schema: "double", - want: avro.Double, + schema: "double", + want: avro.Double, + wantFingerprint: [32]byte{0x73, 0xa, 0x9a, 0x8c, 0x61, 0x16, 0x81, 0xd7, 0xee, 0xf4, 0x42, 0xe0, 0x3c, 0x16, 0xc7, 0xd, 0x13, 0xbc, 0xa3, 0xeb, 0x8b, 0x97, 0x7b, 0xb4, 0x3, 0xea, 0xff, 0x52, 0x17, 0x6a, 0xf2, 0x54}, }, { - schema: `{"type":"double"}`, - want: avro.Double, + schema: `{"type":"double"}`, + want: avro.Double, + wantFingerprint: [32]byte{0x73, 0xa, 0x9a, 0x8c, 0x61, 0x16, 0x81, 0xd7, 0xee, 0xf4, 0x42, 0xe0, 0x3c, 0x16, 0xc7, 0xd, 0x13, 0xbc, 0xa3, 0xeb, 0x8b, 0x97, 0x7b, 0xb4, 0x3, 0xea, 0xff, 0x52, 0x17, 0x6a, 0xf2, 0x54}, }, { - schema: "boolean", - want: avro.Boolean, + schema: "boolean", + want: avro.Boolean, + wantFingerprint: [32]byte{0xa5, 0xb0, 0x31, 0xab, 0x62, 0xbc, 0x41, 0x6d, 0x72, 0xc, 0x4, 0x10, 0xd8, 0x2, 0xea, 0x46, 0xb9, 0x10, 0xc4, 0xfb, 0xe8, 0x5c, 0x50, 0xa9, 0x46, 0xcc, 0xc6, 0x58, 0xb7, 0x4e, 0x67, 0x7e}, }, { - schema: `{"type":"boolean"}`, - want: avro.Boolean, + schema: `{"type":"boolean"}`, + want: avro.Boolean, + wantFingerprint: [32]byte{0xa5, 0xb0, 0x31, 0xab, 0x62, 0xbc, 0x41, 0x6d, 0x72, 0xc, 0x4, 0x10, 0xd8, 0x2, 0xea, 0x46, 0xb9, 0x10, 0xc4, 0xfb, 0xe8, 0x5c, 0x50, 0xa9, 0x46, 0xcc, 0xc6, 0x58, 0xb7, 0x4e, 0x67, 0x7e}, }, } @@ -134,6 +151,7 @@ func TestPrimitiveSchema(t *testing.T) { assert.NoError(t, err) assert.Equal(t, tt.want, s.Type()) + assert.Equal(t, tt.wantFingerprint, s.Fingerprint()) }) } } @@ -293,18 +311,21 @@ func TestRecordSchema_WithReference(t *testing.T) { assert.NoError(t, err) assert.Equal(t, avro.Record, s.Type()) assert.Equal(t, avro.Ref, s.(*avro.RecordSchema).Fields()[1].Type().Type()) + assert.Equal(t, s.Fingerprint(), s.(*avro.RecordSchema).Fields()[1].Type().Fingerprint()) } func TestEnumSchema(t *testing.T) { tests := []struct { - name string - schema string - wantErr bool + name string + schema string + wantName string + wantErr bool }{ { - name: "Valid", - schema: `{"type":"enum", "name":"test", "namespace": "org.apache.avro", "symbols":["TEST"]}`, - wantErr: false, + name: "Valid", + schema: `{"type":"enum", "name":"test", "namespace": "org.apache.avro", "symbols":["TEST"]}`, + wantName: "org.apache.avro.test", + wantErr: false, }, { name: "Invalid Name", @@ -355,7 +376,7 @@ func TestEnumSchema(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s, err := avro.Parse(tt.schema) + schema, err := avro.Parse(tt.schema) if tt.wantErr { assert.Error(t, err) @@ -363,7 +384,9 @@ func TestEnumSchema(t *testing.T) { } assert.NoError(t, err) - assert.Equal(t, avro.Enum, s.Type()) + assert.Equal(t, avro.Enum, schema.Type()) + named := schema.(avro.NamedSchema) + assert.Equal(t, tt.wantName, named.Name()) }) } } @@ -446,19 +469,22 @@ func TestMapSchema(t *testing.T) { func TestUnionSchema(t *testing.T) { tests := []struct { - name string - schema string - wantErr bool + name string + schema string + wantFingerprint [32]byte + wantErr bool }{ { - name: "Valid Simple", - schema: `["null", "int"]`, - wantErr: false, + name: "Valid Simple", + schema: `["null", "int"]`, + wantFingerprint: [32]byte{0xb4, 0x94, 0x95, 0xc5, 0xb1, 0xc2, 0x6f, 0x4, 0x89, 0x6a, 0x5f, 0x68, 0x65, 0xf, 0xe2, 0xb7, 0x64, 0x23, 0x62, 0xc3, 0x41, 0x98, 0xd6, 0xbc, 0x74, 0x65, 0xa1, 0xd9, 0xf7, 0xe1, 0xaf, 0xce}, + wantErr: false, }, { - name: "Valid Complex", - schema: `{"type":["null", "int"]}`, - wantErr: false, + name: "Valid Complex", + schema: `{"type":["null", "int"]}`, + wantFingerprint: [32]byte{0xb4, 0x94, 0x95, 0xc5, 0xb1, 0xc2, 0x6f, 0x4, 0x89, 0x6a, 0x5f, 0x68, 0x65, 0xf, 0xe2, 0xb7, 0x64, 0x23, 0x62, 0xc3, 0x41, 0x98, 0xd6, 0xbc, 0x74, 0x65, 0xa1, 0xd9, 0xf7, 0xe1, 0xaf, 0xce}, + wantErr: false, }, { name: "Invalid Type", @@ -478,20 +504,25 @@ func TestUnionSchema(t *testing.T) { assert.NoError(t, err) assert.Equal(t, avro.Union, s.Type()) + assert.Equal(t, tt.wantFingerprint, s.Fingerprint()) }) } } func TestFixedSchema(t *testing.T) { tests := []struct { - name string - schema string - wantErr bool + name string + schema string + wantName string + wantFingerprint [32]byte + wantErr bool }{ { - name: "Valid", - schema: `{"type":"fixed", "name":"test", "namespace": "org.apache.avro", "size": 12}`, - wantErr: false, + name: "Valid", + schema: `{"type":"fixed", "name":"test", "namespace": "org.apache.avro", "size": 12}`, + wantName: "org.apache.avro.test", + wantFingerprint: [32]byte{0xa8, 0x13, 0xfa, 0xb4, 0xf, 0xd7, 0xe3, 0xc9, 0x3a, 0x98, 0x77, 0x24, 0xaf, 0xa9, 0x36, 0xe6, 0xe9, 0x53, 0xa9, 0x1c, 0x10, 0x70, 0xfe, 0x4e, 0x13, 0x2a, 0x7c, 0x51, 0x6, 0x5f, 0xa4, 0xbc}, + wantErr: false, }, { name: "Invalid Name", @@ -532,7 +563,7 @@ func TestFixedSchema(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s, err := avro.Parse(tt.schema) + schema, err := avro.Parse(tt.schema) if tt.wantErr { assert.Error(t, err) @@ -540,7 +571,10 @@ func TestFixedSchema(t *testing.T) { } assert.NoError(t, err) - assert.Equal(t, avro.Fixed, s.Type()) + assert.Equal(t, avro.Fixed, schema.Type()) + named := schema.(avro.NamedSchema) + assert.Equal(t, tt.wantName, named.Name()) + assert.Equal(t, tt.wantFingerprint, named.Fingerprint()) }) } } diff --git a/types_test.go b/types_test.go index 0b463740..2c51208a 100644 --- a/types_test.go +++ b/types_test.go @@ -47,6 +47,9 @@ func (u *TestUnionType) SetType(typ string) error { case string(avro.Int): u.Val = int(0) + case string(avro.String): + u.Val = nil + default: return fmt.Errorf("unknown type %s", typ) }