Feat/stream utils

go.mod

@@ -17,6 +17,7 @@ require (
 	golang.org/x/crypto v0.12.0
 	golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63
 	golang.org/x/sync v0.3.0
+	github.com/icza/bitio v1.1.0
 )
 
 require (

go.sum

@@ -18,6 +18,9 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo=
 github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
 github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
+github.com/icza/bitio v1.1.0 h1:ysX4vtldjdi3Ygai5m1cWy4oLkhWTAi+SyO6HC8L9T0=
+github.com/icza/bitio v1.1.0/go.mod h1:0jGnlLAx8MKMr9VGnn/4YrvZiprkvBelsVIbA9Jjr9A=
+github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA=
 github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g=
 github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=

std/compress/pipeline.go

@@ -1,5 +1,12 @@
 package compress
 
+import (
+	"bytes"
+	"fmt"
+	"github.com/icza/bitio"
+	"strconv"
+)
+
 // Streams and pipelines are inefficient data structures used for easy experimentation with compression algorithms.
 // They make it easy to swap modules in and out.
 
@@ -8,19 +15,19 @@
 	NbSymbs int
 }
 
-func (s Stream) Len() int {
+func (s *Stream) Len() int {
 	return len(s.D)
 }
 
-func (s Stream) RunLen(i int) int {
+func (s *Stream) RunLen(i int) int {
 	runLen := 1
 	for i+runLen < len(s.D) && s.D[i+runLen] == 0 {
 		runLen++
 	}
 	return runLen
 }
 
-func (s Stream) At(i int) int {
+func (s *Stream) At(i int) int {
 	return s.D[i]
 }
 
@@ -32,6 +39,25 @@
 	return Stream{d, 256}
 }
 
+func NewStream[V any](slice []V, srcBitLen, streamBitLen int) Stream {
+	if srcBitLen%streamBitLen != 0 {
+		panic("not implemented")
+	}
+	dstPerSrc := srcBitLen / streamBitLen
+	d := make([]int, dstPerSrc*len(slice))
+
+	for i := range d {
+		if intVal, err := strconv.Atoi(fmt.Sprint(slice[i])); err != nil { // not intended to be fast
+			panic(err)
+		} else {
+			indexWithinWord := i % dstPerSrc
+			d[i] = (uint(intVal) >> (streamBitLen * indexWithinWord)) & ((1 << streamBitLen) - 1)
+		}
+	}
+
+	return Stream{d, 1 << streamBitLen}
+}
+
 type Pipeline []func(Stream) Stream
 
 func (pipeline Pipeline) Run(in Stream) Stream {
@@ -40,3 +66,89 @@
 	}
 	return in
 }
+
+func (s *Stream) WriteNum(r int, nbWords int) *Stream {
+	for i := 0; i < nbWords; i++ {
+		s.D = append(s.D, r%s.NbSymbs)
+		r /= s.NbSymbs
+	}
+	if r != 0 {
+		panic("overflow")
+	}
+	return s
+}
+
+func (s *Stream) ReadNum(start, nbWords int) int {
+	res := 0
+	for j := nbWords - 1; j >= 0; j-- {
+		res *= s.NbSymbs
+		res += s.D[start+j]
+	}
+	return res
+}
+
+func bitLen(n int) int {
+	bitLen := 0
+	for 1<<bitLen < n {
+		bitLen++
+	}
+	return bitLen
+}
+
+func (s *Stream) Marshal() []byte {
+	wordLen := bitLen(s.NbSymbs)
+
+	nbBytes := (len(s.D)*wordLen + 7) / 8
+	encodeLen := false
+	if s.NbSymbs <= 128 {
+		nbBytes++
+		encodeLen = true
+	}
+	bb := bytes.NewBuffer(make([]byte, 0, nbBytes))
+
+	w := bitio.NewWriter(bb)
+	for i := range s.D {
+		if err := w.WriteBits(uint64(s.D[i]), uint8(wordLen)); err != nil {
+			panic(err)
+		}
+	}
+	if err := w.Close(); err != nil {
+		panic(err)
+	}
+
+	if encodeLen {
+		nbWordsInLastByte := len(s.D) - ((nbBytes-2)*8+wordLen-1)/wordLen
+		bb.WriteByte(byte(nbWordsInLastByte))
+	}
+
+	return bb.Bytes()
+}
+
+func (s *Stream) Unmarshal(b []byte) *Stream {
+	wordLen := bitLen(s.NbSymbs)
+
+	var nbWords int
+	if s.NbSymbs <= 128 {
+		nbWordsNotEntirelyInLastByte := ((len(b)-2)*8 + wordLen - 1) / wordLen
+		nbWords = nbWordsNotEntirelyInLastByte + int(b[len(b)-1])
+		b = b[:len(b)-1]
+	} else {
+		nbWords = (len(b) * 8) / wordLen
+	}
+
+	if cap(s.D) < nbWords {
+		s.D = make([]int, nbWords)
+	}
+	s.D = s.D[:nbWords]
+
+	r := bitio.NewReader(bytes.NewReader(b))
+	for i := range s.D {
+		if n, err := r.ReadBits(uint8(wordLen)); err != nil {
+			panic(err)
+		} else {
+			s.D[i] = int(n)
+		}
+	}
+
+	return s
+}

std/compress/pipeline_test.go

@@ -0,0 +1,34 @@
+package compress
+
+import (
+	"github.com/stretchr/testify/assert"
+	"math/rand"
+	"testing"
+)
+
+// TODO Gopter tests?
+
+func fillRandom(s Stream) {
+	for i := range s.D {
+		s.D[i] = rand.Intn(s.NbSymbs) //#nosec G404 weak rng is fine here
+	}
+}
+
+func TestMarshalRoundTrip(t *testing.T) {
+	d := make([]int, 1000)
+	for i := 0; i < 1000; i++ {
+		var s Stream
+		s.D = d[:rand.Intn(len(d))+1]  //#nosec G404 weak rng is fine here
+		s.NbSymbs = rand.Intn(510) + 2 //#nosec G404 weak rng is fine here
+
+		testMarshal(t, s)
+	}
+}
+
+func testMarshal(t *testing.T, s Stream) {
+	fillRandom(s)
+	marshalled := s.Marshal()
+	sBack := Stream{NbSymbs: s.NbSymbs}
+	sBack.Unmarshal(marshalled)
+	assert.Equal(t, s, sBack, "marshalling round trip failed for nbSymbs %d and size %d", s.NbSymbs, len(s.D))
+}

std/hash/mimc/mimc.go

@@ -35,7 +35,7 @@ type MiMC struct {
 	api    frontend.API        // underlying constraint system
 }
 
-// NewMiMC returns a MiMC instance, than can be used in a gnark circuit
+// NewMiMC returns a MiMC instance, that can be used in a gnark circuit
 func NewMiMC(api frontend.API) (MiMC, error) {
 	// TODO @gbotrel use field
 	if constructor, ok := newMimc[utils.FieldToCurve(api.Compiler().Field())]; ok {