diff --git a/std/compress/lzss_v1/compress.go b/std/compress/lzss_v1/compress.go index 20172bd10d..2147cbcc2f 100644 --- a/std/compress/lzss_v1/compress.go +++ b/std/compress/lzss_v1/compress.go @@ -1,9 +1,8 @@ package lzss_v1 import ( - "bytes" "encoding/binary" - "fmt" + "github.com/consensys/gnark/std/compress" "index/suffixarray" "math/bits" @@ -21,25 +20,22 @@ import ( // In fact, DEFLATE is LZSS + Huffman coding. It is implemented in gzip which is the standard tool for compressing programmatic data. // For more information, refer to Bill Bird's fantastic undergraduate course on Data Compression // In particular those on the LZ family: https://youtu.be/z1I1o7zySUI and DEFLATE: https://youtu.be/SJPvNi4HrWQ -func Compress(d []byte, settings Settings) (c []byte, err error) { +func Compress(d []byte, settings Settings) (c compress.Stream, err error) { // d[i < 0] = Settings.BackRefSettings.Symbol by convention - var out bytes.Buffer + c.NbSymbs = 257 emitBackRef := func(offset, length int) { - out.WriteByte(0) - emit(&out, offset-1, settings.NbBytesAddress) - emit(&out, length-1, settings.NbBytesLength) + c.D = append(c.D, 256) + emit(&c.D, offset-1, settings.NbBytesAddress) + emit(&c.D, length-1, settings.NbBytesLength) } compressor := newCompressor(d, settings) - i := 0 + i := int(settings.StartAt) for i < len(d) { addr, length := compressor.longestMostRecentBackRef(i) if length == -1 { // no backref found - if d[i] == 0 { - return nil, fmt.Errorf("could not find an RLE backref at index %d", i) - } - out.WriteByte(d[i]) + c.D = append(c.D, int(d[i])) i++ continue } @@ -47,7 +43,7 @@ func Compress(d []byte, settings Settings) (c []byte, err error) { i += length } - return out.Bytes(), nil + return } type compressor struct { @@ -91,38 +87,6 @@ func (compressor *compressor) longestMostRecentBackRef(i int) (addr, length int) windowStart := utils.Max(0, minBackRefAddr) endWindow := utils.Min(i+brAddressRange, len(d)) - if d[i] == 0 { // RLE; prune the options - // we can't encode 0 as is, so we must find a backref. - - // runLen := compressor.countZeroes(i, brLengthRange) // utils.Min(getRunLength(d, i), brLengthRange) - runLen := utils.Min(compressor.longestZeroPrefix[i], brLengthRange) - - backrefAddr := -1 - backrefLen := -1 - for j := i - 1; j >= windowStart; j-- { - n := utils.Min(compressor.longestZeroPrefix[j], runLen) - if n == 0 { - continue - } - // check if we can make this backref longer - m := matchLen(d[i+n:endWindow], d[j+n:]) + n - - if m > backrefLen { - if m >= brLengthRange { - // we can stop we won't find a longer backref - return j, brLengthRange - } - backrefLen = m - backrefAddr = j - } - } - if (backrefLen == -1 && minBackRefAddr < 0) || (backrefLen != -1 && minBackRefAddr < 0 && backrefLen < -minBackRefAddr) { - backrefAddr = minBackRefAddr - backrefLen = utils.Min(runLen, -minBackRefAddr) - } - return backrefAddr, backrefLen - } - // else --> // d[i] != 0 @@ -158,16 +122,6 @@ func (compressor *compressor) longestMostRecentBackRef(i int) (addr, length int) } -func countZeroes(a []byte, maxCount int) (count int) { - for i := 0; i < len(a) && count < maxCount; i++ { - if a[i] != 0 { - break - } - count++ - } - return -} - // matchLen returns the maximum common prefix length of a and b. // a must be the shortest of the two. func matchLen(a, b []byte) (n int) { @@ -189,10 +143,10 @@ func matchLen(a, b []byte) (n int) { } -func emit(bb *bytes.Buffer, n int, nbBytes uint) { +func emit(bb *[]int, n int, nbBytes uint) { for i := uint(0); i < nbBytes; i++ { - bb.WriteByte(byte(n)) - n >>= 8 + *bb = append(*bb, n%257) + n /= 257 } if n != 0 { panic("n does not fit in nbBytes") diff --git a/std/compress/lzss_v1/compress_test.go b/std/compress/lzss_v1/compress_test.go index 5f12c5131e..6f2a1f112d 100644 --- a/std/compress/lzss_v1/compress_test.go +++ b/std/compress/lzss_v1/compress_test.go @@ -4,16 +4,13 @@ import ( "bytes" "encoding/hex" "fmt" - "io" + "github.com/stretchr/testify/assert" "os" "strings" "testing" "github.com/consensys/gnark/std/compress" "github.com/consensys/gnark/std/compress/huffman" - "github.com/klauspost/compress/s2" - "github.com/klauspost/compress/zstd" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -26,37 +23,41 @@ func testCompressionRoundTrip(t *testing.T, nbBytesAddress uint, d []byte, testC d, err = os.ReadFile("../test_cases/" + testCaseName[0] + "/data.bin") require.NoError(t, err) } + const contextSize = 256 + d = append(make([]byte, contextSize), d...) settings := Settings{ BackRefSettings: BackRefSettings{ NbBytesAddress: nbBytesAddress, NbBytesLength: 1, }, + StartAt: 256, } c, err := Compress(d, settings) + require.NoError(t, err) + if len(testCaseName) == 1 { - assert.NoError(t, os.WriteFile("../test_cases/"+testCaseName[0]+"/data.lzssv1", c, 0600)) + assert.NoError(t, os.WriteFile("../test_cases/"+testCaseName[0]+"/data.lzssv1", c.Write(), 0600)) } - cStream := compress.NewStreamFromBytes(c) - cHuff := huffman.Encode(cStream) - fmt.Println("Size Compression ratio:", float64(len(d))/float64(len(c))) - fmt.Println("Estimated Compression ratio (with Huffman):", float64(8*len(d))/float64(len(cHuff.D))) - if len(c) > 1024 { - fmt.Printf("Compressed size: %dKB\n", int(float64(len(c)*100)/1024)/100) + + cHuff := huffman.Encode(c) + fmt.Println("Size Compression ratio:", float64(len(d)-contextSize)/float64(c.Len())) + fmt.Println("Estimated Compression ratio (with Huffman):", float64(8*(len(d)-contextSize))/float64(len(cHuff.D))) + if c.Len() > 1024 { + fmt.Printf("Compressed size: %dKB\n", int(float64(c.Len()*100)/1024)/100) fmt.Printf("Compressed size (with Huffman): %dKB\n", int(float64(len(cHuff.D)*100)/8192)/100) } - require.NoError(t, err) dBack, err := DecompressPureGo(c, settings) require.NoError(t, err) - if len(c) < 1024 { - printHex(c) + assert.Equal(t, len(d)-contextSize, len(dBack)) + for i := range dBack { + require.Equal(t, d[contextSize+i], dBack[i], i) } - - require.Equal(t, d, dBack) + //require.Equal(t, d[contextSize:], dBack) // store huffman code lengths - lens := huffman.GetCodeLengths(cStream) + lens := huffman.GetCodeLengths(c) var sbb strings.Builder sbb.WriteString("symbol,code-length\n") for i := range lens { @@ -119,20 +120,6 @@ func TestLongBackrefBug(t *testing.T) { testCompressionRoundTrip(t, 2, nil, "bug") } -func printHex(d []byte) { - for i := range d { - if i%32 == 0 { - fmt.Printf("\n[%d]: ", i) - } - s := fmt.Sprintf("%x", d[i]) - if len(s) == 1 { - s = "0" + s - } - fmt.Print(s) - } - fmt.Println() -} - func TestAverageBatch(t *testing.T) { assert := require.New(t) @@ -221,80 +208,31 @@ func BenchmarkAverageBatch(b *testing.B) { } type compressResult struct { - compressed []byte + compressed compress.Stream inputSize int outputSize int ratio float64 } -func decompressWithS2(data []byte) ([]byte, error) { - r := s2.NewReader(bytes.NewReader(data)) - var dst bytes.Buffer - _, err := io.Copy(&dst, r) - return dst.Bytes(), err -} - -func compressWithS2(data []byte) (compressResult, error) { - var buf bytes.Buffer - w := s2.NewWriter(&buf) - w.Write(data) - w.Close() - - res := compressResult{ - compressed: make([]byte, buf.Len()), - inputSize: len(data), - outputSize: buf.Len(), - ratio: float64(len(data)) / float64(buf.Len()), - } - copy(res.compressed, buf.Bytes()) - return res, nil -} - -func decompressWithZstd(data []byte) ([]byte, error) { - r, err := zstd.NewReader(bytes.NewReader(data)) - if err != nil { - return nil, err - } - var dst bytes.Buffer - _, err = io.Copy(&dst, r) - return dst.Bytes(), err -} - -func compressWithZstd(data []byte) (compressResult, error) { - var buf bytes.Buffer - - w, err := zstd.NewWriter(&buf) - if err != nil { - return compressResult{}, err - } - w.Write(data) - w.Close() - - res := compressResult{ - compressed: make([]byte, buf.Len()), - inputSize: len(data), - outputSize: buf.Len(), - ratio: float64(len(data)) / float64(buf.Len()), - } - copy(res.compressed, buf.Bytes()) - return res, nil -} - -func decompresslzss_v1(data []byte) ([]byte, error) { +func decompresslzss_v1(data compress.Stream) ([]byte, error) { return DecompressPureGo(data, Settings{ BackRefSettings: BackRefSettings{ NbBytesAddress: 2, NbBytesLength: 1, }, + StartAt: 256, }) } func compresslzss_v1(data []byte) (compressResult, error) { + const contextSize = 256 + data = append(make([]byte, contextSize), data...) c, err := Compress(data, Settings{ BackRefSettings: BackRefSettings{ NbBytesAddress: 2, NbBytesLength: 1, }, + StartAt: 256, }) if err != nil { return compressResult{}, err @@ -302,7 +240,7 @@ func compresslzss_v1(data []byte) (compressResult, error) { return compressResult{ compressed: c, inputSize: len(data), - outputSize: len(c), - ratio: float64(len(data)) / float64(len(c)), + outputSize: c.Len(), + ratio: float64(len(data)) / float64(c.Len()), }, nil } diff --git a/std/compress/lzss_v1/compression-summary.txt b/std/compress/lzss_v1/compression-summary.txt deleted file mode 100644 index 90a171bd50..0000000000 --- a/std/compress/lzss_v1/compression-summary.txt +++ /dev/null @@ -1,1202 +0,0 @@ -maxOffset: 25276, maxLen: 108 - -1:31 -80 -62:30 -5f80 -96:32 -276715503a79dccdf8b019705cd624ff37151af4b32c4ebbdc318115d13d2f1 -159:31 -3 -191:31 -60 -222:30 -1dc0 -254:30 -3ac01bbd43ef42341b859a6deb3eba223adab6bd47fdf4e7399b93fa772352da -316:28 -64bbe26f -351:31 -c0 -382:30 -1b60 -414:30 -1b80 -446:30 -1d40 -479:31 -14 -510:30 -280 -542:30 -2e0 -574:30 -340 -606:30 -6 -638:31 -8c0 -670:30 -b80 -702:30 -e40 -734:30 -ee0 -766:30 -f80 -798:30 -fe0 -830:30 -11e0 -862:30 -1240 -894:30 -12a0 -926:30 -13 -958:31 -1360 -990:30 -13c0 -1022:30 -15c0 -1054:30 -1620 -1086:30 -1780 -1118:30 -17e0 -1151:31 -362f482e78823793851468a59a3852e90edd0 -1172:1 -8252894144e9340a1e2ead7123bc9cda246cccd5acb41b872a70d2de7ca02280c0 -1247:41 -362f482e7882383585142b7efd1852e90edd0 -1268:1 -825289487c524f48f9fb7621da5589ea4946371d458869c876a94d74f43 -1299:1 -8980c0 -1342:40 -29d2f929982e7818517d67b398517d67b398323e5d9480e38291e06339d1aab483c65695d -1385:1 -4dbd5c69872386f26fc1 -1397:2 -b92642cc481e -1435:31 -60 -1459:23 -12af11f843c53d5 -1496:28 -64bc111b -1531:31 -1 -1563:31 -20 -1595:31 -60 -1653:57 -2386f26fc1 -1691:33 -1 -1723:31 -20 -1736:12 -7f72e0d8e9abf9133a92322b8b50bd8ef9dcfcb -1787:31 -80 -1850:62 -1 -1883:32 -60 -1896:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -1928:12 -26c9a6c65d9ff6cd9461e3c0d6213711b8ad4790 -1979:31 -2 -2012:32 -c0 -2046:33 -29e2f929a82e781b8512f41c138512f41c1383282449480e38291e06339d1aab483c65695d -2089:1 -4dbd5c69887a1fe16277 -2102:2 -b92642cc481e -2140:31 -60 -2164:23 -376934801145369ad8 -2201:28 -64bc113e -2236:31 -1 -2268:31 -20 -2300:31 -60 -2357:56 -7a1fe16277 -2396:33 -1 -2428:31 -20 -2441:12 -7f72e0d8e9abf9133a92322b8b50bd8ef9dcfcb -2492:31 -80 -2555:62 -1 -2588:32 -60 -2601:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -2633:12 -a67ad761a85ef670a7167ec1fd2ae34415c2d088 -2684:31 -2 -2717:32 -c0 -2750:32 -29b2f929782e78384f3c223b84f3c22478323e519480e38291e06339d1aab483c65695d -2791:1 -4dbd5c69871ff973cafa80 -2803:1 -b92642cc481e -2841:31 -60 -2866:24 -e6efc3 -2870:1 -1aabc86d -2902:28 -64bc113f -2937:31 -1 -2969:31 -20 -3001:31 -60 -3059:57 -1ff973cafa80 -3097:32 -1 -3129:31 -20 -3142:12 -7f72e0d8e9abf9133a92322b8b50bd8ef9dcfcb -3193:31 -80 -3256:62 -1 -3289:32 -60 -3302:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -3334:12 -c34750c357be1bb5bcf826dd0faad78729606 -3385:31 -2 -3418:32 -c0 -3454:35 -29b2f929782e78484f3c223b84f3c22478323e519480e38291e06339d1aab483c65695d -3495:1 -4dbd5c69871ff973cafa80 -3507:1 -b92642cc481e -3545:31 -60 -3570:24 -e6efc3 -3574:1 -1aabc86d -3606:28 -64bc1145 -3641:31 -1 -3673:31 -20 -3705:31 -60 -3763:57 -1ff973cafa80 -3801:32 -1 -3833:31 -20 -3846:12 -7f72e0d8e9abf9133a92322b8b50bd8ef9dcfcb -3897:31 -80 -3960:62 -1 -3993:32 -60 -4006:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -4038:12 -c34750c357be1bb5bcf826dd0faad78729606 -4089:31 -2 -4122:32 -c0 -4159:36 -712f86e82e781d84f0cc32f084f0cc32fa82b6f947d43aabc515c35614549227cee54b608342c0ad80b84495ea7b3 -4220:12 -80e38291e06339d1aab483c65695d -4236:1 -4dbd5c69ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffc0 -4319:46 -712f86e82e781d84f0cc32f084f0cc32fa82c163947d43aabc515c35614549227cee54b608342c0ad80b84495ea7b3 -4380:12 -438670d41d5118 -4388:1 -3b2f42cc466fbadd760dbf4 -4423:23 -153dc0aa5c5568c0 -4479:46 -2f2ed82e78184f0913c184f0913cb82b22494cce9d3f392c135dc38b147ca73ec496f7f89d938084183ff085c0 -4574:47 -1d42f91d082e783984f0913c184f0913cb837208594438670d41d5118 -4607:1 -3b2f42cc466fbadd760dbf480b91a4ea54632 -4639:12 -7d43aabc515c35614549227cee54b608342c0ad -4671:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -4721:30 -12cfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed784fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffedb44fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed748fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffedb8 -4874:23 -198785810cf59e3c1 -4907:24 -3a198fc4263b497 -4938:23 -16c93cb0a07f863b52 -4971:24 -341a2cbe9298b -4991:12 -f257391ef5237a4365189115a7ac6c456ec74452 -5039:28 -64bbe77c0 -5087:43 -362f482e788084f0913c184f0913cb8318f7794a02573c4ad15c16b48f1842aac9c9ea405b65a386d12fc4c60 -5136:1 -841249c58bc0 -5183:41 -372f582e783284f0913c184f0913cb8313078949e66eba12b77fc75cd87b5e60141b85573bc8e8871353a6b3940 -5233:1 -841249c58bc0 -5279:40 -322f082e78184f0913c184f0913cb8252894a54b37a38e111601eec80a1e5ae3d872be2481e875fec5b60ef80 -5328:1 -80c0 -5375:45 -332f182e78184f0913c184f0913cb8252894e4edb277e41dc89ab076a1f049f4a3efa7 -5414:1 -bce88811b81ad5d13a33980c0 -5471:44 -332f182e78f84f0913c184f0913cb827bcc945c58bc6ee73a1164271a246f7eac9563cb3f1c8088646174613a2c3232c0 -5566:43 -1d42f91d082e78484f0913c184f0913cb8379c6f94438670d41d5118 -5599:1 -3b2f42cc466fbadd760dbf480b91a4ea54632 -5631:12 -7d43aabc515c35614549227cee54b608342c0ad -5663:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -5713:30 -12cfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed784fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffedb44fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed748fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffedb8 -5866:23 -139b45e913c9fc4 -5900:25 -24ed47143ddde -5931:24 -fe2b34d2757a4f3 -5964:25 -242aa94b2ade0 -5983:12 -67e4a6b21ebb2fc891afd8a972af495ef7284 -6031:28 -64bbe77c0 -6079:43 -362f482e78484ebfb50a484ebfb50ae82fee94e5d7c2a44ffddf6b295a15c148167daaaf5cf34f874be9f28c6b8 -6128:1 -84d0e3db0c0 -6174:40 -1342f913082e78884ebfb50a484ebfb50ae832b617943228d25a9649a07a44d39916b6ea7b765d61f480b91418cbafe5 -6250:23 -3e32760e978657270 -6284:25 -827d64553cc0c -6322:31 -a0 -6335:12 -7fc99522b9af80a0b2493a81583a9cf089c89f5a -6383:28 -64bbe712 -6418:31 -2 -6431:12 -7d43aabc515c35614549227cee54b608342c0ad -6463:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34fc0 -6527:43 -362f482e78384ebfb50a484ebfb50ae8319df94a02573c4ad15c16b48f1842aac9c9ea405b65a386d12fc4c60 -6576:1 -841249c58bc0 -6622:40 -2742f927082e781184ebfb50a484ebfb50ae832c01d94272e156df8da513c69cb41cc7a99185d53f926bb80b9244ac9650d8 -6706:31 -20 -6738:31 -2 -6770:31 -40 -6801:30 -180 -6833:30 -14a8c9ed67 -6851:12 -7d43aabc515c35614549227cee54b608342c0ad -6883:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -6933:30 -12c -6995:60 -64bbe713 -7022:23 -4423aede9d4954f7c -7056:25 -93dc9aa103371 -7154:91 -44bac37ef7 -7184:25 -93dc9aa103371 -7203:12 -4829a274fa3a6bbb32cd99d41beed68ba8dee728 -7251:28 -c0 -7326:74 -19080c67432656d59144ceff962e8faf8926599bcf8e4edb277e41dc89ab076a1f049f4a3efa7 -7366:1 -bce826c9a6c65d9ff6cd9461e3c0d6213711b8ad4790a67ad761a85ef670a7167ec1fd2ae34415c2d088c34750c357be1bb5bcf826dd0faad78729606c34750c357be1bb5bcf826dd0faad78729606666371f73e4dec63770f424502f8a5f57e84926fc8a59c7a2e049d694a93eb0dac4131f15d31f52e541c0c280419866ba41ce5f728f58d65256a4f257391ef5237a4365189115a7ac6c456ec74452da64ca9e3bef48859d949732110ea055ea64a4b4ada11cbd41e854a38a7b99a9fb96ce2edecf7da54b37a38e111601eec80a1e5ae3d872be2481eb543c22393ea4667e87ff9a751849def6cbe95970a20771bb0f9905ebb6f9742d7186eebec0b167e4a6b21ebb2fc891afd8a972af495ef72848a9c2147dbb5b8dc81985c1f698e06c996e51a07fc99522b9af80a0b2493a81583a9cf089c89f5aaa5e2979df2873c7923a153f7225798ce785ea864829a274fa3a6bbb32cd99d41beed68ba8dee728 -7776:48 -143d6e14c2162685ad64eae1b835e828bdbe6e9bf978e2fe99a1f7cd293b -7836:28 -64bbe27b -7871:31 -c0 -7902:30 -1b20 -7934:30 -1b40 -7966:30 -1ce0 -7999:31 -13 -8030:30 -260 -8062:30 -2c0 -8094:30 -320 -8126:30 -380 -8158:30 -3e0 -8190:30 -440 -8222:30 -6a0 -8254:30 -7 -8286:31 -9c0 -8318:30 -a60 -8350:30 -ac0 -8382:30 -d80 -8414:30 -fe0 -8446:30 -1040 -8478:30 -1160 -8510:30 -12 -8542:31 -1260 -8574:30 -13 -8606:31 -17a0 -8639:31 -30ef821d385254be4 -8650:1 -82528942bf08f2e7bf56a5aa496ccc0d71a1d86e4b291a871ff973cafa80 -8682:1 -8082e788080 -8735:47 -362f482e7882379485142b7efd1852e90edd0 -8756:1 -8252894291ccfa3402d244ebf784d7bddb787567a1d2888711c37937e0807e80c0 -8831:41 -362f482e7882383685142b7efd1852e90edd0 -8852:1 -8252894c0d8cbc733c967d9538d0b237159535dd4f56f871e931c84efe0e80c0 -8927:41 -372f582e7882379585142b7efd1852e90edd0 -8948:1 -8252894a87bedb589a397bcb904711b43ebc67627378b884db73254763 -8980:1 -6980c0 -9023:40 -362f482e7882383785142b7efd1852e90edd0 -9044:1 -8252894254e937b99afe27f9bac697d7e65fa75294950278747de4df82 -9075:1 -3980c0 -9118:40 -232f922f82187985129e4dd668348d9c94cb566e3b6934fa77258d68ea18e931fa75e1aaaa80b9246c459a28 -9177:12 -38de71124f7a447a1d67945a51edce9ff491251 -9228:31 -80 -9257:28 -64bc36cf -9291:30 -140 -9324:31 -84704316e5 -9360:31 -6d935956e138e787d7604fd9224e404bd2a9ecf1e0c82171443d686edf4969fe60 -9423:30 -2 -9425:1 -935956e138e787d7604fd9224e404bd2a9ecf1e0c82171443d686edf4969fe60 -9516:59 -82f795b3c728926c48508262f67def1871d8c01c57b8dac41e59f18f43d86ced15c213469fc52b477bdea12b2c826defe77ec175563f330de214beb83facf1c45eef5af219721f23421c3f9129fc3f91343170744eda981f32791cb167d735cf1f58fe1b6d42beb6d527b2b0a0ece2a67e765696e15c2194d7cc53460cd1b -9677:30 -82e788080 -9727:45 -362f482e788084fb2f58ac84fb2f58ac82afee94e5d7c2a44ffddf6b295a15c148167daaaf5cf34f872f9b3a9ffd80 -9776:1 -84d0e3db0c0 -9822:40 -29b2f929782e788084f83eb88084f83eb88083282449480e38291e06339d1aab483c65695d -9863:1 -4dbd5c6987ae5d0c38375ab92642cc481e -9913:31 -60 -9938:24 -5d1d3272f9ba4fb -9974:28 -64bc1151 -10009:31 -1 -10041:31 -20 -10073:31 -60 -10131:57 -ae5d0c38375a -10169:31 -1 -10201:31 -20 -10214:12 -dcf3c78a6bf862285f579d5672d353b4bf5f973b -10265:31 -80 -10328:62 -1 -10361:32 -60 -10374:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -10406:12 -cb1d4e13a6b8adffa2195cd04bd3ac7b68520 -10457:31 -2 -10490:32 -c0 -10527:36 -712f86e82e78e84f0913c184f0913cb82b751949dd6ea6f9d1fba5ed640651f6802e32ff45522180b84495ea7b3 -10588:12 -272e156df8da513c69cb41cc7a99185d53f926bb -10631:23 -134163e611dac -10640:2 -c0 -10687:46 -332f182e781084f0913c184f0913cb827bcc945c58bc6ee73a1164271a246f7eac9563cb3f1c8088646174613a2c3333c0 -10782:43 -29b2f929782e78584ee6a1cb184ee6a1cbd8323e519480e38291e06339d1aab483c65695d -10823:1 -4dbd5c69871ff973cafa80 -10835:1 -b92642cc481e -10873:31 -60 -10898:24 -e6efc3 -10902:1 -1aabc86d -10934:28 -64bc114a -10969:31 -1 -11001:31 -20 -11033:31 -60 -11091:57 -1ff973cafa80 -11129:32 -1 -11161:31 -20 -11174:12 -7f72e0d8e9abf9133a92322b8b50bd8ef9dcfcb -11225:31 -80 -11288:62 -1 -11321:32 -60 -11334:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -11366:12 -c34750c357be1bb5bcf826dd0faad78729606 -11417:31 -2 -11450:32 -c0 -11486:35 -23b2f923782e788084ee6a1cb184ee6a1cbd8335af894272e156df8da513c69cb41cc7a99185d53f926bb873cca6ec8fd411ab924ac9650d8 -11577:31 -20 -11609:31 -2 -11641:31 -40 -11672:30 -180 -11704:30 -14bfba6b22 -11722:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -11754:12 -265b25e22bcd7f10a5bd6e6410f1537cc7567e8 -11804:30 -3e8 -11818:12 -5b502e93a54e9911824df927287cf59fa886bed1 -11866:28 -64bbe77 -11893:23 -238fd42c5cf4 -11927:27 -3cca6ec8fd411a -12025:91 -41faa4133 -12058:28 -c0 -12095:36 -372f582e78584ee6a1cb184ee6a1cbd831307294db3bb6d5a8eeeafc64c66c1769 -12131:1 -e6b82b23dd5f871c6bf526340 -12145:1 -841249c58bc0 -12191:40 -f22f8ef82e78a84ebfb50a484ebfb50ae83346e894c66149996d263c0b42d3bc05e50db8865816ce80b8c42751cec -12253:12 -7d43aabc515c35614549227cee54b608342c0ad -12297:24 -5e3e379c7b646d8 -12329:24 -fbd15b4e82f06c30 -12362:25 -21dfe9dd5a859c -12381:12 -9765c4ebe7fc81624ad25e8d9c1f7718faf7d2a -12429:28 -64bbe71dc0 -12479:45 -722f86f82e782b84e992df7d84e992df87831419e94 -12503:1 -dfeb5761f17ae83e9d3249d23f9eab355f1f980b8449e2c8a5b -12560:31 -1 -12585:24 -c133757b1140 -12593:1 -c0 -12639:45 -322f082e78184e992df7d84e992df87825289480c67432656d59144ceff962e8faf8926599bcf88716296d8e1c33680c0 -12735:45 -712f86e82e782084e992df7d84e992df8782b4fb94265b25e22bcd7f10a5bd6e6410f1537cc7567e880b84495ea7b3 -12796:12 -438670d41d5118 -12804:1 -3b2f42cc466fbadd760dbf4 -12839:23 -29c627fbad7443d8fc0 -12894:45 -47b2f947782e78384e992df7d84e992df87832e3a3941a7b46c660603ebb5fbe3ae51e80ad21df -12937:1 -bdd187e87d55521a0 -12947:1 -b9444a71c9b7f -12985:31 -c0 -13016:30 -3c0 -13049:31 -1b749da0d4f1ec9feb573e54786d6cb278747b125929f56bb28a0fbef35cd60eb37ea7a31c9c34536187ae8826fac244db95f5e6b6b4d9813a4db7f51b70fc -13142:28 -64bbe14e -13158:12 -8e616d5ac7dcbe596d7ea039a71ab7885023b59b -13208:30 -1a0 -13318:108 -8e616d5ac7dcbe596d7ea039a71ab7885023b59b -13363:25 -cb6daaa7d6c -13400:31 -240 -13433:31 -2 -13494:60 -653275d1 -13510:12 -cb8096628d9538ef7d17b6d792376cd4bb42366 -13561:31 -1 -13593:31 -1 -13625:31 -2 -13638:12 -b62c414abf83c0107db84f8de1c88631c05a8d7b -13688:30 -268b -13721:31 -1 -13753:31 -1 -13862:108 -955af4de9ca03f84c9462457d075acabf1a8afc8 -13907:25 -173fbbbb690 -13945:32 -41fd8ad16691debd19173e4981fd8c1c1496da2ba15b91c49ca5c8244cef48afb999589fefaf967ffb7bca62465597354dcc6f027c86ca62046ff95e65aff1c -14042:31 -c0 -14078:35 -2742f927082e78c84e992df7d84e992df87835bff9942269bceb3f4eaa53d2fc43b1b7c5c5d13b119a580b9244638860eb -14161:30 -1a0 -14193:30 -1e0 -14221:26 -9184e72a0 -14227:1 -93eabea92acf10493df398466db61ac2bb6495a40be16cfb325478ec84b71 -14321:62 -220 -14386:63 -c8 -14417:30 -2ee -14483:64 -e9a770f2f8bd4df198f3f74431bca81efa9292ed26a4dacaab1e9b433c178d1373231 -14578:60 -a636f6c6c656374696f6e -14642:53 -7636f6c6c656374 -14707:57 -c0 -14782:74 -17c2fc617e933a52713247ce25730f6695920b3befe80c67432656d59144ceff962e8faf8926599bcf8e4edb277e41dc89ab076a1f049f4a3efa7 -14842:1 -bce880c67432656d59144ceff962e8faf8926599bcf8e4edb277e41dc89ab076a1f049f4a3efa7 -14882:1 -bce8b8ff877ed78ba520ece21b1de7843a8a57ca47cb5542916f9877d341e8c8f3664d6a1334c08b49cb1d4e13a6b8adffa2195cd04bd3ac7b685205d4d8a8f871776b1bf89daad1acdc98daea16d70a20771bb0f9905ebb6f9742d7186eebec0b1c34750c357be1bb5bcf826dd0faad787296065b502e93a54e9911824df927287cf59fa886bed16fd30ee2bc8d381eefc23d5149fc9eee268321b9765c4ebe7fc81624ad25e8d9c1f7718faf7d2aa024b25c838f3fc6d1a64a2dad2b2a533c5f597f146297486d762e91f8af3c9d29208a8ff7ec1ef7a93928d6dcfef177511dadd21bd482e12605d1cd3671833e3a6d0a82ed886c225101c6419c8b5ea7d6fa9265b6c4de847df3ac6f39d59bf91af2d -15200:36 -155e82bbc64a6925a0d656f80bcb65b45cf62cda6641f4b18b2b5b5ee158c16 -15260:28 -64bbe287 -15295:31 -c0 -15326:30 -21c0 -15358:30 -21e0 -15390:30 -24 -15423:32 -19 -15454:30 -320 -15486:30 -380 -15518:30 -3e0 -15550:30 -5e0 -15582:30 -8a0 -15614:30 -b60 -15646:30 -cc0 -15678:30 -d60 -15710:30 -ec0 -15742:30 -f60 -15774:30 -1220 -15806:30 -1280 -15838:30 -13 -15870:31 -14a0 -15902:30 -16 -15934:31 -17e0 -15966:30 -1940 -15998:30 -19a0 -16030:30 -1c40 -16062:30 -1ca0 -16094:30 -1d -16126:31 -1d60 -16158:30 -1dc0 -16190:30 -1e60 -16222:30 -2060 -16255:31 -362f482e7882383885142b7efd1852e90edd0 -16276:1 -82528948ee76dd941d643fc1ad92c11e555f9a14fb11c18715bf45eab901480c0 -16351:41 -362f482e7882379685142b7efd1852e90edd0 -16372:1 -8252894ae77f212b35cf515a4959fccb03af1e287d7728758d15e1762802680c0 -16446:40 -1d8f91d582171685129e4dd6683123ff494a658742d33ebd2ce2fbdff73515aa797fd161d9869184e72a0 -16492:1 -b91a458941e -16530:31 -6d -16543:12 -db3bb6d5a8eeeafc64c66c1769 -16557:1 -e6b82b23dd5f -16592:29 -3d090935956e138e787d7604fd9224e404bd2a9ecf1e0c82171443d686edf4969fe60935956e138e787d7604fd9224e404bd2a9ecf1e0c82171443d686edf4969fe60 -16690:31 -e0 -16703:12 -4a2556dab7af7ef6d27333de8eb9f243dc29aec -16754:31 -94 -16767:12 -4d73adb72bc3dd368966edd0f0b214841a178e2 -16793:6 -18f -16796:1 -6da184998ec58dc1da77a1f9f1e361541257a5cf4 -16818:1 -b7db3bb6d5a8eeeafc64c66c1769 -16833:1 -e6b82b23dd5f -16851:12 -4a2556dab7af7ef6d27333de8eb9f243dc29aec -16915:44 -82e788080 -16958:38 -2952f929182e78818884f83eb88084f83eb88083250799480e38291e06339d1aab483c65695d -17000:1 -4dbd5c6980b92642cc481e -17043:31 -60 -17068:24 -debd7e3294e4ed2 -17104:28 -64bc1160 -17139:31 -1 -17171:31 -20 -17203:31 -60 -17216:12 -7d43aabc515c35614549227cee54b608342c0ad -17259:23 -662fa148d78b54 -17299:33 -1 -17331:31 -20 -17344:12 -7f72e0d8e9abf9133a92322b8b50bd8ef9dcfcb -17395:31 -80 -17458:62 -1 -17491:32 -60 -17504:12 -7d43aabc515c35614549227cee54b608342c0ad -17536:12 -4f45eceb38221b338b4651cc1cdf51b1b61fddf4 -17587:31 -1 -17620:32 -c0 -17662:41 -29b2f929782e78784f83eb88084f83eb88083282509480e38291e06339d1aab483c65695d -17703:1 -4dbd5c6987287392cbe8b31eb92642cc481e -17753:31 -60 -17777:23 -124ef28593b3b318 -17814:28 -64bc115f -17849:31 -1 -17881:31 -20 -17913:31 -60 -17971:57 -287392cbe8b31e -18009:31 -1 -18041:31 -20 -18054:12 -7f72e0d8e9abf9133a92322b8b50bd8ef9dcfcb -18105:31 -80 -18168:62 -1 -18201:32 -60 -18214:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -18246:12 -99e5f58fbb54d8ef15943e930dd1f91afb573 -18297:31 -2 -18330:32 -c0 -18366:35 -1342f913082e786884efd986e684efd986f483544dd94169c6b4cfb09bfd73a81e6f2bb1eb514d75bb1980b914791ac947 -18446:27 -316bfffa6c -18476:25 -126724cd2c8a30 -18514:31 -a0 -18527:12 -76d5ad6bc939b122a94ad0ca474165b2a0ebb9d -18575:28 -64bbe977 -18610:31 -2 -18623:12 -945a6fd19f633c991cb8fcb2b7f847c48142343 -18655:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34fc0 -18719:43 -712f86e82e78184ebfb50a484ebfb50ae82b4ef947d43aabc515c35614549227cee54b608342c0ad80b84495ea7b3 -18780:12 -80e38291e06339d1aab483c65695d -18796:1 -4dbd5c69 -18824:24 -70d6595bd9985c82c0 -18878:45 -13b2f913782e78c84e992df7d84e992df87833327d94272e156df8da513c69cb41cc7a99185d53f926bb87bc4b381d1880 -18931:1 -b914a8c9ed67 -18950:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -18982:12 -265b25e22bcd7f10a5bd6e6410f1537cc7567e8 -19032:30 -3e8 -19046:12 -6f4aa5c4f30142251d0e8cf347298d09c489c3f -19094:28 -64bbe71f -19123:25 -bc4b381d1880 -19153:24 -6e25c696415ab686b -19194:32 -c0 -19231:36 -722f86f82e784384e992df7d84e992df87831a2dc94 -19255:1 -dfeb5761f17ae83e9d3249d23f9eab355f1f980b8449e2c8a5b -19312:31 -b2 -19336:23 -185c96ca35edfc0 -19345:1 -c0 -19390:44 -29b2f929782e788084e992df7d84e992df8783282449480e38291e06339d1aab483c65695d -19431:1 -4dbd5c69871e13cf4b1dc3b92642cc481e -19481:31 -60 -19506:24 -77753ffc3a363b6 -19542:28 -64bc114d -19577:31 -1 -19609:31 -20 -19641:31 -60 -19699:57 -1e13cf4b1dc3 -19737:31 -1 -19769:31 -20 -19782:12 -7f72e0d8e9abf9133a92322b8b50bd8ef9dcfcb -19833:31 -80 -19896:62 -1 -19929:32 -60 -19942:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -19974:12 -845cb942c143c0e9c4c067145ad1501175282b -20025:31 -2 -20058:32 -c0 -20095:36 -372f582e78284e992df7d84e992df87831307294db3bb6d5a8eeeafc64c66c1769 -20131:1 -e6b82b23dd5f871c6bf526340 -20145:1 -841249c58bc0 -20191:40 -572f85482e788084e992df7d84e992df87833d09094eba2e9bd66dbce1b93a407ef5431c234cbc7ae86763bfbd220 -20241:1 -a41e83409a -20258:12 -3020fd3ce1dffb6d1759a7941fdfccb892ba621ec0 -20318:39 -17b2f917782e781284e992df7d84e992df87833eb84945f45cd59ba7f2f6bcd935663f68ee1debe3b8a10871708c59ebc1e6b914451905636 -20390:12 -18d6f8edfdc989a1bfe47215a09fdbbcc0e73c -20441:31 -9e -20473:31 -e0 -20498:24 -4563918244f4 -20600:96 -120 -20633:31 -1418d6f8edfdc989a1bfe47215a09fdbbcc0e73c -20698:44 -c0 -20734:35 -13c2f913882e78b84e992df7d84e992df878332dca94272e156df8da513c69cb41cc7a99185d53f926bb883311fc8a57 -20788:2 -b914a8c9ed67 -20807:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -20839:12 -7d43aabc515c35614549227cee54b608342c0ad -20889:30 -12c -20903:12 -c3be35f23cfdf3e718852cd6d8c356f64c3e2d2 -20951:28 -64bbe71f -20979:24 -3311fc8a57 -21010:25 -1749de9f6b8caa796 -21051:32 -c0 -21086:34 -1bb2f91b782e78284e992df7d84e992df87831de91c94a02573c4ad15c16b48f1842aac9c9ea405b65a38718cfc3f65e4 -21139:1 -b918451905636 -21158:12 -72ed5e9b2774fc59b4121b12b162831b4512b17 -21209:31 -6a -21241:31 -e0 -21271:29 -3d1577 -21286:12 -72ed5e9b2774fc59b4121b12b162831b4512b17 -21368:62 -120 -21401:31 -1472ed5e9b2774fc59b4121b12b162831b4512b17 -21465:43 -22 -21467:1 -1 -21497:29 -3d40 -21530:30 -c0 -21566:35 -13b2f913782e78284e992df7d84e992df87832d17094272e156df8da513c69cb41cc7a99185d53f926bb8718de76816d80 -21619:1 -b914a8c9ed67 -21638:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -21670:12 -7d43aabc515c35614549227cee54b608342c0ad -21720:30 -12c -21734:12 -e59c5287f771cd7b9e17fc77c34b1bb2c5ab55a -21782:28 -64bbe713 -21811:25 -18de76816d80 -21842:25 -b5732e23baa2ac84 -21882:32 -c0 -21919:36 -332f182e78484e992df7d84e992df878252894e4edb277e41dc89ab076a1f049f4a3efa7 -21958:1 -bce88832e23ce4cbaa33680c0 -22014:43 -2742f927082e78a84e992df7d84e992df87832ec5e94272e156df8da513c69cb41cc7a99185d53f926bb80b9244ac9650d8 -22098:31 -20 -22130:31 -2 -22162:31 -40 -22193:30 -180 -22225:30 -14a8c9ed67 -22243:12 -f5c682501528cdfd0b5693f9f8b5a2233476f5 -22275:12 -e5d7c2a44ffddf6b295a15c148167daaaf5cf34f -22325:30 -3e8 -22387:60 -64bbe713 -22415:24 -18e32191d5afd2f -22448:25 -338b644a4bb735 -22546:91 -44bac37ef7 -22576:25 -338b644a4bb735 -22595:12 -a2ac34bf876f279fef583aa1a5459e90ea974215 -22643:28 -c0 -22687:43 -372f582e78184e992df7d84e992df87831307294db3bb6d5a8eeeafc64c66c1769 -22723:1 -e6b82b23dd5f871c6bf526340 -22737:1 -841249c58bc0 -22783:40 -332f182e78984e992df7d84e992df87825289480c67432656d59144ceff962e8faf8926599bcf8881e804608dd33680c0 -22879:44 -372f582e78284e992df7d84e992df8782fee94e5d7c2a44ffddf6b295a15c148167daaaf5cf34f88dbaf8159ac82e4184d0e3db0c0 -22975:40 -332f182e781184e992df7d84e992df87827bcc945c58bc6ee73a1164271a246f7eac9563cb3f1c8088646174613a2c3434c0 -23071:44 -712f86e82e78284e992df7d84e992df8782fb429466627f389ae46d881773b7131139b2411980e09e80b84495ea7b3 -23132:12 -272e156df8da513c69cb41cc7a99185d53f926bb -23180:28 -11984a7c0 -23230:45 -1db2f91d782e78184e2598cb84e2598c12833bfda949e66eba12b77fc75cd87b5e60141b85573bc8e8872f59a9cfde7b91a451905636 -23302:12 -c697d8f15463079c5202ecf48b4e126b16979a -23353:31 -b8 -23385:31 -e0 -23414:28 -17d8123 -23430:12 -c697d8f15463079c5202ecf48b4e126b16979a -23512:62 -120 -23545:31 -14c697d8f15463079c5202ecf48b4e126b16979a -23609:43 -56 -23611:1 -2 -23641:29 -55730 -23669:25 -1e283dbb20 -23676:1 -c697d8f15463079c5202ecf48b4e126b16979a -23706:10 -c0 -23743:36 -572f85482e788084e2598cb84e2598c12833d09094eba2e9bd66dbce1b93a407ef5431c234cbc7ae86763bfbd220 -23793:1 -a41e83409a -23810:12 -87d38c798398966fdf3aada2edcfb573efb490efc0 -23902:71 -1f4e4edb277e41dc89ab076a1f049f4a3efa7 -23922:1 -bce880c67432656d59144ceff962e8faf8926599bcf8e93685f3bba03016f02bd1828badd6195988d9504f45eceb38221b338b4651cc1cdf51b1b61fddf499e5f58fbb54d8ef15943e930dd1f91afb57376d5ad6bc939b122a94ad0ca474165b2a0ebb9d7854b214321d49417e27 -24035:1 -2fff5f6a6b7651e2c6f4aa5c4f30142251d0e8cf347298d09c489c3fa19a4efc6bdaf922e4889cef1a8f1592208b76845cb942c143c0e9c4c067145ad1501175282bce22a64d8873996735a25ff069b6a9ae2d716bb93020fd3ce1dffb6d1759a7941fdfccb892ba621e18d6f8edfdc989a1bfe47215a09fdbbcc0e73cc3be35f23cfdf3e718852cd6d8c356f64c3e2d272ed5e9b2774fc59b4121b12b162831b4512b17e59c5287f771cd7b9e17fc77c34b1bb2c5ab55a60d690b144bfe7eb9218a55a302e6fa3d45a2bc5a2ac34bf876f279fef583aa1a5459e90ea974215bb21688b29825e7c48098ec22b177d5a98988a5462f728de5fe92ae811214803bfb151990b463a9233acb3f5daa927aa767937e3bd582ea4f63ba70a20771bb0f9905ebb6f9742d7186eebec0b1ee6c8492e5b0f9e61ae8662b929e78599cdcb464c697d8f15463079c5202ecf48b4e126b16979a87d38c798398966fdf3aada2edcfb573efb490ef -24478:74 -460a8d52f6a227a4d07eea365a73f5dfae692e73d168a939bd8ca17b9a76d8762fcdb53ca7552d21e1e4af12ee5f075b8784da8b8e92df233a5d7d16c23f1a5e1264 -24549:1 -87979fc7c1599d718c67e414855354742d6666f69904391f064ba287492189aa3c5691b24d17c6f6ac4afb98ef829ae2a8d9a6cb5ad2d378d25d7e45f56bd85cdb6ab8e567cd76d86e14efdd67e14e1b9fbffa2bf1d324a8154a555699ef26e84644f3fa4f33caaf7cc9ad13857db117114f5e733e722d26d6bd9e301899d7d6ee53e992bea9c25875eda9c22597ea7655626e5f7ef1b3d2461889c49d1652dfa9fb2c9ebd348fb4a324c6b1de25823614ab50e3a11650e6d395651c3af0cb1aad93d7531863bbe2212c5260ec84ef0e38d4cf621a8145f9da83a9592b7364a0ca44163bbd7c4981c74c258c182f62ee15afaab2ddbd9fad2e734a71955b8670b233aa671b23f15a0b969a2b4ba557fe1a1289e -24835:1 -134c73652cd4fa73b78c3fb156db7fb190898db439c91a90124794e023bc5ba0cd4de14f3b9fcfee7363321d68e4ef907fa5df91b1d9a7568b81125b8c725879a46c606440e5cf9c1652a6f75cd5b3adc1a3ae54ea4e8fb1595b9602441e4edc1da55fb5b67aa64f7df8b36f59e8bd78d1bcc912220bcd1558ac13a451865f53e0265cf13ca5c8d898e4b7b2695f78468eb6c6847ac136b3fafd3e40eb765c367f0fbdfd3907c212a2c68c28381ab74afc1f19dcb9d6cb9f6efde8ea1e09bf6bf39a2e8728f78bedb3bc82a201222d120981eef6518e1591d90e0433919c56a7893ad0d3cc27f78916d563cf90b8c1fea87c0b0ad8b10676068c99896169b4d48658a7ece48d3f767fff81e305817fd7a65d5ed0a3dc4e86dcf5b69b4935f1bcc7ab383d82a6c2fc4da21915 -25153:1 -499c69f6d0cc73350e3616e39f8a9a78ed73ceb310ccf052142cc67adc9e12b5102586e53acb419395353e3290be83ec278334219aa238b9cf79e0888122cdf6de6f554f8f6f78931aab1f254643a7866a72881e777ad245916fa4d9154d2872855bde1f9c38f8f4c52b8ffe9639db1b41f08fd8a630 -25276:1 -b39ccfd1fc13be853fdc0c81f9f62d2468dc04726b5bea772c3d5fb8c6c15fcb6135139fb5d5d1d7dfa9f53c5255a44adfe1f51b4affc74123e1d131821f228d91f8c89f3426740c7e2d6898b652778f917b6d0209cbe83812cfe36e0ff68878fd151a6da84c3b4444e166663cc353fa06d3b4828c547e5ccabedf8627f735713311580d77df7ee0ebff96178ca572b990679dfe54d7d97c83a85eb5825b49d47a25bbe71da0b88468b44bcf6165fdb5e9ccc84cd40671e1aee88b015cb636ced5485ec050df8cafda15f1b9d1ef9f69f1a3c3d7f2b6e4113a170d493d50d554dc343e7f26c89f1a46fc7533a39dad1f1443afa629ade8014f71381321c389aebca22366150b5db28243b24be2b6eda6a43f05476211bc67457296f3f19dca5a7672c859f73963e348114c3638f5498950b2717a9036ad7795 \ No newline at end of file diff --git a/std/compress/lzss_v1/config.go b/std/compress/lzss_v1/config.go index 97cd0b8529..9bd45f4489 100644 --- a/std/compress/lzss_v1/config.go +++ b/std/compress/lzss_v1/config.go @@ -11,4 +11,5 @@ func (s BackRefSettings) NbBytes() int { type Settings struct { BackRefSettings + StartAt uint } diff --git a/std/compress/lzss_v1/decompress.go b/std/compress/lzss_v1/decompress.go index 278094d651..26c660ddf6 100644 --- a/std/compress/lzss_v1/decompress.go +++ b/std/compress/lzss_v1/decompress.go @@ -1,53 +1,38 @@ package lzss_v1 import ( - "bytes" + "fmt" + "github.com/consensys/gnark/std/compress" ) -func DecompressPureGo(c []byte, settings Settings) (d []byte, err error) { +func DecompressPureGo(c compress.Stream, settings Settings) (d []byte, err error) { // d[i < 0] = Settings.BackRefSettings.Symbol by convention - var out bytes.Buffer - in := bytes.NewReader(c) - copyBuf := make([]byte, settings.NbBytesAddress+settings.NbBytesLength) - - outAt := func(i int) byte { - if i < 0 { - return 0 - } - return out.Bytes()[i] - } + d = make([]byte, settings.StartAt) readBackRef := func() (offset, length int) { - _, err = in.Read(copyBuf) - offset = readNum(copyBuf[:settings.NbBytesAddress]) + 1 - length = readNum(copyBuf[settings.NbBytesAddress:settings.NbBytesAddress+settings.NbBytesLength]) + 1 + offset = c.ReadNum(1, int(settings.NbBytesAddress)) + 1 + length = c.ReadNum(1+int(settings.NbBytesAddress), int(settings.NbBytesLength)) + 1 return } - s, err := in.ReadByte() - for err == nil { - if s == 0 { + for len(c.D) != 0 { + if len(d) == 325 { + fmt.Println("trouble ahead") + } + if c.D[0] == 256 { offset, length := readBackRef() if err != nil { return nil, err } - for i := 0; i < length; i++ { - out.WriteByte(outAt(out.Len() - offset)) + for ; length > 0; length-- { + d = append(d, d[len(d)-offset]) } + c.D = c.D[settings.NbBytesAddress+settings.NbBytesLength+1:] } else { - out.WriteByte(s) + d = append(d, byte(c.D[0])) + c.D = c.D[1:] } - s, err = in.ReadByte() } - return out.Bytes(), nil -} - -func readNum(bytes []byte) int { //little endian - var res int - for i := len(bytes) - 1; i >= 0; i-- { - res <<= 8 - res |= int(bytes[i]) - } - return res + return d[settings.StartAt:], nil } diff --git a/std/compress/lzss_v1/log_test.go b/std/compress/lzss_v1/log_test.go deleted file mode 100644 index 4828af0b46..0000000000 --- a/std/compress/lzss_v1/log_test.go +++ /dev/null @@ -1,172 +0,0 @@ -package lzss_v1 - -import ( - "fmt" - "github.com/stretchr/testify/assert" - "io" - "os" - "strconv" - "strings" - "testing" -) - -func TestBackrefsToCsv(t *testing.T) { - backRefsToCsv(t, "../test_cases/large/") -} - -func TestCompareCoverage(t *testing.T) { - t.SkipNow() - compareBackrefs(t, "../test_cases/large/backrefs.csv", "../test_cases/large/backrefs_new.csv") -} - -func backRefsToCsv(t *testing.T, filename string) { - in, err := os.OpenFile(filename+"data.lzssv1", os.O_RDONLY, 0600) - assert.NoError(t, err) - - out, err := os.OpenFile(filename+"backrefs_new.csv", os.O_CREATE|os.O_WRONLY, 0600) - assert.NoError(t, err) - - _, err = out.WriteString("dst,src,len,offset,content\n") - assert.NoError(t, err) - - d := make([]byte, 256) - buff := []byte{0, 0, 0} - _, err = in.Read(buff[:1]) - for err == nil { - if buff[0] == 0 { - _, err = in.Read(buff) - assert.NoError(t, err) - offs := (uint16(buff[0]) | (uint16(buff[1]) << 8)) + 1 - length := uint16(buff[2]) + 1 - - src := len(d) - int(offs) - dst := len(d) - d = appnd(d, src, int(length)) - - _, err = out.WriteString( - fmt.Sprintf("%d,%d,%d,%d,%s\n", dst-256, src-256, length, offs, toHex(d[dst:])), - ) - assert.NoError(t, err) - } else { - d = append(d, buff[0]) - } - _, err = in.Read(buff[:1]) - } - if err != io.EOF { - assert.NoError(t, err) - } - - assert.NoError(t, out.Close()) - assert.NoError(t, in.Close()) -} - -func compareBackrefs(t *testing.T, oldFilename, newFilename string) { - old := readBackrefRecords(t, oldFilename) - nw := readBackrefRecords(t, newFilename) - - oldCov := coverageList(old) - newCov := coverageList(nw) - - // coverage for new must be no less than old - for i := range oldCov { - if oldCov[i] != -1 && newCov[i] == -1 && old[oldCov[i]].length >= 4 { - j := i - 1 - for newCov[j] == -1 { - j-- - } - oldBr := old[oldCov[i]] - newBr := nw[newCov[j]] - - // if there was a neutral choice here, the non-coverage is okay - if j != newBr.dst+newBr.length && newBr.length < oldBr.length { - t.Errorf("index %d is covered by backref %v in old but not in new\n\tlast new backref is %v", i, oldBr, newBr) - } - } - } -} - -func toHex(slice []byte) string { - slice = append(slice, 1) - var sbb strings.Builder - nbZeros := 0 - for _, b := range slice { - if b == 0 { - nbZeros++ - } else { - if nbZeros > 3 { - sbb.WriteString(fmt.Sprintf("[00^%d]", nbZeros)) - nbZeros = 0 - } - for nbZeros > 0 { - sbb.WriteString("00") - nbZeros-- - } - sbb.WriteByte(toHexDigit(b >> 4)) - sbb.WriteByte(toHexDigit(b & 0xf)) - } - } - return sbb.String()[0 : len(sbb.String())-2] -} - -func toHexDigit(b byte) byte { - if b < 10 { - return b + '0' - } - return b - 10 + 'a' -} - -type backrefRecord struct { - dst, src, length, offset int - content string -} - -func readBackrefRecords(t *testing.T, filename string) []backrefRecord { - res := make([]backrefRecord, 0) - file, err := os.ReadFile(filename) - assert.NoError(t, err) - lines := strings.Split(string(file), "\n") - for i, line := range lines[1:] { - if line == "" { - continue - } - fields := strings.Split(line, ",") - if len(fields) != 5 { - fmt.Println("uh oh", i, line) - } - assert.Equal(t, 5, len(fields)) - var r backrefRecord - r.dst, err = strconv.Atoi(fields[0]) - assert.NoError(t, err) - r.src, err = strconv.Atoi(fields[1]) - assert.NoError(t, err) - r.length, err = strconv.Atoi(fields[2]) - assert.NoError(t, err) - r.offset, err = strconv.Atoi(fields[3]) - assert.NoError(t, err) - r.content = fields[4] - res = append(res, r) - } - return res -} - -func coverageList(br []backrefRecord) []int { - res := make([]int, 0) - for i, r := range br { - for len(res) < r.dst+r.length { - val := -1 - if len(res) >= r.dst { - val = i - } - res = append(res, val) - } - } - return res -} - -// not sub-slice friendly -func appnd(slice []byte, src, length int) []byte { - for i := 0; i < length; i++ { - slice = append(slice, slice[src+i]) - } - return slice -} diff --git a/std/compress/lzss_v1/lzssv1-analyze/build b/std/compress/lzss_v1/lzssv1-analyze/build new file mode 100755 index 0000000000..96e6fb68a4 --- /dev/null +++ b/std/compress/lzss_v1/lzssv1-analyze/build @@ -0,0 +1,2 @@ +go build +mv lzssv1-analyze $GOPATH/bin/lzssv1-analyze \ No newline at end of file diff --git a/std/compress/lzss_v1/lzssv1-analyze/main.go b/std/compress/lzss_v1/lzssv1-analyze/main.go new file mode 100644 index 0000000000..6fa5cbe012 --- /dev/null +++ b/std/compress/lzss_v1/lzssv1-analyze/main.go @@ -0,0 +1,100 @@ +package main + +import ( + "encoding/hex" + "flag" + "fmt" + "github.com/consensys/gnark/std/compress" + "github.com/consensys/gnark/std/compress/lzss_v1/lzssv1-analyze/require" + "os" + "strings" +) + +func main() { + inFilename := flag.String("in", "", "lzssv1 input file") + //diffIn := flag.String("diff", "", "second lzssv1 input file to diff against") + posU := flag.Uint("pos", 1<<63, "position of interest (decompressed)") + hx := flag.Bool("hx", false, "read input files in hx format") + symb := flag.Int("symb", 0, "symbol that indicates a backref (default: 0)") + nbBytesLenU := flag.Uint("len", 1, "number of bytes used to encode the length of a backref (default: 1)") + nbBytesAddrU := flag.Uint("addr", 2, "number of bytes used to encode the address of a backref (default: 2)") + ctx := flag.String("ctx", "", "address of context file (defaults to 2^len zeros)") + flag.Parse() + + nbBytesLen := int(*nbBytesLenU) + nbBytesAddr := int(*nbBytesAddrU) + pos := int(*posU) + + var c compress.Stream + if *inFilename == "" { + require.NotEqual(*inFilename, "", "TODO: Read from stdin when no file is provided") + } else { + b, err := os.ReadFile(*inFilename) + require.NoError(err) + if *hx { + b, err = hex.DecodeString(string(b)) + require.NoError(err) + } + c.Read(b) + } + + var d []byte + if *ctx == "" { + d = make([]byte, 1<<(nbBytesLen*8)) + } else { + panic("TODO: read context file") + } + + var br0, br1 backref + startAt := len(d) + cI := 0 + for br1.dst <= pos { + if c.D[cI] == *symb { // backref + length := c.ReadNum(cI+1, nbBytesLen) + 1 + offset := c.ReadNum(cI+1+nbBytesLen, nbBytesAddr) + 1 + + br0 = br1 + br1 = backref{offset, length, len(d) - startAt} + + for ; length > 0; length-- { + d = append(d, d[len(d)-offset]) + } + + cI += 1 + nbBytesLen + nbBytesAddr + } else { + d = append(d, byte(c.D[cI])) + cI++ + } + } + + printFindings(d[startAt:], pos, br0, br1) +} + +func printFindings(d []byte, pos int, br0, br1 backref) { + s0 := br0.string() + s1 := br1.string() + const lenEachSide = 44 + arrow := strings.Repeat(" ", lenEachSide) + arrow = arrow + "^" + arrow + + fmt.Print(s0) + if len(s0)%2 != 0 { + fmt.Print(" ") + } + nbL := lenEachSide/2 - (len(s0)+1)/2 + nbR := lenEachSide/2 - (len(s1)+1)/2 + fmt.Print(hex.EncodeToString(d[pos-nbL : pos+nbL+nbR])) + if len(s1)%2 != 0 { + fmt.Print(" ") + } + fmt.Println(s1) + fmt.Println(arrow) +} + +type backref struct { + offset, len, dst int +} + +func (b backref) string() string { + return fmt.Sprintf("[(%d,%d)@%d->%d]", -b.offset, b.len, b.dst-b.offset, b.dst) +} diff --git a/std/compress/lzss_v1/lzssv1-analyze/require/require.go b/std/compress/lzss_v1/lzssv1-analyze/require/require.go new file mode 100644 index 0000000000..e806e3920e --- /dev/null +++ b/std/compress/lzss_v1/lzssv1-analyze/require/require.go @@ -0,0 +1,13 @@ +package require + +func NotEqual(unexpected, actual interface{}, message string) { + if unexpected == actual { + panic(message) + } +} + +func NoError(err error) { + if err != nil { + panic(err) + } +} diff --git a/std/compress/lzss_v1/snark_test.go b/std/compress/lzss_v1/snark_test.go deleted file mode 100644 index 36c5aaded8..0000000000 --- a/std/compress/lzss_v1/snark_test.go +++ /dev/null @@ -1,313 +0,0 @@ -package lzss_v1 - -import ( - "fmt" - "github.com/consensys/gnark-crypto/ecc" - "github.com/consensys/gnark/backend/plonk" - "github.com/consensys/gnark/constraint" - "github.com/consensys/gnark/frontend" - "github.com/consensys/gnark/frontend/cs/scs" - "github.com/consensys/gnark/profile" - "github.com/consensys/gnark/std/compress" - "github.com/consensys/gnark/test" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "os" - "testing" - "time" -) - -func Test1ZeroSnark(t *testing.T) { - testCompressionRoundTripSnark(t, 1, []byte{0}) - testCompressionRoundTripSnark(t, 2, []byte{0}) -} - -func Test2ZeroSnark(t *testing.T) { - testCompressionRoundTripSnark(t, 1, []byte{0, 0}) - testCompressionRoundTripSnark(t, 2, []byte{0, 0}) -} - -func Test8ZerosSnark(t *testing.T) { - testCompressionRoundTripSnark(t, 1, []byte{0, 0, 0, 0, 0, 0, 0, 0}) - testCompressionRoundTripSnark(t, 2, []byte{0, 0, 0, 0, 0, 0, 0, 0}) -} - -func TestTwoConsecutiveBackrefsSnark(t *testing.T) { - testDecompressionSnark(t, 1, make([]byte, 6), []byte{0, 0}) -} -func Test300ZerosSnark(t *testing.T) { // probably won't happen in our calldata - testCompressionRoundTripSnark(t, 1, make([]byte, 300)) - testCompressionRoundTripSnark(t, 2, make([]byte, 300)) -} - -func TestSingleNonzeroSnark(t *testing.T) { - testCompressionRoundTripSnark(t, 1, []byte{1}) - testCompressionRoundTripSnark(t, 2, []byte{1}) -} - -func TestHiSnark(t *testing.T) { - testCompressionRoundTripSnark(t, 1, []byte{'h', 'i'}) - testCompressionRoundTripSnark(t, 2, []byte{'h', 'i'}) -} - -func TestZeroAfterNonzeroSnark(t *testing.T) { - testCompressionRoundTripSnark(t, 1, []byte{1, 0}) - testCompressionRoundTripSnark(t, 2, []byte{1, 0}) -} - -func TestTwoZerosAfterNonzeroSnark(t *testing.T) { - testCompressionRoundTripSnark(t, 1, []byte{1, 0, 0}) - testCompressionRoundTripSnark(t, 2, []byte{1, 0, 0}) -} - -func Test8ZerosAfterNonzeroSnark(t *testing.T) { - testCompressionRoundTripSnark(t, 1, append([]byte{1}, make([]byte, 8)...)) - testCompressionRoundTripSnark(t, 2, append([]byte{1}, make([]byte, 8)...)) -} - -func TestTwoBackrefsAfterNonzeroSnark(t *testing.T) { - testDecompressionSnark(t, 1, []byte{1, 0, 1, 0, 0, 0, 0}, []byte{1, 0, 0}) -} - -func Test257ZerosAfterNonzeroSnark(t *testing.T) { // probably won't happen in our calldata - testCompressionRoundTripSnark(t, 1, append([]byte{1}, make([]byte, 257)...)) - testCompressionRoundTripSnark(t, 2, append([]byte{1}, make([]byte, 257)...)) -} - -func Test300ZerosAfterNonzeroSnark(t *testing.T) { // probably won't happen in our calldata - testCompressionRoundTripSnark(t, 1, append([]byte{'h', 'i'}, make([]byte, 300)...)) - testCompressionRoundTripSnark(t, 2, append([]byte{'h', 'i'}, make([]byte, 300)...)) -} - -func TestRepeatedNonzeroSnark(t *testing.T) { - testCompressionRoundTripSnark(t, 1, []byte{'h', 'i', 'h', 'i', 'h', 'i'}) - testCompressionRoundTripSnark(t, 2, []byte{'h', 'i', 'h', 'i', 'h', 'i'}) -} - -func TestCalldataSnark(t *testing.T) { - t.Parallel() - folders := []string{ - "3c2943", - } - for _, folder := range folders { - d, err := os.ReadFile("../test_cases/" + folder + "/data.bin") - require.NoError(t, err) - t.Run(folder, func(t *testing.T) { - testCompressionRoundTripSnark(t, 2, d) - }) - } -} - -func BenchmarkCompilation64KBSnark(b *testing.B) { - c := DecompressionTestCircuit{ - C: make([]frontend.Variable, 21333), - D: make([]byte, 64000), - Settings: Settings{ - BackRefSettings: BackRefSettings{ - NbBytesAddress: 2, - NbBytesLength: 1, - }, - }, - } - - p := profile.Start() - _, err := frontend.Compile(ecc.BN254.ScalarField(), scs.NewBuilder, &c) - assert.NoError(b, err) - p.Stop() - fmt.Println(p.NbConstraints(), "constraints") -} - -func BenchmarkCompilation300KBSnark(b *testing.B) { - c := DecompressionTestCircuit{ - C: make([]frontend.Variable, 70000), - D: make([]byte, 300000), - Settings: Settings{ - BackRefSettings: BackRefSettings{ - NbBytesAddress: 2, - NbBytesLength: 1, - }, - }, - } - - testCaseName := "large" - - // compilation - fmt.Println("compilation") - p := profile.Start() - cs, err := frontend.Compile(ecc.BN254.ScalarField(), scs.NewBuilder, &c) - assert.NoError(b, err) - p.Stop() - fmt.Println("26KB:", p.NbConstraints(), "constraints, estimated", (p.NbConstraints()*300000)/26000, "constraints for 600KB at", float64(p.NbConstraints())/26000.0, "constraints per uncompressed byte") - assert.NoError(b, compress.GzWrite("../test_cases/"+testCaseName+"/300KB.cs.gz", cs)) - - // setup - fmt.Println("setup") - kzgSrs, err := test.NewKZGSRS(cs) - require.NoError(b, err) - pk, _, err := plonk.Setup(cs, kzgSrs) - require.NoError(b, err) - assert.NoError(b, compress.GzWrite("../test_cases/"+testCaseName+"/300KB.pk.gz", pk)) -} - -// TODO Change name to reflect that setup is also occurring -func compile26KBSnark(t require.TestingT, testCaseName string) { - c := DecompressionTestCircuit{ - C: make([]frontend.Variable, 7300), - D: make([]byte, 26000), - Settings: Settings{ - BackRefSettings: BackRefSettings{ - NbBytesAddress: 2, - NbBytesLength: 1, - }, - }, - } - - startTimer := func() {} - stopTimer := func() {} - if b, ok := t.(*testing.B); ok { - startTimer = func() { - b.StartTimer() - } - - stopTimer = func() { - b.StopTimer() - } - } - - // compilation - fmt.Println("compilation") - p := profile.Start() - startTimer() - cs, err := frontend.Compile(ecc.BN254.ScalarField(), scs.NewBuilder, &c) - assert.NoError(t, err) - stopTimer() - p.Stop() - fmt.Println("26KB:", p.NbConstraints(), "constraints, estimated", (p.NbConstraints()*600000)/26000, "constraints for 600KB at", float64(p.NbConstraints())/26000.0, "constraints per uncompressed byte") - assert.NoError(t, compress.GzWrite("../test_cases/"+testCaseName+"/cs.gz", cs)) - - // setup - fmt.Println("setup") - startTimer() - kzgSrs, err := test.NewKZGSRS(cs) - require.NoError(t, err) - pk, _, err := plonk.Setup(cs, kzgSrs) - require.NoError(t, err) - stopTimer() - assert.NoError(t, compress.GzWrite("../test_cases/"+testCaseName+"/pk.gz", pk)) -} - -func BenchmarkCompilation26KBSnark(b *testing.B) { - compile26KBSnark(b, "3c2943") -} - -func BenchmarkProof26KBSnark(b *testing.B) { - cs := plonk.NewCS(ecc.BN254) - pk := plonk.NewProvingKey(ecc.BN254) - - if err := compress.GzRead("../test_cases/3c2943/cs.gz", cs); err != nil { // we don't have the constraints stored. compile and try again - fmt.Println("reading constraints failed. attempting to recreate...") - compile26KBSnark(b, "3c2943") - fmt.Println("created constraints and proving key") - cs = plonk.NewCS(ecc.BN254) - assert.NoError(b, compress.GzRead("../test_cases/3c2943/cs.gz", cs)) - } - fmt.Println("constraints loaded") - assert.NoError(b, compress.GzRead("../test_cases/3c2943/pk.gz", pk)) - fmt.Println("proving key loaded") - c, err := os.ReadFile("../test_cases/3c2943/data.lzssv1") - assert.NoError(b, err) - proveDecompressionSnark(b, cs, pk, c, 7300) -} - -func BenchmarkCompilation600KBSnark(b *testing.B) { - c := DecompressionTestCircuit{ - C: make([]frontend.Variable, 120000), - D: make([]byte, 612000), - Settings: Settings{ - BackRefSettings: BackRefSettings{ - NbBytesAddress: 2, - NbBytesLength: 1, - }, - }, - } - - p := profile.Start() - _, err := frontend.Compile(ecc.BN254.ScalarField(), scs.NewBuilder, &c) - assert.NoError(b, err) - p.Stop() - fmt.Println(p.NbConstraints(), "constraints") -} - -func testCompressionRoundTripSnark(t *testing.T, nbBytesOffset uint, d []byte) { - settings := Settings{ - BackRefSettings: BackRefSettings{ - NbBytesAddress: nbBytesOffset, - NbBytesLength: 1, - }, - } - - c, err := Compress(d, settings) - require.NoError(t, err) - testDecompressionSnark(t, nbBytesOffset, c, d) -} - -func testDecompressionSnark(t *testing.T, nbBytesOffset uint, c []byte, d []byte) { - settings := Settings{ - BackRefSettings: BackRefSettings{ - NbBytesAddress: nbBytesOffset, - NbBytesLength: 1, - }, - } - - cMax := len(c) * 3 - - decompressor := &DecompressionTestCircuit{ - C: make([]frontend.Variable, cMax), - D: d, - Settings: settings, - CheckCorrectness: true, - } - //p := profile.Start() - cs, err := frontend.Compile(ecc.BN254.ScalarField(), scs.NewBuilder, decompressor) - //p.Stop() - require.NoError(t, err) - - kzgSrs, err := test.NewKZGSRS(cs) - require.NoError(t, err) - pk, _, err := plonk.Setup(cs, kzgSrs) - require.NoError(t, err) - - proveDecompressionSnark(t, cs, pk, c, cMax) -} - -func proveDecompressionSnark(t require.TestingT, cs constraint.ConstraintSystem, pk plonk.ProvingKey, c []byte, cMax int) { - - cVars := make([]frontend.Variable, cMax) - for i := range c { - cVars[i] = frontend.Variable(c[i]) - } - - for i := len(c); i < len(cVars); i++ { - cVars[i] = 0 - } - - var start int64 - restartTimer := func() { - if start != 0 { - fmt.Println("time taken:", time.Now().UnixMilli()-start, "ms") - } - start = time.Now().UnixMilli() - } - - fmt.Println("constructing witness") - _witness, err := frontend.NewWitness(&DecompressionTestCircuit{ - C: cVars, - CLength: len(c), - }, ecc.BN254.ScalarField()) - require.NoError(t, err) - restartTimer() - fmt.Println("proving") - _, err = plonk.Prove(cs, pk, _witness) - require.NoError(t, err) - restartTimer() -} diff --git a/std/compress/pipeline.go b/std/compress/pipeline.go index 274fcfcc8f..254ca0fe34 100644 --- a/std/compress/pipeline.go +++ b/std/compress/pipeline.go @@ -8,11 +8,11 @@ type Stream struct { NbSymbs int } -func (s Stream) Len() int { +func (s *Stream) Len() int { return len(s.D) } -func (s Stream) RunLen(i int) int { +func (s *Stream) RunLen(i int) int { runLen := 1 for i+runLen < len(s.D) && s.D[i+runLen] == 0 { runLen++ @@ -20,7 +20,7 @@ func (s Stream) RunLen(i int) int { return runLen } -func (s Stream) At(i int) int { +func (s *Stream) At(i int) int { return s.D[i] } @@ -32,6 +32,42 @@ func NewStreamFromBytes(in []byte) Stream { return Stream{d, 256} } +func (s *Stream) Read(in []byte) { + s.D = make([]int, (len(in)-1)/int(in[0])) + for i := range s.D { + s.D[i] = ReadNum(in[1+i*int(in[0]) : 1+(i+1)*int(in[0])]) + } + s.NbSymbs = s.D[0] + s.D = s.D[1:] +} + +func (s *Stream) Write() []byte { + bytePerSymb := 0 + remainder := s.NbSymbs + for remainder > 0 { + bytePerSymb++ + remainder >>= 8 + } + res := make([]byte, 1+bytePerSymb*(len(s.D)+1)) + res[0] = byte(bytePerSymb) + + WriteNum(s.NbSymbs, res[1:1+bytePerSymb]) + for i := range s.D { + WriteNum(s.D[i], res[1+bytePerSymb*(i+1):1+bytePerSymb*(i+2)]) + } + + return res +} + +func (s *Stream) ReadNum(start, length int) int { + res := 0 + for i := start + length - 1; i >= start; i-- { + res *= s.NbSymbs + res += s.D[i] + } + return res +} + type Pipeline []func(Stream) Stream func (pipeline Pipeline) Run(in Stream) Stream { @@ -40,3 +76,22 @@ func (pipeline Pipeline) Run(in Stream) Stream { } return in } + +func ReadNum(bytes []byte) int { //little endian + var res int + for i := len(bytes) - 1; i >= 0; i-- { + res <<= 8 + res |= int(bytes[i]) + } + return res +} + +func WriteNum(n int, bytes []byte) { + for i := range bytes { + bytes[i] = byte(n) + n >>= 8 + } + if n != 0 { + panic("didn't fit") + } +} diff --git a/std/compress/test_cases/large/neg-table.bin b/std/compress/test_cases/large/neg-table.bin new file mode 100644 index 0000000000..78176872be Binary files /dev/null and b/std/compress/test_cases/large/neg-table.bin differ