diff --git a/tools/rsync/algorithm.go b/tools/rsync/algorithm.go index 3f21a6509..51b856e2a 100644 --- a/tools/rsync/algorithm.go +++ b/tools/rsync/algorithm.go @@ -16,8 +16,6 @@ import ( "fmt" "hash" "io" - - "kitty/tools/utils" ) // If no BlockSize is specified in the RSync instance, this value is used. @@ -47,14 +45,14 @@ type Operation struct { var bin = binary.LittleEndian -func (self Operation) Serialize() string { +func (self Operation) Serialize() []byte { ans := make([]byte, 24+len(self.Data)) bin.PutUint32(ans, uint32(len(self.Data))) bin.PutUint32(ans[4:], uint32(self.Type)) bin.PutUint64(ans[8:], self.BlockIndex) bin.PutUint64(ans[16:], self.BlockIndexEnd) copy(ans[24:], self.Data) - return utils.UnsafeBytesToString(ans) + return ans } func (self *Operation) Unserialize(data []byte) (n int, err error) { @@ -74,9 +72,9 @@ func (self *Operation) Unserialize(data []byte) (n int, err error) { } self.BlockIndex = bin.Uint64(data[8:]) self.BlockIndexEnd = bin.Uint64(data[16:]) - self.Data = data[24 : 24+dlen] n = 24 + dlen - return + self.Data = data[24:n] + return n, nil } // Signature hash item generated from target. diff --git a/tools/rsync/api.go b/tools/rsync/api.go index 3ece38072..7dbf432b8 100644 --- a/tools/rsync/api.go +++ b/tools/rsync/api.go @@ -136,18 +136,12 @@ func (self *Patcher) StartDelta(delta_output io.Writer, delta_input io.ReadSeeke // Apply a chunk of delta data func (self *Patcher) UpdateDelta(data []byte) (err error) { - if len(self.unconsumed_delta_data) > 0 { - data = append(self.unconsumed_delta_data, data...) - self.unconsumed_delta_data = nil - } - consumed, err := self.update_delta(data) + self.unconsumed_delta_data = append(self.unconsumed_delta_data, data...) + consumed, err := self.update_delta(self.unconsumed_delta_data) if err != nil { return err } - data = data[consumed:] - if len(data) > 0 { - self.unconsumed_delta_data = data - } + self.unconsumed_delta_data = utils.ShiftLeft(self.unconsumed_delta_data, consumed) return } @@ -189,7 +183,7 @@ func (self *Patcher) CreateSignature(src io.Reader, callback func([]byte) error) } // Create a serialized delta based on the previously loaded signature -func (self *Differ) CreateDelta(src io.Reader, output_callback func(string) error) (err error) { +func (self *Differ) CreateDelta(src io.Reader, output_callback func([]byte) error) (err error) { if err = self.finish_signature_data(); err != nil { return } @@ -204,26 +198,19 @@ func (self *Differ) CreateDelta(src io.Reader, output_callback func(string) erro // Add more external signature data func (self *Differ) AddSignatureData(data []byte) (err error) { - if len(self.unconsumed_signature_data) > 0 { - data = append(self.unconsumed_signature_data, data...) - self.unconsumed_signature_data = nil - } + self.unconsumed_signature_data = append(self.unconsumed_signature_data, data...) if self.rsync.UniqueHasher == nil { - consumed, err := self.read_signature_header(data) + consumed, err := self.read_signature_header(self.unconsumed_signature_data) if err != nil { if consumed < 0 { - self.unconsumed_signature_data = data return nil } return err } - data = data[consumed:] - } - consumed := self.read_signature_blocks(data) - data = data[consumed:] - if len(data) > 0 { - self.unconsumed_signature_data = data + self.unconsumed_signature_data = utils.ShiftLeft(self.unconsumed_signature_data, consumed) } + consumed := self.read_signature_blocks(self.unconsumed_signature_data) + self.unconsumed_signature_data = utils.ShiftLeft(self.unconsumed_signature_data, consumed) return nil } diff --git a/tools/rsync/api_test.go b/tools/rsync/api_test.go new file mode 100644 index 000000000..0848252e0 --- /dev/null +++ b/tools/rsync/api_test.go @@ -0,0 +1,77 @@ +// License: GPLv3 Copyright: 2023, Kovid Goyal, + +package rsync + +import ( + "bytes" + "fmt" + "kitty/tools/utils" + "kitty/tools/utils/random" + "testing" + + "golang.org/x/exp/slices" +) + +var _ = fmt.Print + +func TestRsyncRoundtrip(t *testing.T) { + src_data := make([]byte, 4*1024*1024) + random.Bytes(src_data) + + random_patch := func(data []byte) (size int) { + if offset := random.Int(len(data)); offset < len(data) { + max_size := utils.Min(256, len(data)-offset) + if size = random.Int(max_size); size > 0 { + random.Bytes(data[offset : offset+size]) + } + } + return + } + + changed := slices.Clone(src_data) + for i := 0; i < 8; i++ { + random_patch(changed) + } + + p := NewPatcher(int64(len(src_data))) + sigbuf := bytes.Buffer{} + if err := p.CreateSignature(bytes.NewReader(changed), func(p []byte) error { _, err := sigbuf.Write(p); return err }); err != nil { + t.Fatal(err) + } + d := NewDiffer() + if err := d.AddSignatureData(sigbuf.Bytes()); err != nil { + t.Fatal(err) + } + deltabuf := bytes.Buffer{} + if err := d.CreateDelta(bytes.NewReader(src_data), func(b []byte) error { _, err := deltabuf.Write(b); return err }); err != nil { + t.Fatal(err) + } + outputbuf := bytes.Buffer{} + p.StartDelta(&outputbuf, bytes.NewReader(src_data)) + b := make([]byte, 30*1024) + for { + n, _ := deltabuf.Read(b) + if n <= 0 { + break + } + if err := p.UpdateDelta(b[:n]); err != nil { + t.Fatal(err) + } + } + if err := p.FinishDelta(); err != nil { + t.Fatal(err) + } + + output := outputbuf.Bytes() + if !bytes.Equal(src_data, output) { + first_diff := utils.Min(len(src_data), len(output)) + for i := 0; i < first_diff; i++ { + if src_data[i] != output[i] { + first_diff = i + break + } + } + t.Fatalf("Patching failed: %d extra_bytes first different byte at: %d", len(outputbuf.Bytes())-len(src_data), first_diff) + } + +} diff --git a/tools/utils/misc.go b/tools/utils/misc.go index 300ad952b..1bbeb2086 100644 --- a/tools/utils/misc.go +++ b/tools/utils/misc.go @@ -219,6 +219,14 @@ func Concat[T any](slices ...[]T) []T { return result } +func ShiftLeft[T any](s []T, amt int) []T { + leftover := len(s) - amt + if leftover > 0 { + copy(s, s[amt:]) + } + return s[:leftover] +} + func SetStructDefaults(v reflect.Value) (err error) { for _, field := range reflect.VisibleFields(v.Type()) { if defval := field.Tag.Get("default"); defval != "" { diff --git a/tools/utils/random/random.go b/tools/utils/random/random.go new file mode 100644 index 000000000..928459894 --- /dev/null +++ b/tools/utils/random/random.go @@ -0,0 +1,35 @@ +// License: GPLv3 Copyright: 2023, Kovid Goyal, + +package random + +import ( + "crypto/rand" + "fmt" + "math/big" + + "golang.org/x/exp/constraints" +) + +var _ = fmt.Print + +// Return a random integer in the range [0, limit). limit must be > 0 +func Int[T constraints.Integer](limit T) T { + b := big.NewInt(int64(limit)) + n, err := rand.Int(rand.Reader, b) + if err != nil { + panic(err) + } + return T(n.Uint64()) +} + +// Return one of items randomnly +func Choice[T any](items ...T) T { + return items[Int(len(items))] +} + +// Write randomn bytes into the provided slice +func Bytes(b []byte) { + if _, err := rand.Read(b); err != nil { + panic(err) + } +}