From 52335df0e96fa9c509c6fa2966697620beef6dc3 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Wed, 2 Sep 2020 02:48:45 -0700 Subject: [PATCH] reader: Fix missing buffers (#35) When resetting before reading all content temporary buffers were lost. Make sure to re-add temporary buffers. Fixes #34 --- .travis.yml | 17 ++++---- gunzip.go | 11 +++++ gunzip_test.go | 95 ++++++++++++++++++++++++++++++++++++++++++- testdata/bigempty.gz | Bin 0 -> 98307 bytes 4 files changed, 115 insertions(+), 8 deletions(-) create mode 100644 testdata/bigempty.gz diff --git a/.travis.yml b/.travis.yml index 6e9fca0..acfec4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,19 +1,22 @@ language: go -sudo: false - os: - linux - osx go: - - 1.9.x - - 1.10.x + - 1.13.x + - 1.14.x + - 1.15.x - master -script: - - go test -v -cpu=1,2,4 . - - go test -v -cpu=2 -race -short . +env: + - GO111MODULE=off + +script: + - diff <(gofmt -d .) <(printf "") + - go test -v -cpu=1,2,4 . + - go test -v -cpu=2 -race -short . matrix: allow_failures: diff --git a/gunzip.go b/gunzip.go index 93efec7..d1ae730 100644 --- a/gunzip.go +++ b/gunzip.go @@ -331,6 +331,16 @@ func (z *Reader) killReadAhead() error { // Wait for decompressor to be closed and return error, if any. e, ok := <-z.closeErr z.activeRA = false + + for blk := range z.readAhead { + if blk.b != nil { + z.blockPool <- blk.b + } + } + if cap(z.current) > 0 { + z.blockPool <- z.current + z.current = nil + } if !ok { // Channel is closed, so if there was any error it has already been returned. return nil @@ -418,6 +428,7 @@ func (z *Reader) doReadAhead() { case z.readAhead <- read{b: buf, err: err}: case <-closeReader: // Sent on close, we don't care about the next results + z.blockPool <- buf return } if err != nil { diff --git a/gunzip_test.go b/gunzip_test.go index 8e7d12e..135a6a5 100644 --- a/gunzip_test.go +++ b/gunzip_test.go @@ -11,6 +11,7 @@ import ( "io" "io/ioutil" "os" + "runtime/pprof" "strings" "testing" "time" @@ -331,6 +332,98 @@ func TestDecompressor(t *testing.T) { } } +func TestDecompressorReset(t *testing.T) { + b := new(bytes.Buffer) + var gzip *Reader + + for _, tt := range gunzipTests { + in := bytes.NewReader(tt.gzip) + if gzip == nil { + var err error + gzip, err = NewReader(in) + if err != nil { + t.Fatalf("NewReader: %s", err) + } + defer gzip.Close() + } else { + err := gzip.Reset(in) + if err != nil { + t.Errorf("%s: Reset: %s", tt.name, err) + continue + } + } + if tt.name != gzip.Name { + t.Errorf("%s: got name %s", tt.name, gzip.Name) + } + b.Reset() + + n, err := io.Copy(b, gzip) + if err != tt.err { + t.Errorf("%s: io.Copy: %v want %v", tt.name, err, tt.err) + } + s := b.String() + if s != tt.raw { + t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.name, n, s, len(tt.raw), tt.raw) + } + + // Test Reader Reset. + in = bytes.NewReader(tt.gzip) + err = gzip.Reset(in) + if err != nil { + t.Errorf("%s: Reset: %s", tt.name, err) + continue + } + if tt.name != gzip.Name { + t.Errorf("%s: got name %s", tt.name, gzip.Name) + } + b.Reset() + n, err = io.Copy(b, gzip) + if err != tt.err { + t.Errorf("%s: io.Copy: %v want %v", tt.name, err, tt.err) + } + s = b.String() + if s != tt.raw { + t.Errorf("%s: got %d-byte %q want %d-byte %q", tt.name, n, s, len(tt.raw), tt.raw) + } + } +} + +func TestDecompressorResetNoRead(t *testing.T) { + done := make(chan struct{}) + defer close(done) + go func() { + select { + // Typical runtime is 2-3s, so we add an order of magnitude. + case <-time.After(30 * time.Second): + pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) + case <-done: + } + }() + in, err := ioutil.ReadFile("testdata/bigempty.gz") + if err != nil { + t.Fatal(err) + } + gz, err := NewReader(bytes.NewBuffer(in)) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 100; i++ { + if testing.Short() && i > 10 { + break + } + err := gz.Reset(bytes.NewBuffer(in)) + if err != nil { + t.Fatal(i, err) + } + // Read 100KB, ignore the rest + lr := io.LimitedReader{N: 100 << 10, R: gz} + _, err = io.Copy(ioutil.Discard, &lr) + if err != nil { + t.Fatal(i, err) + } + } +} + func TestIssue6550(t *testing.T) { f, err := os.Open("testdata/issue6550.gz") if err != nil { @@ -637,7 +730,7 @@ func TestTruncatedGunzipBlocks(t *testing.T) { rand.Read(in) var buf bytes.Buffer for i := 0; i < len(in); i += 512 { - enc,_ := kpgzip.NewWriterLevel(&buf, 0) + enc, _ := kpgzip.NewWriterLevel(&buf, 0) _, err := enc.Write(in[:i]) if err != nil { t.Fatal(err) diff --git a/testdata/bigempty.gz b/testdata/bigempty.gz new file mode 100644 index 0000000000000000000000000000000000000000..aa65f2c90e9d98dcfccd282ce3d1e63622eaef37 GIT binary patch literal 98307 zcmeI%y=oLu00rR0BqR`s>1_O2d%;!+LSir2SSbiTfCdYTfSAN4V3SAi1^lbTMpIbi z3Di!7bV((oP$ZgJoSi$5Fz0?LcJHLx^PLL|doDd+m}>sc{5{&6o<2Qnzc_jQVCT)1 z@qfJ#ATVfw??-!EO@PhO)5En_R|lQNhbKT_EHHZfeYH8P?tVNTeK|M&pBDlI1}(6> zvOa18jP}26>>e%-I*SiafWTN_=fbt+=CFM5@cRDo92EithA;4G>HPTDvxT4UE*|_o zGyFgj5Fl`WerdK*nEiFIum-FvG3_OgFPk_Lwz~&zJS@f@8O9+fPmRLphOV}^aae;eGg9r0tC#~ z0VRq+pf6yy?t6G55FlW-4k%Fs0zCn0z-;Xf5Qq~nTgO!@5COBbJ3t^#z-%2?sX)~Wn60b7 z0s!&^%+`6GN>sgo*}D2G03c7mY@OGsMAZwJt*gHR0P+OP)_I*uRK0-Ny80^sAWy(- zo!6;EwF{W7Yrg^V2MU<214|XDb^)_>?KeRFKmoILV5uV2E?~B<{RYS%C}6e@ELEi1 z1