Reuse buffer

flower/control.go

@@ -1,6 +1,7 @@
 package flower
 
 import (
+	"net"
 	"sync"
 	"time"
 
@@ -135,5 +136,25 @@ func (c *Controler) HandleUnknownMessage(msg interface{}) {
 }
 
 func (c *Controler) HandleError(err error) {
+	if err == nil {
+		return
+	}
 	c.logger.Errorf("Error: %+v", err)
+	if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
+		return
+	}
+	/*
+		opErr, ok := err.(*net.OpError)
+		if !ok {
+			return
+		}
+		if opErr.Err == syscall.ECONNABORTED {
+		}
+	*/
+	c.Lock()
+	for thash, prx := range c.proxies {
+		delete(c.proxies, thash) // it does not free up memory
+		prx.Close()
+	}
+	c.Unlock()
 }

flower/proxy.go

@@ -81,6 +81,9 @@ func (p *TCPProxy) Serve() {
 			p.logger.Warnf("Accept stream: %v", err)
 			continue
 		}
+		if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
+			continue
+		}
 
 		if p.isclosed() {
 			break

msg/msg.go

@@ -39,11 +39,9 @@ func Write(w net.Conn, v interface{}) error {
 		return err
 	}
 
-	buf := bufpool.Get()
-	defer bufpool.Put(buf)
-
 	sz := m.Size()
-	buf.Grow(sz)
+	buf := bufpool.GrowGet(sz)
+	defer bufpool.Put(buf)
 	p := buf.Bytes()[:sz]
 
 	if sz, err = m.MarshalTo(p); err != nil {
@@ -145,6 +143,7 @@ func Read(r net.Conn) (interface{}, error) {
 	buf := bufpool.Get()
 	defer bufpool.Put(buf)
 
+	// bytes.Buffer has WriteTo, no need additional buffer
 	if _, err := io.CopyN(buf, r, int64(sz)); err != nil {
 		return nil, errors.WithStack(err)
 	}

pkg/bufpool/bufpool.go

@@ -15,6 +15,12 @@ func Get() *bytes.Buffer {
 	return bufPool.Get().(*bytes.Buffer)
 }
 
+func GrowGet(n int) *bytes.Buffer {
+	buf := bufPool.Get().(*bytes.Buffer)
+	buf.Grow(n)
+	return buf
+}
+
 func Put(buf *bytes.Buffer) {
 	buf.Reset()
 	bufPool.Put(buf)

pkg/conn/conn.go

@@ -4,6 +4,8 @@ import (
 	"io"
 	"net"
 	"sync"
+
+	"github.com/damnever/sunflower/pkg/bufpool"
 )
 
 // LinkStream links two connection,
@@ -16,7 +18,9 @@ func LinkStream(inConn, outConn net.Conn) (int64, int64) {
 		defer wg.Done()
 		defer out.Close()
 
-		*traffic, _ = io.Copy(out, in)
+		buf := bufpool.GrowGet(2048)
+		defer bufpool.Put(buf)
+		*traffic, _ = io.CopyBuffer(out, in, buf.Bytes()[:2048])
 		if tc, ok := out.(*net.TCPConn); ok {
 			tc.CloseRead()
 			tc.CloseWrite()

pkg/doc.go

@@ -1,3 +1,3 @@
 // Package pkg contains some util packages,
-// aim to let those packages can also be used by other projects.
+// aim to let most of those packages can also be used by other projects.
 package pkg

sun/storage/db.go

@@ -18,7 +18,7 @@ type DB struct {
 }
 
 func New(datadir string) (*DB, error) {
-	if err := os.MkdirAll(datadir, 0700); err != nil {
+	if err := os.MkdirAll(datadir, 0750); err != nil {
 		return nil, err
 	}
 	fpath := filepath.Join(datadir, "sqlite3.db")

sun/web/builder.go

@@ -21,10 +21,9 @@ import (
 )
 
 var (
-	tmpDir   = filepath.Join(util.TempDir(), "sunflower")
-	goPath   = filepath.Join(tmpDir, version.Full())
-	pkgPath  = filepath.Join(goPath, "src/github.com/damnever/sunflower")
-	mainPath = filepath.Join(pkgPath, "")
+	tmpDir  = filepath.Join(util.TempDir(), "sunflower")
+	goPath  = filepath.Join(tmpDir, version.Full())
+	pkgPath = filepath.Join(goPath, "src/github.com/damnever/sunflower")
 
 	errUnknown    = errors.New("building process has problem or platform is not supported")
 	errIsBuilding = errors.New("is building, please wait for a minute")
@@ -45,6 +44,7 @@ func fmtPlatform(os, arch, arm string) string {
 	return fmt.Sprintf("%s/%s%s", os, arch, arm)
 }
 
+// https://golang.org/doc/install/source#introduction
 // https://github.com/golang/go/wiki/GoArm
 // The order is the priorities, armv4 is not supported.
 // TODO(damnever): make it configurable
@@ -59,23 +59,33 @@ var platforms = []platform{
 	platform{GOOS: "linux", GOARCH: "arm", GOARM: "6"}, // armv6
 	platform{GOOS: "linux", GOARCH: "arm64"},           // armv8
 	platform{GOOS: "linux", GOARCH: "arm", GOARM: "5"}, // armv5
+	platform{GOOS: "linux", GOARCH: "mips64"},
+	platform{GOOS: "linux", GOARCH: "mips64le"},
+	platform{GOOS: "linux", GOARCH: "mips"},
+	platform{GOOS: "linux", GOARCH: "mipsle"},
 }
 
 type Builder struct {
 	logger      *zap.SugaredLogger
 	done        int32
 	ctx         context.Context
+	bindir      string
 	agentConfig string
 	Cancel      func()
 }
 
-func NewBuilder(agentConfig string) (*Builder, error) {
+func NewBuilder(datadir string, agentConfig string) (*Builder, error) {
 	if err := os.RemoveAll(tmpDir); err != nil {
 		return nil, err
 	}
+	bindir := filepath.Join(datadir, "bin")
+	if err := os.RemoveAll(bindir); err != nil {
+		return nil, err
+	}
 	ctx, cancel := context.WithCancel(context.Background())
 	return &Builder{
 		logger:      log.New("web[b]"),
+		bindir:      bindir,
 		agentConfig: agentConfig,
 		done:        0,
 		ctx:         ctx,
@@ -89,7 +99,7 @@ func (b *Builder) TryGetPkg(username, ahash, os, arch, arm string) ([]byte, erro
 		ext = ".exe"
 	}
 	binName := "flower" + ext
-	binPath := filepath.Join(pkgPath, "bin", fmtPlatform(os, arch, arm), binName)
+	binPath := filepath.Join(b.bindir, fmtPlatform(os, arch, arm), binName)
 
 	if !util.FileExist(binPath) {
 		if atomic.LoadInt32(&b.done) == int32(1) {
@@ -106,7 +116,7 @@ func (b *Builder) StartCrossPlatformBuild() {
 	b.logger.Info("Start cross platform build")
 
 	for _, platform := range platforms {
-		err := makeFlower(b.ctx, platform)
+		err := b.makeFlower(platform)
 		if err == nil {
 			b.logger.Infof("Build %s success", platform.String())
 			continue
@@ -118,20 +128,23 @@ func (b *Builder) StartCrossPlatformBuild() {
 	}
 
 	atomic.StoreInt32(&b.done, 1)
+	if err := os.RemoveAll(goPath); err != nil {
+		b.logger.Errorf("Remove %v failed: %v", err)
+	}
 	b.logger.Info("Cross platform build done")
 }
 
-func makeFlower(ctx context.Context, p platform) error {
+func (b *Builder) makeFlower(p platform) error {
 	if err := tryRestorePkgPath(); err != nil {
 		return err
 	}
-	binDir := filepath.Join(pkgPath, "bin", p.String())
+	binDir := filepath.Join(b.bindir, p.String())
 	if err := os.MkdirAll(binDir, 0750); err != nil && !os.IsExist(err) {
 		return err
 	}
 
 	buidlCmd := fmt.Sprintf("cd %s && go build -o '%s/flower%s' ./cmd/flower", pkgPath, binDir, p.Ext)
-	cmd := exec.CommandContext(ctx, "/bin/bash", "-c", buidlCmd)
+	cmd := exec.CommandContext(b.ctx, "/bin/bash", "-c", buidlCmd)
 	cmd.Env = append(
 		os.Environ(),
 		"GOPATH="+goPath,
@@ -186,7 +199,9 @@ func zipBin(confData, binPath, binName string) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	if _, err := io.Copy(binw, binf); err != nil {
+	tmpBuf := bufpool.GrowGet(32768) // 32 * 1024
+	defer bufpool.Put(tmpBuf)
+	if _, err := io.CopyBuffer(binw, binf, tmpBuf.Bytes()[:32768]); err != nil {
 		return nil, err
 	}
 
@@ -214,17 +229,3 @@ func zipBin(confData, binPath, binName string) ([]byte, error) {
 	}
 	return zipBuf.Bytes(), nil
 }
-
-/*
-type binFile struct {
-	name string
-	size int64
-}
-
-func (b binFile) Name() string       { return b.name }
-func (b binFile) Size() int64        { return b.size }
-func (b binFile) Mode() os.FileMode  { return 0755 }
-func (b binFile) ModTime() time.Time { return time.Now().Local() }
-func (b binFile) IsDir() bool        { return false }
-func (b binFile) Sys() interface{}   { return nil }
-*/