-
Notifications
You must be signed in to change notification settings - Fork 10
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
9da0b93
commit f13385a
Showing
3 changed files
with
206 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,18 +1,39 @@ | ||
| module github.com/zhiqiangxu/util | ||
|
|
||
| go 1.15 | ||
| go 1.18 | ||
|
|
||
| require ( | ||
| github.com/VividCortex/gohistogram v1.0.0 // indirect | ||
| github.com/dgrijalva/jwt-go v3.2.1-0.20210802184156-9742bd7fca1c+incompatible | ||
| github.com/go-kit/kit v0.9.0 | ||
| github.com/google/go-cmp v0.3.1 // indirect | ||
| github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 | ||
| github.com/prometheus/client_golang v1.2.1 | ||
| github.com/rs/zerolog v1.28.0 | ||
| github.com/zhiqiangxu/rpheap v0.0.0-20191222053847-9002d7e5a1a1 | ||
| go.uber.org/zap v1.13.0 | ||
| golang.org/x/exp v0.0.0-20200320212757-167ffe94c325 | ||
| golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a | ||
| golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b | ||
| gotest.tools v2.2.0+incompatible | ||
| ) | ||
|
|
||
| require ( | ||
| github.com/BurntSushi/toml v0.3.1 // indirect | ||
| github.com/VividCortex/gohistogram v1.0.0 // indirect | ||
| github.com/beorn7/perks v1.0.1 // indirect | ||
| github.com/cespare/xxhash/v2 v2.1.0 // indirect | ||
| github.com/golang/protobuf v1.3.2 // indirect | ||
| github.com/google/go-cmp v0.3.1 // indirect | ||
| github.com/mattn/go-colorable v0.1.12 // indirect | ||
| github.com/mattn/go-isatty v0.0.14 // indirect | ||
| github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect | ||
| github.com/pkg/errors v0.9.1 // indirect | ||
| github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect | ||
| github.com/prometheus/common v0.7.0 // indirect | ||
| github.com/prometheus/procfs v0.0.5 // indirect | ||
| go.uber.org/atomic v1.5.0 // indirect | ||
| go.uber.org/multierr v1.3.0 // indirect | ||
| go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee // indirect | ||
| golang.org/x/lint v0.0.0-20190930215403-16217165b5de // indirect | ||
| golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa // indirect | ||
| honnef.co/go/tools v0.0.1-2019.2.3 // indirect | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,169 @@ | ||
| package parallel | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "sync" | ||
| "sync/atomic" | ||
| "time" | ||
|
|
||
| zlog "github.com/rs/zerolog/log" | ||
| ) | ||
|
|
||
| type Result struct { | ||
| R interface{} | ||
| I int | ||
| } | ||
|
|
||
| func First[R any](ctx context.Context, workers int, handleFunc func(context.Context, int) (R, error), cd time.Duration) (r R, i int, err error) { | ||
| if workers == 0 { | ||
| err = fmt.Errorf("workers == 0") | ||
| return | ||
| } | ||
|
|
||
| if workers == 1 { | ||
| for { | ||
| r, err = handleFunc(ctx, 0) | ||
| if err == nil { | ||
| return | ||
| } | ||
|
|
||
| select { | ||
| case <-ctx.Done(): | ||
| return | ||
| default: | ||
| } | ||
| time.Sleep(cd) | ||
| } | ||
| } | ||
|
|
||
| resultCh := make(chan Result, 1) | ||
| doneCh := make(chan struct{}) | ||
| for i := 0; i < workers; i++ { | ||
| go func(i int) { | ||
| var ( | ||
| r R | ||
| err error | ||
| ) | ||
| for { | ||
| r, err = handleFunc(ctx, i) | ||
| if err != nil { | ||
| select { | ||
| case <-doneCh: | ||
| return | ||
| case <-ctx.Done(): | ||
| return | ||
| default: | ||
| } | ||
| time.Sleep(cd) | ||
| continue | ||
| } | ||
|
|
||
| select { | ||
| case resultCh <- Result{R: r, I: i}: | ||
| default: | ||
| } | ||
| return | ||
| } | ||
|
|
||
| }(i) | ||
| } | ||
|
|
||
| select { | ||
| case result := <-resultCh: | ||
| r = result.R.(R) | ||
| i = result.I | ||
| close(doneCh) | ||
| case <-ctx.Done(): | ||
| err = ctx.Err() | ||
| } | ||
|
|
||
| return | ||
| } | ||
|
|
||
| type subTask struct { | ||
| from, to int | ||
| } | ||
|
|
||
| func All(ctx context.Context, total, unit, workers int, handleFunc func(context.Context, int, int, int) error, dispatchCB func(int, int), retry int, cd time.Duration) { | ||
| if workers == 0 { | ||
| panic("workers == 0") | ||
| } | ||
|
|
||
| taskChan := make(chan subTask, workers) | ||
| doneCh := make(chan struct{}) | ||
|
|
||
| var ( | ||
| wg sync.WaitGroup | ||
| doneCount int64 | ||
| ) | ||
| wg.Add(workers) | ||
| for i := 0; i < workers; i++ { | ||
| go func(i int) { | ||
| defer wg.Done() | ||
|
|
||
| var ( | ||
| err error | ||
| task subTask | ||
| ) | ||
| for { | ||
| select { | ||
| case task = <-taskChan: | ||
| for k := 0; k < retry; k++ { | ||
| err = handleFunc(ctx, i, task.from, task.to) | ||
| if err != nil { | ||
| if k == retry-1 { | ||
| break | ||
| } | ||
| zlog.Warn().Int("i", i).Err(err).Msg("handleFunc") | ||
| time.Sleep(cd) | ||
| continue | ||
| } | ||
| if atomic.AddInt64(&doneCount, int64(task.to-task.from)) == int64(total) { | ||
| close(doneCh) | ||
| return | ||
| } | ||
| break | ||
| } | ||
| if err != nil { | ||
| // switch a worker and try again | ||
| select { | ||
| case taskChan <- task: | ||
| case <-ctx.Done(): | ||
| return | ||
| } | ||
| time.Sleep(cd) | ||
| } | ||
|
|
||
| case <-ctx.Done(): | ||
| return | ||
| case <-doneCh: | ||
| return | ||
| } | ||
| } | ||
| }(i) | ||
| } | ||
|
|
||
| for from := 0; from < total; from += unit { | ||
| to := from + unit | ||
| if to > total { | ||
| to = total | ||
| } | ||
| if dispatchCB != nil { | ||
| dispatchCB(from, to) | ||
| } | ||
| select { | ||
| case taskChan <- subTask{from: from, to: to}: | ||
| case <-ctx.Done(): | ||
| return | ||
| } | ||
| } | ||
|
|
||
| select { | ||
| case <-doneCh: | ||
| case <-ctx.Done(): | ||
| } | ||
|
|
||
| wg.Wait() | ||
| return | ||
| } |