mirror of
https://github.com/thegeeklab/drone-s3-sync.git
synced 2024-11-05 02:40:40 +00:00
add flag to allow user set dynamic concurrency
This commit is contained in:
parent
d98a437ca3
commit
c0496f588c
8
main.go
8
main.go
@ -116,6 +116,13 @@ func main() {
|
||||
Name: "env-file",
|
||||
Usage: "source env file",
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "max-concurrency",
|
||||
Usage: "customize number concurrent files to process",
|
||||
Value: "100",
|
||||
EnvVar: "PLUGIN_MAX_CONCURRENCY",
|
||||
|
||||
},
|
||||
}
|
||||
|
||||
if err := app.Run(os.Args); err != nil {
|
||||
@ -145,6 +152,7 @@ func run(c *cli.Context) error {
|
||||
Redirects: c.Generic("redirects").(*MapFlag).Get(),
|
||||
CloudFrontDistribution: c.String("cloudfront-distribution"),
|
||||
DryRun: c.Bool("dry-run"),
|
||||
MaxConcurrency: c.String("max-concurrency"),
|
||||
}
|
||||
|
||||
return plugin.Exec()
|
||||
|
12
plugin.go
12
plugin.go
@ -6,6 +6,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type Plugin struct {
|
||||
@ -28,10 +29,9 @@ type Plugin struct {
|
||||
PathStyle bool
|
||||
client AWS
|
||||
jobs []job
|
||||
MaxConcurrency string
|
||||
}
|
||||
|
||||
const maxConcurrent = 100
|
||||
|
||||
type job struct {
|
||||
local string
|
||||
remote string
|
||||
@ -156,7 +156,13 @@ func (p *Plugin) createInvalidateJob() {
|
||||
|
||||
func (p *Plugin) runJobs() {
|
||||
client := p.client
|
||||
jobChan := make(chan struct{}, maxConcurrent)
|
||||
maxConcurrency, err := strconv.Atoi(p.MaxConcurrency)
|
||||
if err != nil {
|
||||
fmt.Printf("ERROR: invalid input for max-concurrency")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
jobChan := make(chan struct{}, maxConcurrency)
|
||||
results := make(chan *result, len(p.jobs))
|
||||
var invalidateJob *job
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user