add flag to allow user set dynamic concurrency

This commit is contained in:
stephensxu 2018-08-14 14:48:58 -07:00
parent d98a437ca3
commit c0496f588c
2 changed files with 17 additions and 3 deletions

View File

@ -116,6 +116,13 @@ func main() {
Name: "env-file",
Usage: "source env file",
},
cli.StringFlag{
Name: "max-concurrency",
Usage: "customize number concurrent files to process",
Value: "100",
EnvVar: "PLUGIN_MAX_CONCURRENCY",
},
}
if err := app.Run(os.Args); err != nil {
@ -145,6 +152,7 @@ func run(c *cli.Context) error {
Redirects: c.Generic("redirects").(*MapFlag).Get(),
CloudFrontDistribution: c.String("cloudfront-distribution"),
DryRun: c.Bool("dry-run"),
MaxConcurrency: c.String("max-concurrency"),
}
return plugin.Exec()

View File

@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"strings"
"strconv"
)
type Plugin struct {
@ -28,10 +29,9 @@ type Plugin struct {
PathStyle bool
client AWS
jobs []job
MaxConcurrency string
}
const maxConcurrent = 100
type job struct {
local string
remote string
@ -156,7 +156,13 @@ func (p *Plugin) createInvalidateJob() {
func (p *Plugin) runJobs() {
client := p.client
jobChan := make(chan struct{}, maxConcurrent)
maxConcurrency, err := strconv.Atoi(p.MaxConcurrency)
if err != nil {
fmt.Printf("ERROR: invalid input for max-concurrency")
os.Exit(1)
}
jobChan := make(chan struct{}, maxConcurrency)
results := make(chan *result, len(p.jobs))
var invalidateJob *job