add flag to allow user set dynamic concurrency

This commit is contained in:
stephensxu 2018-08-14 14:48:58 -07:00
parent d98a437ca3
commit c0496f588c
2 changed files with 17 additions and 3 deletions

View File

@ -116,6 +116,13 @@ func main() {
Name: "env-file", Name: "env-file",
Usage: "source env file", Usage: "source env file",
}, },
cli.StringFlag{
Name: "max-concurrency",
Usage: "customize number concurrent files to process",
Value: "100",
EnvVar: "PLUGIN_MAX_CONCURRENCY",
},
} }
if err := app.Run(os.Args); err != nil { if err := app.Run(os.Args); err != nil {
@ -145,6 +152,7 @@ func run(c *cli.Context) error {
Redirects: c.Generic("redirects").(*MapFlag).Get(), Redirects: c.Generic("redirects").(*MapFlag).Get(),
CloudFrontDistribution: c.String("cloudfront-distribution"), CloudFrontDistribution: c.String("cloudfront-distribution"),
DryRun: c.Bool("dry-run"), DryRun: c.Bool("dry-run"),
MaxConcurrency: c.String("max-concurrency"),
} }
return plugin.Exec() return plugin.Exec()

View File

@ -6,6 +6,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
"strconv"
) )
type Plugin struct { type Plugin struct {
@ -28,10 +29,9 @@ type Plugin struct {
PathStyle bool PathStyle bool
client AWS client AWS
jobs []job jobs []job
MaxConcurrency string
} }
const maxConcurrent = 100
type job struct { type job struct {
local string local string
remote string remote string
@ -156,7 +156,13 @@ func (p *Plugin) createInvalidateJob() {
func (p *Plugin) runJobs() { func (p *Plugin) runJobs() {
client := p.client client := p.client
jobChan := make(chan struct{}, maxConcurrent) maxConcurrency, err := strconv.Atoi(p.MaxConcurrency)
if err != nil {
fmt.Printf("ERROR: invalid input for max-concurrency")
os.Exit(1)
}
jobChan := make(chan struct{}, maxConcurrency)
results := make(chan *result, len(p.jobs)) results := make(chan *result, len(p.jobs))
var invalidateJob *job var invalidateJob *job