From c0496f588c554a55ce3afe2b3e98176f40ebc4b5 Mon Sep 17 00:00:00 2001 From: stephensxu Date: Tue, 14 Aug 2018 14:48:58 -0700 Subject: [PATCH] add flag to allow user set dynamic concurrency --- main.go | 8 ++++++++ plugin.go | 12 +++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index baed8f5..8c08de3 100644 --- a/main.go +++ b/main.go @@ -116,6 +116,13 @@ func main() { Name: "env-file", Usage: "source env file", }, + cli.StringFlag{ + Name: "max-concurrency", + Usage: "customize number concurrent files to process", + Value: "100", + EnvVar: "PLUGIN_MAX_CONCURRENCY", + + }, } if err := app.Run(os.Args); err != nil { @@ -145,6 +152,7 @@ func run(c *cli.Context) error { Redirects: c.Generic("redirects").(*MapFlag).Get(), CloudFrontDistribution: c.String("cloudfront-distribution"), DryRun: c.Bool("dry-run"), + MaxConcurrency: c.String("max-concurrency"), } return plugin.Exec() diff --git a/plugin.go b/plugin.go index d0040ee..d078d66 100644 --- a/plugin.go +++ b/plugin.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "strconv" ) type Plugin struct { @@ -28,10 +29,9 @@ type Plugin struct { PathStyle bool client AWS jobs []job + MaxConcurrency string } -const maxConcurrent = 100 - type job struct { local string remote string @@ -156,7 +156,13 @@ func (p *Plugin) createInvalidateJob() { func (p *Plugin) runJobs() { client := p.client - jobChan := make(chan struct{}, maxConcurrent) + maxConcurrency, err := strconv.Atoi(p.MaxConcurrency) + if err != nil { + fmt.Printf("ERROR: invalid input for max-concurrency") + os.Exit(1) + } + + jobChan := make(chan struct{}, maxConcurrency) results := make(chan *result, len(p.jobs)) var invalidateJob *job