From c0496f588c554a55ce3afe2b3e98176f40ebc4b5 Mon Sep 17 00:00:00 2001 From: stephensxu Date: Tue, 14 Aug 2018 14:48:58 -0700 Subject: [PATCH 1/2] add flag to allow user set dynamic concurrency --- main.go | 8 ++++++++ plugin.go | 12 +++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index baed8f5..8c08de3 100644 --- a/main.go +++ b/main.go @@ -116,6 +116,13 @@ func main() { Name: "env-file", Usage: "source env file", }, + cli.StringFlag{ + Name: "max-concurrency", + Usage: "customize number concurrent files to process", + Value: "100", + EnvVar: "PLUGIN_MAX_CONCURRENCY", + + }, } if err := app.Run(os.Args); err != nil { @@ -145,6 +152,7 @@ func run(c *cli.Context) error { Redirects: c.Generic("redirects").(*MapFlag).Get(), CloudFrontDistribution: c.String("cloudfront-distribution"), DryRun: c.Bool("dry-run"), + MaxConcurrency: c.String("max-concurrency"), } return plugin.Exec() diff --git a/plugin.go b/plugin.go index d0040ee..d078d66 100644 --- a/plugin.go +++ b/plugin.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "strconv" ) type Plugin struct { @@ -28,10 +29,9 @@ type Plugin struct { PathStyle bool client AWS jobs []job + MaxConcurrency string } -const maxConcurrent = 100 - type job struct { local string remote string @@ -156,7 +156,13 @@ func (p *Plugin) createInvalidateJob() { func (p *Plugin) runJobs() { client := p.client - jobChan := make(chan struct{}, maxConcurrent) + maxConcurrency, err := strconv.Atoi(p.MaxConcurrency) + if err != nil { + fmt.Printf("ERROR: invalid input for max-concurrency") + os.Exit(1) + } + + jobChan := make(chan struct{}, maxConcurrency) results := make(chan *result, len(p.jobs)) var invalidateJob *job From 364ee8f5db6b015a96cf1be11402455ff6d8275a Mon Sep 17 00:00:00 2001 From: stephensxu Date: Wed, 15 Aug 2018 11:13:06 -0700 Subject: [PATCH 2/2] using int instead of string --- main.go | 6 +++--- plugin.go | 11 ++--------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/main.go b/main.go index 8c08de3..62930fb 100644 --- a/main.go +++ b/main.go @@ -116,10 +116,10 @@ func main() { Name: "env-file", Usage: "source env file", }, - cli.StringFlag{ + cli.IntFlag{ Name: "max-concurrency", Usage: "customize number concurrent files to process", - Value: "100", + Value: 100, EnvVar: "PLUGIN_MAX_CONCURRENCY", }, @@ -152,7 +152,7 @@ func run(c *cli.Context) error { Redirects: c.Generic("redirects").(*MapFlag).Get(), CloudFrontDistribution: c.String("cloudfront-distribution"), DryRun: c.Bool("dry-run"), - MaxConcurrency: c.String("max-concurrency"), + MaxConcurrency: c.Int("max-concurrency"), } return plugin.Exec() diff --git a/plugin.go b/plugin.go index d078d66..2834a21 100644 --- a/plugin.go +++ b/plugin.go @@ -6,7 +6,6 @@ import ( "os" "path/filepath" "strings" - "strconv" ) type Plugin struct { @@ -29,7 +28,7 @@ type Plugin struct { PathStyle bool client AWS jobs []job - MaxConcurrency string + MaxConcurrency int } type job struct { @@ -156,13 +155,7 @@ func (p *Plugin) createInvalidateJob() { func (p *Plugin) runJobs() { client := p.client - maxConcurrency, err := strconv.Atoi(p.MaxConcurrency) - if err != nil { - fmt.Printf("ERROR: invalid input for max-concurrency") - os.Exit(1) - } - - jobChan := make(chan struct{}, maxConcurrency) + jobChan := make(chan struct{}, p.MaxConcurrency) results := make(chan *result, len(p.jobs)) var invalidateJob *job