From 6a1167dbb3097012b10e9ff03105c977960adce8 Mon Sep 17 00:00:00 2001 From: Thomas Boerger Date: Mon, 28 Jan 2019 16:32:24 +0100 Subject: [PATCH 1/3] Add convert sub-command --- main.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/main.go b/main.go index ab4d76c..7215c67 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,7 @@ import ( "github.com/drone/drone-yaml/yaml" "github.com/drone/drone-yaml/yaml/compiler" "github.com/drone/drone-yaml/yaml/compiler/transform" + "github.com/drone/drone-yaml/yaml/converter" "github.com/drone/drone-yaml/yaml/linter" "github.com/drone/drone-yaml/yaml/pretty" "github.com/drone/drone-yaml/yaml/signer" @@ -28,6 +29,10 @@ var ( formatSave = format.Flag("save", "save result to source").Short('s').Bool() formatFile = format.Arg("source", "source file location").Default(".drone.yml").File() + convert = kingpin.Command("convert", "convert the yaml file") + convertSave = convert.Flag("save", "save result to source").Short('s').Bool() + convertFile = convert.Arg("source", "source file location").Default(".drone.yml").File() + lint = kingpin.Command("lint", "lint the yaml file") lintPriv = lint.Flag("privileged", "privileged mode").Short('p').Bool() lintFile = lint.Arg("source", "source file location").Default(".drone.yml").File() @@ -50,6 +55,8 @@ func main() { switch kingpin.Parse() { case format.FullCommand(): kingpin.FatalIfError(runFormat(), "") + case convert.FullCommand(): + kingpin.FatalIfError(runConvert(), "") case lint.FullCommand(): kingpin.FatalIfError(runLint(), "") case sign.FullCommand(): @@ -78,6 +85,26 @@ func runFormat() error { return err } +func runConvert() error { + f := *convertFile + d, err := ioutil.ReadAll(f) + if err != nil { + return err + } + m := converter.Metadata{ + Filename: f.Name(), + } + b, err := converter.Convert(d, m) + if err != nil { + return err + } + if *formatSave { + return ioutil.WriteFile(f.Name(), b, 0644) + } + _, err = io.Copy(os.Stderr, bytes.NewReader(b)) + return err +} + func runLint() error { f := *lintFile m, err := yaml.Parse(f) From 1c2e4e99167d4f3917ebe7e25674717b48cad888 Mon Sep 17 00:00:00 2001 From: Thomas Boerger Date: Mon, 28 Jan 2019 16:32:48 +0100 Subject: [PATCH 2/3] Add matrix legacy convert testdata --- .../legacy/internal/testdata/matrix_1.yml | 19 +++++ .../internal/testdata/matrix_1.yml.golden | 61 ++++++++++++++ .../legacy/internal/testdata/matrix_2.yml | 19 +++++ .../internal/testdata/matrix_2.yml.golden | 81 +++++++++++++++++++ 4 files changed, 180 insertions(+) create mode 100644 yaml/converter/legacy/internal/testdata/matrix_1.yml create mode 100644 yaml/converter/legacy/internal/testdata/matrix_1.yml.golden create mode 100644 yaml/converter/legacy/internal/testdata/matrix_2.yml create mode 100644 yaml/converter/legacy/internal/testdata/matrix_2.yml.golden diff --git a/yaml/converter/legacy/internal/testdata/matrix_1.yml b/yaml/converter/legacy/internal/testdata/matrix_1.yml new file mode 100644 index 0000000..77e3331 --- /dev/null +++ b/yaml/converter/legacy/internal/testdata/matrix_1.yml @@ -0,0 +1,19 @@ + +pipeline: + test: + image: golang:${GO_VERSION} + commands: + - go test -v ./... + +services: + redis: + image: redis:2.6 + +matrix: + include: + - GO_VERSION: 1.11 + REDIS_VERSION: 2.6 + - GO_VERSION: 1.10 + REDIS_VERSION: 2.6 + - GO_VERSION: 1.9 + REDIS_VERSION: 2.6 diff --git a/yaml/converter/legacy/internal/testdata/matrix_1.yml.golden b/yaml/converter/legacy/internal/testdata/matrix_1.yml.golden new file mode 100644 index 0000000..a78c2a6 --- /dev/null +++ b/yaml/converter/legacy/internal/testdata/matrix_1.yml.golden @@ -0,0 +1,61 @@ +--- +kind: pipeline +name: matrix-1 + +platform: + os: linux + arch: amd64 + +steps: +- name: test + pull: default + image: golang:1.11 + commands: + - go test -v ./... + +services: +- name: redis + pull: default + image: redis:2.6 + +--- +kind: pipeline +name: matrix-2 + +platform: + os: linux + arch: amd64 + +steps: +- name: test + pull: default + image: golang:1.10 + commands: + - go test -v ./... + +services: +- name: redis + pull: default + image: redis:2.6 + +--- +kind: pipeline +name: matrix-3 + +platform: + os: linux + arch: amd64 + +steps: +- name: test + pull: default + image: golang:1.9 + commands: + - go test -v ./... + +services: +- name: redis + pull: default + image: redis:2.6 + +... diff --git a/yaml/converter/legacy/internal/testdata/matrix_2.yml b/yaml/converter/legacy/internal/testdata/matrix_2.yml new file mode 100644 index 0000000..58ba6a5 --- /dev/null +++ b/yaml/converter/legacy/internal/testdata/matrix_2.yml @@ -0,0 +1,19 @@ + +pipeline: + test: + image: golang:${GO_VERSION} + commands: + - go test -v ./... + +services: + redis: + image: redis:${REDIS_VERSION} + +matrix: + GO_VERSION: + - 1.11 + - 1.10 + + REDIS_VERSION: + - 2.6 + - 2.8 diff --git a/yaml/converter/legacy/internal/testdata/matrix_2.yml.golden b/yaml/converter/legacy/internal/testdata/matrix_2.yml.golden new file mode 100644 index 0000000..609709d --- /dev/null +++ b/yaml/converter/legacy/internal/testdata/matrix_2.yml.golden @@ -0,0 +1,81 @@ +--- +kind: pipeline +name: matrix-1 + +platform: + os: linux + arch: amd64 + +steps: +- name: test + pull: default + image: golang:1.11 + commands: + - go test -v ./... + +services: +- name: redis + pull: default + image: redis:2.6 + +--- +kind: pipeline +name: matrix-2 + +platform: + os: linux + arch: amd64 + +steps: +- name: test + pull: default + image: golang:1.11 + commands: + - go test -v ./... + +services: +- name: redis + pull: default + image: redis:2.8 + +--- +kind: pipeline +name: matrix-3 + +platform: + os: linux + arch: amd64 + +steps: +- name: test + pull: default + image: golang:1.10 + commands: + - go test -v ./... + +services: +- name: redis + pull: default + image: redis:2.6 + +--- +kind: pipeline +name: matrix-4 + +platform: + os: linux + arch: amd64 + +steps: +- name: test + pull: default + image: golang:1.10 + commands: + - go test -v ./... + +services: +- name: redis + pull: default + image: redis:2.8 + +... From 3f2bfc0e9cae81a586916e99b9eb0e3892c850c2 Mon Sep 17 00:00:00 2001 From: Thomas Boerger Date: Mon, 28 Jan 2019 16:34:05 +0100 Subject: [PATCH 3/3] Add initial matrix conversion support --- yaml/converter/legacy/internal/config.go | 48 ++++++- yaml/converter/legacy/internal/config_test.go | 8 ++ yaml/converter/legacy/matrix/matrix.go | 117 ++++++++++++++++++ 3 files changed, 170 insertions(+), 3 deletions(-) create mode 100644 yaml/converter/legacy/matrix/matrix.go diff --git a/yaml/converter/legacy/internal/config.go b/yaml/converter/legacy/internal/config.go index ee82218..cf5676b 100644 --- a/yaml/converter/legacy/internal/config.go +++ b/yaml/converter/legacy/internal/config.go @@ -6,6 +6,7 @@ import ( "strings" droneyaml "github.com/drone/drone-yaml/yaml" + "github.com/drone/drone-yaml/yaml/converter/legacy/matrix" "github.com/drone/drone-yaml/yaml/pretty" "gopkg.in/yaml.v2" @@ -21,6 +22,7 @@ type Config struct { Pipeline Containers Services Containers Branches Constraint + Matrix interface{} Secrets map[string]struct { Driver string DriverOpts map[string]string `yaml:"driver_opts"` @@ -34,11 +36,14 @@ type Config struct { func Convert(d []byte) ([]byte, error) { from := new(Config) err := yaml.Unmarshal(d, from) + if err != nil { return nil, err } - pipeline := &droneyaml.Pipeline{} + manifest := &droneyaml.Manifest{} + + pipeline := droneyaml.Pipeline{} pipeline.Name = "default" pipeline.Kind = "pipeline" @@ -65,20 +70,57 @@ func Convert(d []byte) ([]byte, error) { toContainer(container), ) } + pipeline.Volumes = toVolumes(from) pipeline.Trigger.Branch.Include = from.Branches.Include pipeline.Trigger.Branch.Exclude = from.Branches.Exclude - manifest := &droneyaml.Manifest{} - manifest.Resources = append(manifest.Resources, pipeline) + if from.Matrix != nil { + axes, err := matrix.Parse(d) + + if err != nil { + return nil, err + } + + for index, environ := range axes { + current := pipeline + current.Name = fmt.Sprintf("matrix-%d", index+1) + + marshaled, err := yaml.Marshal(¤t) + + if err != nil { + return nil, err + } + + transformed := string(marshaled) + + for key, value := range environ { + if strings.Contains(value, "\n") { + value = fmt.Sprintf("%q", value) + } + + transformed = strings.Replace(transformed, fmt.Sprintf("${%s}", key), value, -1) + } + + if err := yaml.Unmarshal([]byte(transformed), ¤t); err != nil { + return nil, err + } + + manifest.Resources = append(manifest.Resources, ¤t) + } + } else { + manifest.Resources = append(manifest.Resources, &pipeline) + } secrets := toSecrets(from) + if secrets != nil { manifest.Resources = append(manifest.Resources, secrets) } buf := new(bytes.Buffer) pretty.Print(buf, manifest) + return buf.Bytes(), nil } diff --git a/yaml/converter/legacy/internal/config_test.go b/yaml/converter/legacy/internal/config_test.go index 66f74fb..8ee736b 100644 --- a/yaml/converter/legacy/internal/config_test.go +++ b/yaml/converter/legacy/internal/config_test.go @@ -26,6 +26,14 @@ func TestConvert(t *testing.T) { before: "testdata/vault_3.yml", after: "testdata/vault_3.yml.golden", }, + { + before: "testdata/matrix_1.yml", + after: "testdata/matrix_1.yml.golden", + }, + { + before: "testdata/matrix_2.yml", + after: "testdata/matrix_2.yml.golden", + }, } for _, test := range tests { diff --git a/yaml/converter/legacy/matrix/matrix.go b/yaml/converter/legacy/matrix/matrix.go new file mode 100644 index 0000000..76a0f6b --- /dev/null +++ b/yaml/converter/legacy/matrix/matrix.go @@ -0,0 +1,117 @@ +package matrix + +import ( + "strings" + + "gopkg.in/yaml.v2" +) + +const ( + limitTags = 10 + limitAxis = 25 +) + +// Matrix represents the build matrix. +type Matrix map[string][]string + +// Axis represents a single permutation of entries from the build matrix. +type Axis map[string]string + +// String returns a string representation of an Axis as a comma-separated list +// of environment variables. +func (a Axis) String() string { + var envs []string + for k, v := range a { + envs = append(envs, k+"="+v) + } + return strings.Join(envs, " ") +} + +// Parse parses the Yaml matrix definition. +func Parse(data []byte) ([]Axis, error) { + + axis, err := parseList(data) + if err == nil && len(axis) != 0 { + return axis, nil + } + + matrix, err := parse(data) + if err != nil { + return nil, err + } + + // if not a matrix build return an array with just the single axis. + if len(matrix) == 0 { + return nil, nil + } + + return calc(matrix), nil +} + +// ParseString parses the Yaml string matrix definition. +func ParseString(data string) ([]Axis, error) { + return Parse([]byte(data)) +} + +func calc(matrix Matrix) []Axis { + // calculate number of permutations and extract the list of tags + // (ie go_version, redis_version, etc) + var perm int + var tags []string + for k, v := range matrix { + perm *= len(v) + if perm == 0 { + perm = len(v) + } + tags = append(tags, k) + } + + // structure to hold the transformed result set + axisList := []Axis{} + + // for each axis calculate the uniqe set of values that should be used. + for p := 0; p < perm; p++ { + axis := map[string]string{} + decr := perm + for i, tag := range tags { + elems := matrix[tag] + decr = decr / len(elems) + elem := p / decr % len(elems) + axis[tag] = elems[elem] + + // enforce a maximum number of tags in the build matrix. + if i > limitTags { + break + } + } + + // append to the list of axis. + axisList = append(axisList, axis) + + // enforce a maximum number of axis that should be calculated. + if p > limitAxis { + break + } + } + + return axisList +} + +func parse(raw []byte) (Matrix, error) { + data := struct { + Matrix map[string][]string + }{} + err := yaml.Unmarshal(raw, &data) + return data.Matrix, err +} + +func parseList(raw []byte) ([]Axis, error) { + data := struct { + Matrix struct { + Include []Axis + } + }{} + + err := yaml.Unmarshal(raw, &data) + return data.Matrix.Include, err +}