Merge pull request #31 from tboerger/matrix-convert

Initial matrix conversion support

main.go

@@ -15,6 +15,7 @@ import (
 	"github.com/drone/drone-yaml/yaml"
 	"github.com/drone/drone-yaml/yaml/compiler"
 	"github.com/drone/drone-yaml/yaml/compiler/transform"
+	"github.com/drone/drone-yaml/yaml/converter"
 	"github.com/drone/drone-yaml/yaml/linter"
 	"github.com/drone/drone-yaml/yaml/pretty"
 	"github.com/drone/drone-yaml/yaml/signer"
@@ -28,6 +29,10 @@ var (
 	formatSave = format.Flag("save", "save result to source").Short('s').Bool()
 	formatFile = format.Arg("source", "source file location").Default(".drone.yml").File()
 
+	convert     = kingpin.Command("convert", "convert the yaml file")
+	convertSave = convert.Flag("save", "save result to source").Short('s').Bool()
+	convertFile = convert.Arg("source", "source file location").Default(".drone.yml").File()
+
 	lint     = kingpin.Command("lint", "lint the yaml file")
 	lintPriv = lint.Flag("privileged", "privileged mode").Short('p').Bool()
 	lintFile = lint.Arg("source", "source file location").Default(".drone.yml").File()
@@ -50,6 +55,8 @@ func main() {
 	switch kingpin.Parse() {
 	case format.FullCommand():
 		kingpin.FatalIfError(runFormat(), "")
+	case convert.FullCommand():
+		kingpin.FatalIfError(runConvert(), "")
 	case lint.FullCommand():
 		kingpin.FatalIfError(runLint(), "")
 	case sign.FullCommand():
@@ -78,6 +85,26 @@ func runFormat() error {
 	return err
 }
 
+func runConvert() error {
+	f := *convertFile
+	d, err := ioutil.ReadAll(f)
+	if err != nil {
+		return err
+	}
+	m := converter.Metadata{
+		Filename: f.Name(),
+	}
+	b, err := converter.Convert(d, m)
+	if err != nil {
+		return err
+	}
+	if *formatSave {
+		return ioutil.WriteFile(f.Name(), b, 0644)
+	}
+	_, err = io.Copy(os.Stderr, bytes.NewReader(b))
+	return err
+}
+
 func runLint() error {
 	f := *lintFile
 	m, err := yaml.Parse(f)

yaml/converter/legacy/internal/config.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	droneyaml "github.com/drone/drone-yaml/yaml"
+	"github.com/drone/drone-yaml/yaml/converter/legacy/matrix"
 	"github.com/drone/drone-yaml/yaml/pretty"
 
 	"gopkg.in/yaml.v2"
@@ -21,6 +22,7 @@ type Config struct {
 	Pipeline Containers
 	Services Containers
 	Branches Constraint
+	Matrix   interface{}
 	Secrets  map[string]struct {
 		Driver     string
 		DriverOpts map[string]string `yaml:"driver_opts"`
@@ -34,11 +36,14 @@ type Config struct {
 func Convert(d []byte) ([]byte, error) {
 	from := new(Config)
 	err := yaml.Unmarshal(d, from)
+
 	if err != nil {
 		return nil, err
 	}
 
-	pipeline := &droneyaml.Pipeline{}
+	manifest := &droneyaml.Manifest{}
+
+	pipeline := droneyaml.Pipeline{}
 	pipeline.Name = "default"
 	pipeline.Kind = "pipeline"
 
@@ -65,20 +70,57 @@ func Convert(d []byte) ([]byte, error) {
 			toContainer(container),
 		)
 	}
+
 	pipeline.Volumes = toVolumes(from)
 	pipeline.Trigger.Branch.Include = from.Branches.Include
 	pipeline.Trigger.Branch.Exclude = from.Branches.Exclude
 
-	manifest := &droneyaml.Manifest{}
-	manifest.Resources = append(manifest.Resources, pipeline)
+	if from.Matrix != nil {
+		axes, err := matrix.Parse(d)
+
+		if err != nil {
+			return nil, err
+		}
+
+		for index, environ := range axes {
+			current := pipeline
+			current.Name = fmt.Sprintf("matrix-%d", index+1)
+
+			marshaled, err := yaml.Marshal(&current)
+
+			if err != nil {
+				return nil, err
+			}
+
+			transformed := string(marshaled)
+
+			for key, value := range environ {
+				if strings.Contains(value, "\n") {
+					value = fmt.Sprintf("%q", value)
+				}
+
+				transformed = strings.Replace(transformed, fmt.Sprintf("${%s}", key), value, -1)
+			}
+
+			if err := yaml.Unmarshal([]byte(transformed), &current); err != nil {
+				return nil, err
+			}
+
+			manifest.Resources = append(manifest.Resources, &current)
+		}
+	} else {
+		manifest.Resources = append(manifest.Resources, &pipeline)
+	}
 
 	secrets := toSecrets(from)
+
 	if secrets != nil {
 		manifest.Resources = append(manifest.Resources, secrets)
 	}
 
 	buf := new(bytes.Buffer)
 	pretty.Print(buf, manifest)
+
 	return buf.Bytes(), nil
 }
 

yaml/converter/legacy/internal/config_test.go

@@ -26,6 +26,14 @@ func TestConvert(t *testing.T) {
 			before: "testdata/vault_3.yml",
 			after:  "testdata/vault_3.yml.golden",
 		},
+		{
+			before: "testdata/matrix_1.yml",
+			after:  "testdata/matrix_1.yml.golden",
+		},
+		{
+			before: "testdata/matrix_2.yml",
+			after:  "testdata/matrix_2.yml.golden",
+		},
 	}
 
 	for _, test := range tests {

yaml/converter/legacy/internal/testdata/matrix_1.yml

@@ -0,0 +1,19 @@
+
+pipeline:
+  test:
+    image: golang:${GO_VERSION}
+    commands:
+    - go test -v ./...
+
+services:
+  redis:
+    image: redis:2.6
+
+matrix:
+  include:
+    - GO_VERSION: 1.11
+      REDIS_VERSION: 2.6
+    - GO_VERSION: 1.10
+      REDIS_VERSION: 2.6
+    - GO_VERSION: 1.9
+      REDIS_VERSION: 2.6

yaml/converter/legacy/internal/testdata/matrix_1.yml.golden

@@ -0,0 +1,61 @@
+---
+kind: pipeline
+name: matrix-1
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: test
+  pull: default
+  image: golang:1.11
+  commands:
+  - go test -v ./...
+
+services:
+- name: redis
+  pull: default
+  image: redis:2.6
+
+---
+kind: pipeline
+name: matrix-2
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: test
+  pull: default
+  image: golang:1.10
+  commands:
+  - go test -v ./...
+
+services:
+- name: redis
+  pull: default
+  image: redis:2.6
+
+---
+kind: pipeline
+name: matrix-3
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: test
+  pull: default
+  image: golang:1.9
+  commands:
+  - go test -v ./...
+
+services:
+- name: redis
+  pull: default
+  image: redis:2.6
+
+...

yaml/converter/legacy/internal/testdata/matrix_2.yml

@@ -0,0 +1,19 @@
+
+pipeline:
+  test:
+    image: golang:${GO_VERSION}
+    commands:
+    - go test -v ./...
+
+services:
+  redis:
+    image: redis:${REDIS_VERSION}
+
+matrix:
+  GO_VERSION:
+  - 1.11
+  - 1.10
+
+  REDIS_VERSION:
+  - 2.6
+  - 2.8

yaml/converter/legacy/internal/testdata/matrix_2.yml.golden

@@ -0,0 +1,81 @@
+---
+kind: pipeline
+name: matrix-1
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: test
+  pull: default
+  image: golang:1.11
+  commands:
+  - go test -v ./...
+
+services:
+- name: redis
+  pull: default
+  image: redis:2.6
+
+---
+kind: pipeline
+name: matrix-2
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: test
+  pull: default
+  image: golang:1.11
+  commands:
+  - go test -v ./...
+
+services:
+- name: redis
+  pull: default
+  image: redis:2.8
+
+---
+kind: pipeline
+name: matrix-3
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: test
+  pull: default
+  image: golang:1.10
+  commands:
+  - go test -v ./...
+
+services:
+- name: redis
+  pull: default
+  image: redis:2.6
+
+---
+kind: pipeline
+name: matrix-4
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: test
+  pull: default
+  image: golang:1.10
+  commands:
+  - go test -v ./...
+
+services:
+- name: redis
+  pull: default
+  image: redis:2.8
+
+...

yaml/converter/legacy/matrix/matrix.go

@@ -0,0 +1,117 @@
+package matrix
+
+import (
+	"strings"
+
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	limitTags = 10
+	limitAxis = 25
+)
+
+// Matrix represents the build matrix.
+type Matrix map[string][]string
+
+// Axis represents a single permutation of entries from the build matrix.
+type Axis map[string]string
+
+// String returns a string representation of an Axis as a comma-separated list
+// of environment variables.
+func (a Axis) String() string {
+	var envs []string
+	for k, v := range a {
+		envs = append(envs, k+"="+v)
+	}
+	return strings.Join(envs, " ")
+}
+
+// Parse parses the Yaml matrix definition.
+func Parse(data []byte) ([]Axis, error) {
+
+	axis, err := parseList(data)
+	if err == nil && len(axis) != 0 {
+		return axis, nil
+	}
+
+	matrix, err := parse(data)
+	if err != nil {
+		return nil, err
+	}
+
+	// if not a matrix build return an array with just the single axis.
+	if len(matrix) == 0 {
+		return nil, nil
+	}
+
+	return calc(matrix), nil
+}
+
+// ParseString parses the Yaml string matrix definition.
+func ParseString(data string) ([]Axis, error) {
+	return Parse([]byte(data))
+}
+
+func calc(matrix Matrix) []Axis {
+	// calculate number of permutations and extract the list of tags
+	// (ie go_version, redis_version, etc)
+	var perm int
+	var tags []string
+	for k, v := range matrix {
+		perm *= len(v)
+		if perm == 0 {
+			perm = len(v)
+		}
+		tags = append(tags, k)
+	}
+
+	// structure to hold the transformed result set
+	axisList := []Axis{}
+
+	// for each axis calculate the uniqe set of values that should be used.
+	for p := 0; p < perm; p++ {
+		axis := map[string]string{}
+		decr := perm
+		for i, tag := range tags {
+			elems := matrix[tag]
+			decr = decr / len(elems)
+			elem := p / decr % len(elems)
+			axis[tag] = elems[elem]
+
+			// enforce a maximum number of tags in the build matrix.
+			if i > limitTags {
+				break
+			}
+		}
+
+		// append to the list of axis.
+		axisList = append(axisList, axis)
+
+		// enforce a maximum number of axis that should be calculated.
+		if p > limitAxis {
+			break
+		}
+	}
+
+	return axisList
+}
+
+func parse(raw []byte) (Matrix, error) {
+	data := struct {
+		Matrix map[string][]string
+	}{}
+	err := yaml.Unmarshal(raw, &data)
+	return data.Matrix, err
+}
+
+func parseList(raw []byte) ([]Axis, error) {
+	data := struct {
+		Matrix struct {
+			Include []Axis
+		}
+	}{}
+
+	err := yaml.Unmarshal(raw, &data)
+	return data.Matrix.Include, err
+}