Merge pull request #31 from tboerger/matrix-convert

Initial matrix conversion support
This commit is contained in:
Brad Rydzewski 2019-01-29 15:17:43 -08:00 committed by GitHub
commit d6a8227226
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 377 additions and 3 deletions

27
main.go
View File

@ -15,6 +15,7 @@ import (
"github.com/drone/drone-yaml/yaml" "github.com/drone/drone-yaml/yaml"
"github.com/drone/drone-yaml/yaml/compiler" "github.com/drone/drone-yaml/yaml/compiler"
"github.com/drone/drone-yaml/yaml/compiler/transform" "github.com/drone/drone-yaml/yaml/compiler/transform"
"github.com/drone/drone-yaml/yaml/converter"
"github.com/drone/drone-yaml/yaml/linter" "github.com/drone/drone-yaml/yaml/linter"
"github.com/drone/drone-yaml/yaml/pretty" "github.com/drone/drone-yaml/yaml/pretty"
"github.com/drone/drone-yaml/yaml/signer" "github.com/drone/drone-yaml/yaml/signer"
@ -28,6 +29,10 @@ var (
formatSave = format.Flag("save", "save result to source").Short('s').Bool() formatSave = format.Flag("save", "save result to source").Short('s').Bool()
formatFile = format.Arg("source", "source file location").Default(".drone.yml").File() formatFile = format.Arg("source", "source file location").Default(".drone.yml").File()
convert = kingpin.Command("convert", "convert the yaml file")
convertSave = convert.Flag("save", "save result to source").Short('s').Bool()
convertFile = convert.Arg("source", "source file location").Default(".drone.yml").File()
lint = kingpin.Command("lint", "lint the yaml file") lint = kingpin.Command("lint", "lint the yaml file")
lintPriv = lint.Flag("privileged", "privileged mode").Short('p').Bool() lintPriv = lint.Flag("privileged", "privileged mode").Short('p').Bool()
lintFile = lint.Arg("source", "source file location").Default(".drone.yml").File() lintFile = lint.Arg("source", "source file location").Default(".drone.yml").File()
@ -50,6 +55,8 @@ func main() {
switch kingpin.Parse() { switch kingpin.Parse() {
case format.FullCommand(): case format.FullCommand():
kingpin.FatalIfError(runFormat(), "") kingpin.FatalIfError(runFormat(), "")
case convert.FullCommand():
kingpin.FatalIfError(runConvert(), "")
case lint.FullCommand(): case lint.FullCommand():
kingpin.FatalIfError(runLint(), "") kingpin.FatalIfError(runLint(), "")
case sign.FullCommand(): case sign.FullCommand():
@ -78,6 +85,26 @@ func runFormat() error {
return err return err
} }
func runConvert() error {
f := *convertFile
d, err := ioutil.ReadAll(f)
if err != nil {
return err
}
m := converter.Metadata{
Filename: f.Name(),
}
b, err := converter.Convert(d, m)
if err != nil {
return err
}
if *formatSave {
return ioutil.WriteFile(f.Name(), b, 0644)
}
_, err = io.Copy(os.Stderr, bytes.NewReader(b))
return err
}
func runLint() error { func runLint() error {
f := *lintFile f := *lintFile
m, err := yaml.Parse(f) m, err := yaml.Parse(f)

View File

@ -6,6 +6,7 @@ import (
"strings" "strings"
droneyaml "github.com/drone/drone-yaml/yaml" droneyaml "github.com/drone/drone-yaml/yaml"
"github.com/drone/drone-yaml/yaml/converter/legacy/matrix"
"github.com/drone/drone-yaml/yaml/pretty" "github.com/drone/drone-yaml/yaml/pretty"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
@ -21,6 +22,7 @@ type Config struct {
Pipeline Containers Pipeline Containers
Services Containers Services Containers
Branches Constraint Branches Constraint
Matrix interface{}
Secrets map[string]struct { Secrets map[string]struct {
Driver string Driver string
DriverOpts map[string]string `yaml:"driver_opts"` DriverOpts map[string]string `yaml:"driver_opts"`
@ -34,11 +36,14 @@ type Config struct {
func Convert(d []byte) ([]byte, error) { func Convert(d []byte) ([]byte, error) {
from := new(Config) from := new(Config)
err := yaml.Unmarshal(d, from) err := yaml.Unmarshal(d, from)
if err != nil { if err != nil {
return nil, err return nil, err
} }
pipeline := &droneyaml.Pipeline{} manifest := &droneyaml.Manifest{}
pipeline := droneyaml.Pipeline{}
pipeline.Name = "default" pipeline.Name = "default"
pipeline.Kind = "pipeline" pipeline.Kind = "pipeline"
@ -65,20 +70,57 @@ func Convert(d []byte) ([]byte, error) {
toContainer(container), toContainer(container),
) )
} }
pipeline.Volumes = toVolumes(from) pipeline.Volumes = toVolumes(from)
pipeline.Trigger.Branch.Include = from.Branches.Include pipeline.Trigger.Branch.Include = from.Branches.Include
pipeline.Trigger.Branch.Exclude = from.Branches.Exclude pipeline.Trigger.Branch.Exclude = from.Branches.Exclude
manifest := &droneyaml.Manifest{} if from.Matrix != nil {
manifest.Resources = append(manifest.Resources, pipeline) axes, err := matrix.Parse(d)
if err != nil {
return nil, err
}
for index, environ := range axes {
current := pipeline
current.Name = fmt.Sprintf("matrix-%d", index+1)
marshaled, err := yaml.Marshal(&current)
if err != nil {
return nil, err
}
transformed := string(marshaled)
for key, value := range environ {
if strings.Contains(value, "\n") {
value = fmt.Sprintf("%q", value)
}
transformed = strings.Replace(transformed, fmt.Sprintf("${%s}", key), value, -1)
}
if err := yaml.Unmarshal([]byte(transformed), &current); err != nil {
return nil, err
}
manifest.Resources = append(manifest.Resources, &current)
}
} else {
manifest.Resources = append(manifest.Resources, &pipeline)
}
secrets := toSecrets(from) secrets := toSecrets(from)
if secrets != nil { if secrets != nil {
manifest.Resources = append(manifest.Resources, secrets) manifest.Resources = append(manifest.Resources, secrets)
} }
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
pretty.Print(buf, manifest) pretty.Print(buf, manifest)
return buf.Bytes(), nil return buf.Bytes(), nil
} }

View File

@ -26,6 +26,14 @@ func TestConvert(t *testing.T) {
before: "testdata/vault_3.yml", before: "testdata/vault_3.yml",
after: "testdata/vault_3.yml.golden", after: "testdata/vault_3.yml.golden",
}, },
{
before: "testdata/matrix_1.yml",
after: "testdata/matrix_1.yml.golden",
},
{
before: "testdata/matrix_2.yml",
after: "testdata/matrix_2.yml.golden",
},
} }
for _, test := range tests { for _, test := range tests {

View File

@ -0,0 +1,19 @@
pipeline:
test:
image: golang:${GO_VERSION}
commands:
- go test -v ./...
services:
redis:
image: redis:2.6
matrix:
include:
- GO_VERSION: 1.11
REDIS_VERSION: 2.6
- GO_VERSION: 1.10
REDIS_VERSION: 2.6
- GO_VERSION: 1.9
REDIS_VERSION: 2.6

View File

@ -0,0 +1,61 @@
---
kind: pipeline
name: matrix-1
platform:
os: linux
arch: amd64
steps:
- name: test
pull: default
image: golang:1.11
commands:
- go test -v ./...
services:
- name: redis
pull: default
image: redis:2.6
---
kind: pipeline
name: matrix-2
platform:
os: linux
arch: amd64
steps:
- name: test
pull: default
image: golang:1.10
commands:
- go test -v ./...
services:
- name: redis
pull: default
image: redis:2.6
---
kind: pipeline
name: matrix-3
platform:
os: linux
arch: amd64
steps:
- name: test
pull: default
image: golang:1.9
commands:
- go test -v ./...
services:
- name: redis
pull: default
image: redis:2.6
...

View File

@ -0,0 +1,19 @@
pipeline:
test:
image: golang:${GO_VERSION}
commands:
- go test -v ./...
services:
redis:
image: redis:${REDIS_VERSION}
matrix:
GO_VERSION:
- 1.11
- 1.10
REDIS_VERSION:
- 2.6
- 2.8

View File

@ -0,0 +1,81 @@
---
kind: pipeline
name: matrix-1
platform:
os: linux
arch: amd64
steps:
- name: test
pull: default
image: golang:1.11
commands:
- go test -v ./...
services:
- name: redis
pull: default
image: redis:2.6
---
kind: pipeline
name: matrix-2
platform:
os: linux
arch: amd64
steps:
- name: test
pull: default
image: golang:1.11
commands:
- go test -v ./...
services:
- name: redis
pull: default
image: redis:2.8
---
kind: pipeline
name: matrix-3
platform:
os: linux
arch: amd64
steps:
- name: test
pull: default
image: golang:1.10
commands:
- go test -v ./...
services:
- name: redis
pull: default
image: redis:2.6
---
kind: pipeline
name: matrix-4
platform:
os: linux
arch: amd64
steps:
- name: test
pull: default
image: golang:1.10
commands:
- go test -v ./...
services:
- name: redis
pull: default
image: redis:2.8
...

View File

@ -0,0 +1,117 @@
package matrix
import (
"strings"
"gopkg.in/yaml.v2"
)
const (
limitTags = 10
limitAxis = 25
)
// Matrix represents the build matrix.
type Matrix map[string][]string
// Axis represents a single permutation of entries from the build matrix.
type Axis map[string]string
// String returns a string representation of an Axis as a comma-separated list
// of environment variables.
func (a Axis) String() string {
var envs []string
for k, v := range a {
envs = append(envs, k+"="+v)
}
return strings.Join(envs, " ")
}
// Parse parses the Yaml matrix definition.
func Parse(data []byte) ([]Axis, error) {
axis, err := parseList(data)
if err == nil && len(axis) != 0 {
return axis, nil
}
matrix, err := parse(data)
if err != nil {
return nil, err
}
// if not a matrix build return an array with just the single axis.
if len(matrix) == 0 {
return nil, nil
}
return calc(matrix), nil
}
// ParseString parses the Yaml string matrix definition.
func ParseString(data string) ([]Axis, error) {
return Parse([]byte(data))
}
func calc(matrix Matrix) []Axis {
// calculate number of permutations and extract the list of tags
// (ie go_version, redis_version, etc)
var perm int
var tags []string
for k, v := range matrix {
perm *= len(v)
if perm == 0 {
perm = len(v)
}
tags = append(tags, k)
}
// structure to hold the transformed result set
axisList := []Axis{}
// for each axis calculate the uniqe set of values that should be used.
for p := 0; p < perm; p++ {
axis := map[string]string{}
decr := perm
for i, tag := range tags {
elems := matrix[tag]
decr = decr / len(elems)
elem := p / decr % len(elems)
axis[tag] = elems[elem]
// enforce a maximum number of tags in the build matrix.
if i > limitTags {
break
}
}
// append to the list of axis.
axisList = append(axisList, axis)
// enforce a maximum number of axis that should be calculated.
if p > limitAxis {
break
}
}
return axisList
}
func parse(raw []byte) (Matrix, error) {
data := struct {
Matrix map[string][]string
}{}
err := yaml.Unmarshal(raw, &data)
return data.Matrix, err
}
func parseList(raw []byte) ([]Axis, error) {
data := struct {
Matrix struct {
Include []Axis
}
}{}
err := yaml.Unmarshal(raw, &data)
return data.Matrix.Include, err
}