initial commit

.drone.yml

@@ -0,0 +1,20 @@
+---
+kind: pipeline
+name: default
+
+platform:
+  os: linux
+  arch: amd64
+
+workspace:
+  base: /go
+  path: src/github.com/drone/drone-yaml
+
+steps:
+- name: test
+  image: golang
+  commands:
+  - go get -t -v ./...
+  - go test ./...
+
+...

.gitignore

@@ -0,0 +1,3 @@
+NOTES*
+*.out
+*.env

README.md

@@ -0,0 +1,33 @@
+Package yaml provides a parser, linter, formatter and compiler for the [drone](https://github.com/drone/drone) configuration file format.
+
+Lint the yaml file:
+
+```text
+$ drone-yaml lint samples/simple.yml
+```
+
+Format the yaml file:
+
+```text
+$ drone-yaml fmt samples/simple.yml
+$ drone-yaml fmt samples/simple.yml --save
+```
+
+Sign the yaml file using a 32-bit secret key:
+
+```text
+$ drone-yaml sign 642909eb4c3d47e33999235c0598353c samples/simple.yml
+$ drone-yaml sign 642909eb4c3d47e33999235c0598353c samples/simple.yml --save
+```
+
+Verify the yaml file signature:
+
+```text
+$ drone-yaml verify 642909eb4c3d47e33999235c0598353c samples/simple.yml
+```
+
+Compile the yaml file:
+
+```text
+$ drone-yaml compile samples/simple.yml > samples/simple.json
+```

main.go

@@ -0,0 +1,295 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/drone/drone-yaml/yaml"
+	"github.com/drone/drone-yaml/yaml/compiler"
+	"github.com/drone/drone-yaml/yaml/compiler/transform"
+	"github.com/drone/drone-yaml/yaml/linter"
+	"github.com/drone/drone-yaml/yaml/pretty"
+	"github.com/drone/drone-yaml/yaml/signer"
+
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+var (
+	format     = kingpin.Command("fmt", "format the yaml file")
+	formatPriv = format.Flag("privileged", "privileged mode").Short('p').Bool()
+	formatSave = format.Flag("save", "save result to source").Short('s').Bool()
+	formatFile = format.Arg("source", "source file location").Default(".drone.yml").File()
+
+	lint     = kingpin.Command("lint", "lint the yaml file")
+	lintPriv = lint.Flag("privileged", "privileged mode").Short('p').Bool()
+	lintFile = lint.Arg("source", "source file location").Default(".drone.yml").File()
+
+	sign     = kingpin.Command("sign", "sign the yaml file")
+	signKey  = sign.Arg("key", "secret key").Required().String()
+	signFile = sign.Arg("source", "source file location").Default(".drone.yml").File()
+	signSave = sign.Flag("save", "save result to source").Short('s').Bool()
+
+	verify     = kingpin.Command("verify", "verify the yaml signature")
+	verifyKey  = verify.Arg("key", "secret key").Required().String()
+	verifyFile = verify.Arg("source", "source file location").Default(".drone.yml").File()
+
+	compile     = kingpin.Command("compile", "compile the yaml file")
+	compileIn   = compile.Arg("source", "source file location").Default(".drone.yml").File()
+	compileName = compile.Flag("name", "pipeline name").String()
+)
+
+func main() {
+	switch kingpin.Parse() {
+	case format.FullCommand():
+		kingpin.FatalIfError(runFormat(), "")
+	case lint.FullCommand():
+		kingpin.FatalIfError(runLint(), "")
+	case sign.FullCommand():
+		kingpin.FatalIfError(runSign(), "")
+	case verify.FullCommand():
+		kingpin.FatalIfError(runVerify(), "")
+	case compile.FullCommand():
+		kingpin.FatalIfError(runCompile(), "")
+	}
+}
+
+func runFormat() error {
+	f := *formatFile
+	m, err := yaml.Parse(f)
+	if err != nil {
+		return err
+	}
+
+	b := new(bytes.Buffer)
+	pretty.Print(b, m)
+
+	if *formatSave {
+		return ioutil.WriteFile(f.Name(), b.Bytes(), 0644)
+	}
+	_, err = io.Copy(os.Stderr, b)
+	return err
+}
+
+func runLint() error {
+	f := *lintFile
+	m, err := yaml.Parse(f)
+	if err != nil {
+		return err
+	}
+	for _, r := range m.Resources {
+		err := linter.Lint(r, *lintPriv)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func runSign() error {
+	f := *signFile
+	d, err := ioutil.ReadAll(f)
+	if err != nil {
+		return err
+	}
+
+	k := signer.KeyString(*signKey)
+
+	if *signSave {
+		out, err := signer.SignUpdate(d, k)
+		if err != nil {
+			return err
+		}
+		return ioutil.WriteFile(f.Name(), out, 0644)
+	}
+
+	hmac, err := signer.Sign(d, k)
+	if err != nil {
+		return err
+	}
+	fmt.Println(hmac)
+	return nil
+}
+
+func runVerify() error {
+	f := *verifyFile
+	d, err := ioutil.ReadAll(f)
+	if err != nil {
+		return err
+	}
+
+	k := signer.KeyString(*verifyKey)
+	ok, err := signer.Verify(d, k)
+	if err != nil {
+		return err
+	} else if !ok {
+		return errors.New("cannot verify yaml signature")
+	}
+
+	fmt.Println("success: yaml signature verified")
+	return nil
+}
+
+var (
+	trusted    = compile.Flag("trusted", "trusted mode").Bool()
+	labels     = compile.Flag("label", "container labels").StringMap()
+	clone      = compile.Flag("clone", "clone step").Bool()
+	volume     = compile.Flag("volume", "attached volumes").StringMap()
+	network    = compile.Flag("network", "attached networks").Strings()
+	environ    = compile.Flag("env", "environment variable").StringMap()
+	dind       = compile.Flag("dind", "dind images").Default("plugins/docker").Strings()
+	event      = compile.Flag("event", "event type").PlaceHolder("<event>").Enum("push", "pull_request", "tag", "deployment")
+	repo       = compile.Flag("repo", "repository name").PlaceHolder("octocat/hello-world").String()
+	remote     = compile.Flag("git-remote", "git remote url").PlaceHolder("https://github.com/octocat/hello-world.git").String()
+	branch     = compile.Flag("git-branch", "git commit branch").PlaceHolder("master").String()
+	ref        = compile.Flag("git-ref", "git commit ref").PlaceHolder("refs/heads/master").String()
+	sha        = compile.Flag("git-sha", "git commit sha").String()
+	creds      = compile.Flag("git-creds", "git credentials").URLList()
+	instance   = compile.Flag("instance", "drone instance hostname").PlaceHolder("drone.company.com").String()
+	deploy     = compile.Flag("deploy-to", "target deployment").PlaceHolder("production").String()
+	secrets    = compile.Flag("secret", "secret variable").StringMap()
+	registries = compile.Flag("registry", "registry credentials").URLList()
+	username   = compile.Flag("netrc-username", "netrc username").PlaceHolder("<token>").String()
+	password   = compile.Flag("netrc-password", "netrc password").PlaceHolder("x-oauth-basic").String()
+	machine    = compile.Flag("netrc-machine", "netrc machine").PlaceHolder("github.com").String()
+	memlimit   = compile.Flag("mem-limit", "memory limit").PlaceHolder("1GB").Bytes()
+	cpulimit   = compile.Flag("cpu-limit", "cpu limit").PlaceHolder("2").Int64()
+)
+
+func runCompile() error {
+	m, err := yaml.Parse(*compileIn)
+	if err != nil {
+		return err
+	}
+
+	var p *yaml.Pipeline
+	for _, r := range m.Resources {
+		v, ok := r.(*yaml.Pipeline)
+		if !ok {
+			continue
+		}
+		if *compileName == "" ||
+			*compileName == v.Name {
+			p = v
+			break
+		}
+	}
+
+	if p == nil {
+		return errors.New("cannot find pipeline resource")
+	}
+
+	// the user has the option to disable the git clone
+	// if the pipeline is being executed on the local
+	// codebase.
+	if *clone == false {
+		p.Clone.Disable = true
+	}
+
+	var auths []*engine.DockerAuth
+	for _, uri := range *registries {
+		if uri.User == nil {
+			log.Fatalln("Expect registry format [user]:[password]@hostname")
+		}
+		password, ok := uri.User.Password()
+		if !ok {
+			log.Fatalln("Invalid or missing registry password")
+		}
+		auths = append(auths, &engine.DockerAuth{
+			Address:  uri.Host,
+			Username: uri.User.Username(),
+			Password: password,
+		})
+	}
+
+	comp := new(compiler.Compiler)
+	comp.GitCredentialsFunc = defaultCreds // TODO create compiler.GitCredentialsFunc and compiler.GlobalGitCredentialsFunc
+	comp.NetrcFunc = nil                   // TODO create compiler.NetrcFunc and compiler.GlobalNetrcFunc
+	comp.PrivilegedFunc = compiler.DindFunc(*dind)
+	comp.SkipFunc = compiler.SkipFunc(
+		compiler.SkipData{
+			Branch:   *branch,
+			Event:    *event,
+			Instance: *instance,
+			Ref:      *ref,
+			Repo:     *repo,
+			Target:   *deploy,
+		},
+	)
+	comp.TransformFunc = transform.Combine(
+		transform.WithAuths(auths),
+		transform.WithEnviron(*environ),
+		transform.WithEnviron(defaultEnvs()),
+		transform.WithLables(*labels),
+		transform.WithLimits(int64(*memlimit), int64(*cpulimit)),
+		transform.WithNetrc(*machine, *username, *password),
+		transform.WithNetworks(*network),
+		transform.WithProxy(),
+		transform.WithSecrets(*secrets),
+		transform.WithVolumes(*volume),
+	)
+	compiled := comp.Compile(p)
+
+	// // for drone-exec we will need to change the workspace
+	// // to a host volume mount, to the current working dir.
+	// for _, volume := range compiled.Docker.Volumes {
+	// 	if volume.Metadata.Name == "workspace" {
+	// 		volume.EmptyDir = nil
+	// 		volume.HostPath = &engine.VolumeHostPath{
+	// 			Path: "", // pwd
+	// 		}
+	// 		break
+	// 	}
+	// }
+	// // then we need to change the base mount for every container
+	// // to use the workspace base + path.
+	// for _, container := range compiled.Steps {
+	// 	for _, volume := range container.Volumes {
+	// 		if volume.Name == "workspace" {
+	// 			volume.Path = container.Envs["DRONE_WORKSPACE"]
+	// 		}
+	// 	}
+	// }
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	return enc.Encode(compiled)
+}
+
+// helper function returns the git credential function,
+// used to return a git credentials file.
+func defaultCreds() []byte {
+	urls := *creds
+	if len(urls) == 0 {
+		return nil
+	}
+	var buf bytes.Buffer
+	for _, url := range urls {
+		buf.WriteString(url.String())
+		buf.WriteByte('\n')
+	}
+	return buf.Bytes()
+}
+
+// helper function returns the minimum required environment
+// variables to clone a repository. All other environment
+// variables should be passed via the --env flag.
+func defaultEnvs() map[string]string {
+	envs := map[string]string{}
+	envs["DRONE_COMMIT_BRANCH"] = *branch
+	envs["DRONE_COMMIT_SHA"] = *sha
+	envs["DRONE_COMMIT_REF"] = *ref
+	envs["DRONE_REMOTE_URL"] = *remote
+	envs["DRONE_BUILD_EVENT"] = *event
+	if strings.HasPrefix(*ref, "refs/tags/") {
+		envs["DRONE_TAG"] = strings.TrimPrefix(*ref, "refs/tags/")
+	}
+	return envs
+}

samples/complex.yml

@@ -0,0 +1,71 @@
+kind: pipeline
+name: build
+
+steps:
+- name: backend
+  image: golang:1.11
+  commands:
+  - go build
+  - go test -v
+
+- name: frontend
+  image: node
+  commands:
+    - npm install
+    - npm run test
+    - npm run lint
+
+services:
+- name: redis
+  image: redis:latest
+  ports:
+  - 6379
+  volumes:
+  - name: foo
+    path: /bar
+
+volumes:
+- name: foo
+  temp: {}
+
+---
+kind: pipeline
+name: notify
+
+steps:
+- name: notify
+  image: plugins/slack
+  settings:
+    room: general
+    token:
+      $secret: token
+
+node:
+  disk: ssd
+
+depends_on:
+- build
+
+---
+kind: cron
+name: nightly
+spec:
+  schedule: "1 * * * *"
+  branch: master
+  deployment:
+    target: production
+
+---
+kind: secret
+type: encrypted
+
+data:
+  token: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK
+
+---
+kind: registry
+type: encrypted
+
+data:
+  index.drone.io: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK
+

samples/simple.yml

@@ -0,0 +1,22 @@
+kind: pipeline
+name: default
+
+steps:
+- name: backend
+  image: golang:1.11
+  commands:
+  - go build
+  - go test -v
+
+- name: frontend
+  image: node
+  commands:
+    - npm install
+    - npm run test
+    - npm run lint
+
+services:
+- name: redis
+  image: redis:latest
+  ports:
+  - 6379

yaml/build.go

@@ -0,0 +1,40 @@
+package yaml
+
+type (
+	// Build configures a Docker build.
+	Build struct {
+		Args       map[string]string `json:"args,omitempty"`
+		CacheFrom  []string          `json:"cache_from,omitempty" yaml:"cache_from"`
+		Context    string            `json:"context,omitempty"`
+		Dockerfile string            `json:"dockerfile,omitempty"`
+		Image      string            `json:"image,omitempty"`
+		Labels     map[string]string `json:"labels,omitempty"`
+	}
+
+	// build is a tempoary type used to unmarshal
+	// the Build struct when long format is used.
+	build struct {
+		Args       map[string]string
+		CacheFrom  []string `yaml:"cache_from"`
+		Context    string
+		Dockerfile string
+		Image      string
+		Labels     map[string]string
+	}
+)
+
+// UnmarshalYAML implements yaml unmarshalling.
+func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	d := new(build)
+	err := unmarshal(&d.Image)
+	if err != nil {
+		err = unmarshal(d)
+	}
+	b.Args = d.Args
+	b.CacheFrom = d.CacheFrom
+	b.Context = d.Context
+	b.Dockerfile = d.Dockerfile
+	b.Labels = d.Labels
+	b.Image = d.Image
+	return err
+}

yaml/build_test.go

@@ -0,0 +1,44 @@
+package yaml
+
+import (
+	"testing"
+
+	"gopkg.in/yaml.v2"
+)
+
+func TestBuild(t *testing.T) {
+	tests := []struct {
+		yaml  string
+		image string
+	}{
+		{
+			yaml:  "bar",
+			image: "bar",
+		},
+		{
+			yaml:  "{ image: foo }",
+			image: "foo",
+		},
+	}
+	for _, test := range tests {
+		in := []byte(test.yaml)
+		out := new(Build)
+		err := yaml.Unmarshal(in, out)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		if got, want := out.Image, test.image; got != want {
+			t.Errorf("Want image %q, got %q", want, got)
+		}
+	}
+}
+
+func TestBuildError(t *testing.T) {
+	in := []byte("[]")
+	out := new(Build)
+	err := yaml.Unmarshal(in, out)
+	if err == nil {
+		t.Errorf("Expect unmarshal error")
+	}
+}

yaml/compiler/clone.go

@@ -0,0 +1,75 @@
+package compiler
+
+import (
+	"strconv"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/drone/drone-yaml/yaml"
+	"github.com/drone/drone-yaml/yaml/compiler/internal/rand"
+)
+
+// default name of the clone step.
+const cloneStepName = "clone"
+
+// helper function returns the preferred clone image
+// based on the target architecture.
+func cloneImage(src *yaml.Pipeline) string {
+	switch {
+	case src.Platform.OS == "linux" && src.Platform.Arch == "arm":
+		return "drone/git:linux-arm"
+	case src.Platform.OS == "linux" && src.Platform.Arch == "arm64":
+		return "drone/git:linux-arm64"
+	case src.Platform.OS == "windows":
+		return "drone/git:windows-1803"
+	default:
+		return "drone/git"
+	}
+}
+
+// helper function configures the clone depth parameter,
+// specific to the clone plugin.
+//
+// TODO(bradrydzewski) rename to setupCloneParams
+func setupCloneDepth(src *yaml.Pipeline, dst *engine.Step) {
+	if depth := src.Clone.Depth; depth > 0 {
+		dst.Envs["PLUGIN_DEPTH"] = strconv.Itoa(depth)
+	}
+	if skipVerify := src.Clone.SkipVerify; skipVerify {
+		dst.Envs["GIT_SSL_NO_VERIFY"] = "true"
+		dst.Envs["PLUGIN_SKIP_VERIFY"] = "true"
+	}
+}
+
+// helper function configures the .git-clone credentials
+// file. The file is mounted into the container, pointed
+// to by XDG_CONFIG_HOME
+// see https://git-scm.com/docs/git-credential-store
+func setupCloneCredentials(spec *engine.Spec, dst *engine.Step, data []byte) {
+	if len(data) == 0 {
+		return
+	}
+	// TODO(bradrydzewski) we may need to update the git
+	// clone plugin to configure the git credential store.
+	dst.Files = append(dst.Files, &engine.FileMount{
+		Name: ".git-credentials",
+		Path: "/root/.git-credentials",
+	})
+	spec.Files = append(spec.Files, &engine.File{
+		Metadata: engine.Metadata{
+			UID:       rand.String(),
+			Namespace: spec.Metadata.Namespace,
+			Name:      ".git-credentials",
+		},
+		Data: data,
+	})
+}
+
+// helper function creates a default container configuration
+// for the clone stage. The clone stage is automatically
+// added to each pipeline.
+func createClone(src *yaml.Pipeline) *yaml.Container {
+	return &yaml.Container{
+		Name:  cloneStepName,
+		Image: cloneImage(src),
+	}
+}

yaml/compiler/clone_test.go

@@ -0,0 +1,103 @@
+package compiler
+
+import (
+	"testing"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/drone/drone-yaml/yaml"
+)
+
+func TestCloneImage(t *testing.T) {
+	tests := []struct {
+		platform yaml.Platform
+		image    string
+	}{
+		{
+			platform: yaml.Platform{OS: "linux", Arch: "amd64"},
+			image:    "drone/git",
+		},
+		{
+			platform: yaml.Platform{OS: "linux", Arch: "arm"},
+			image:    "drone/git:linux-arm",
+		},
+		{
+			platform: yaml.Platform{OS: "linux", Arch: "arm64"},
+			image:    "drone/git:linux-arm64",
+		},
+		{
+			platform: yaml.Platform{OS: "windows", Arch: "amd64"},
+			image:    "drone/git:windows-1803",
+		},
+		{
+			platform: yaml.Platform{},
+			image:    "drone/git",
+		},
+	}
+	for _, test := range tests {
+		pipeline := &yaml.Pipeline{Platform: test.platform}
+		image := cloneImage(pipeline)
+		if got, want := image, test.image; got != want {
+			t.Errorf("Want clone image %s, got %s", want, got)
+		}
+	}
+}
+
+func TestSetupCloneDepth(t *testing.T) {
+	// test zero depth
+	src := &yaml.Pipeline{
+		Clone: yaml.Clone{
+			Depth: 0,
+		},
+	}
+	dst := &engine.Step{
+		Envs: map[string]string{},
+	}
+	setupCloneDepth(src, dst)
+	if _, ok := dst.Envs["PLUGIN_DEPTH"]; ok {
+		t.Errorf("Expect depth ignored when zero value")
+	}
+
+	// test non-zero depth
+	src = &yaml.Pipeline{
+		Clone: yaml.Clone{
+			Depth: 50,
+		},
+	}
+	dst = &engine.Step{
+		Envs: map[string]string{},
+	}
+	setupCloneDepth(src, dst)
+	if got, want := dst.Envs["PLUGIN_DEPTH"], "50"; got != want {
+		t.Errorf("Expect depth %s, got %s", want, got)
+	}
+}
+
+func TestSetupCloneSkipVerify(t *testing.T) {
+	// test zero depth
+	src := &yaml.Pipeline{
+		Clone: yaml.Clone{
+			SkipVerify: false,
+		},
+	}
+	dst := &engine.Step{
+		Envs: map[string]string{},
+	}
+	setupCloneDepth(src, dst)
+	if _, ok := dst.Envs["PLUGIN_SKIP_VERIFY"]; ok {
+		t.Errorf("Expect skip verify not set")
+	}
+
+	// test non-zero depth
+	src = &yaml.Pipeline{
+		Clone: yaml.Clone{
+			SkipVerify: true,
+		},
+	}
+	dst = &engine.Step{
+		Envs: map[string]string{},
+	}
+	setupCloneDepth(src, dst)
+	if got, want := dst.Envs["PLUGIN_SKIP_VERIFY"], "true"; got != want {
+		t.Errorf("Expect skip verify %s, got %s", want, got)
+	}
+}

yaml/compiler/compiler.go

@@ -0,0 +1,303 @@
+package compiler
+
+import (
+	"github.com/drone/drone-runtime/engine"
+	"github.com/drone/drone-yaml/yaml"
+	"github.com/drone/drone-yaml/yaml/compiler/image"
+	"github.com/drone/drone-yaml/yaml/compiler/internal/rand"
+)
+
+// A Compiler compiles the pipeline configuration to an
+// intermediate representation that can be executed by
+// the Drone runtime engine.
+type Compiler struct {
+	// GitCredentialsFunc returns a .git-credentials file
+	// that can be used by the default clone step to
+	// authenticate to the remote repository.
+	GitCredentialsFunc func() []byte
+
+	// NetrcFunc returns a .netrc file that can be used by
+	// the default clone step to authenticate to the remote
+	// repository.
+	NetrcFunc func() []byte
+
+	// PrivilegedFunc returns true if the container should
+	// be started in privileged mode. The intended use is
+	// for plugins that run Docker-in-Docker. This will be
+	// deprecated in a future release.
+	PrivilegedFunc func(*yaml.Container) bool
+
+	// SkipFunc returns true if the step should be skipped.
+	// The skip function can be used to evaluate the when
+	// clause of each step, and return true if it should
+	// be skipped.
+	SkipFunc func(*yaml.Container) bool
+
+	// TransformFunc can be used to modify the compiled
+	// output prior to completion. This can be useful when
+	// you need to programatically modify the output,
+	// set defaults, etc.
+	TransformFunc func(*engine.Spec)
+
+	// WorkspaceFunc can be used to set the workspace volume
+	// that is created for the entire pipeline. The primary
+	// use case for this function is running local builds,
+	// where the workspace is mounted to the host machine
+	// working directory.
+	WorkspaceFunc func(*engine.Spec)
+
+	// WorkspaceMountFunc can be used to override the default
+	// workspace volume mount.
+	WorkspaceMountFunc func(step *engine.Step, base, path, full string)
+}
+
+// Compile returns an intermediate representation of the
+// pipeline configuration that can be executed by the
+// Drone runtime engine.
+func (c *Compiler) Compile(from *yaml.Pipeline) *engine.Spec {
+	namespace := rand.String()
+
+	isSerial := true
+	for _, step := range from.Steps {
+		if len(step.DependsOn) != 0 {
+			isSerial = false
+			break
+		}
+	}
+
+	spec := &engine.Spec{
+		Metadata: engine.Metadata{
+			UID:       namespace,
+			Name:      namespace,
+			Namespace: namespace,
+			Labels: map[string]string{
+				"io.drone.pipeline.name": from.Name,
+				"io.drone.pipeline.kind": from.Kind,
+				"io.drone.pipeline.type": from.Type,
+			},
+		},
+		Platform: engine.Platform{
+			OS:      from.Platform.OS,
+			Arch:    from.Platform.Arch,
+			Version: from.Platform.Version,
+			Variant: from.Platform.Variant,
+		},
+		Docker:  &engine.DockerConfig{},
+		Files:   nil,
+		Secrets: nil,
+	}
+
+	// create the default workspace path. If a container
+	// does not specify a working directory it defaults
+	// to the workspace path.
+	base, dir, workspace := createWorkspace(from)
+
+	// create the default workspace volume definition.
+	// the volume will be mounted to each container in
+	// the pipeline.
+	c.setupWorkspace(spec)
+
+	// for each volume defined in the yaml configuration
+	// file, convert to a runtime volume and append to the
+	// specification.
+	for _, from := range from.Volumes {
+		to := &engine.Volume{
+			Metadata: engine.Metadata{
+				UID:       rand.String(),
+				Name:      from.Name,
+				Namespace: namespace,
+				Labels:    map[string]string{},
+			},
+		}
+		if from.EmptyDir != nil {
+			// if the yaml configuration specifies an empty
+			// directory volume (data volume) or an in-memory
+			// file system.
+			to.EmptyDir = &engine.VolumeEmptyDir{
+				Medium:    from.EmptyDir.Medium,
+				SizeLimit: int64(from.EmptyDir.SizeLimit),
+			}
+		} else if from.HostPath != nil {
+			// if the yaml configuration specifies a bind
+			// mount to the host machine.
+			to.HostPath = &engine.VolumeHostPath{
+				Path: from.HostPath.Path,
+			}
+		}
+		spec.Docker.Volumes = append(spec.Docker.Volumes, to)
+	}
+
+	if !from.Clone.Disable {
+		src := createClone(from)
+		dst := createStep(spec, src)
+		dst.Docker.PullPolicy = engine.PullIfNotExists
+		setupCloneDepth(from, dst)
+		setupCloneCredentials(spec, dst, c.gitCredentials())
+		setupWorkingDir(src, dst, workspace)
+		setupWorkspaceEnv(dst, base, dir, workspace)
+		c.setupWorkspaceMount(dst, base, dir, workspace)
+		spec.Steps = append(spec.Steps, dst)
+	}
+
+	// for each pipeline service defined in the yaml
+	// configuration file, convert to a runtime step
+	// and append to the specification.
+	for _, service := range from.Services {
+		step := createStep(spec, service)
+		// note that all services are automatically
+		// set to run in detached mode.
+		step.Detach = true
+		setupWorkingDir(service, step, workspace)
+		setupWorkspaceEnv(step, base, dir, workspace)
+		c.setupWorkspaceMount(step, base, dir, workspace)
+		// if the skip callback function returns true,
+		// modify the runtime step to never execute.
+		if c.skip(service) {
+			step.RunPolicy = engine.RunNever
+		}
+		// if the step is a plugin and should be executed
+		// in privileged mode, set the privileged flag.
+		if c.privileged(service) {
+			step.Docker.Privileged = true
+		}
+		// if the clone step is enabled, the service should
+		// not start until the clone step is complete. Add
+		// the clone step as a dependency in the graph.
+		if isSerial == false && from.Clone.Disable == false {
+			step.DependsOn = append(step.DependsOn, cloneStepName)
+		}
+		spec.Steps = append(spec.Steps, step)
+	}
+
+	// rename will store a list of container names
+	// that should be mapped to their temporary alias.
+	rename := map[string]string{}
+
+	// for each pipeline step defined in the yaml
+	// configuration file, convert to a runtime step
+	// and append to the specification.
+	for _, container := range from.Steps {
+		var step *engine.Step
+		switch {
+		case container.Build != nil:
+			step = createBuildStep(spec, container)
+			rename[container.Build.Image] = step.Metadata.UID
+		default:
+			step = createStep(spec, container)
+		}
+		setupWorkingDir(container, step, workspace)
+		setupWorkspaceEnv(step, base, dir, workspace)
+		c.setupWorkspaceMount(step, base, dir, workspace)
+		// if the skip callback function returns true,
+		// modify the runtime step to never execute.
+		if c.skip(container) {
+			step.RunPolicy = engine.RunNever
+		}
+		// if the step is a plugin and should be executed
+		// in privileged mode, set the privileged flag.
+		if c.privileged(container) {
+			step.Docker.Privileged = true
+		}
+		// if the clone step is enabled, the step should
+		// not start until the clone step is complete. If
+		// no dependencies are defined, at a minimum, the
+		// step depends on the initial clone step completing.
+		if isSerial == false && from.Clone.Disable == false && len(step.DependsOn) == 0 {
+			step.DependsOn = append(step.DependsOn, cloneStepName)
+		}
+		spec.Steps = append(spec.Steps, step)
+	}
+
+	// if the pipeline includes any build and publish
+	// steps we should create an entry for the host
+	// machine docker socket.
+	if spec.Docker != nil && len(rename) > 0 {
+		v := &engine.Volume{
+			Metadata: engine.Metadata{
+				UID:       rand.String(),
+				Name:      "_docker_socket",
+				Namespace: namespace,
+				Labels:    map[string]string{},
+			},
+			HostPath: &engine.VolumeHostPath{
+				Path: "/var/run/docker.sock",
+			},
+		}
+		spec.Docker.Volumes = append(spec.Docker.Volumes, v)
+	}
+
+	// images created during the pipeline are assigned a
+	// random alias. All references to the origin image
+	// name must be changed to the alias.
+	for _, step := range spec.Steps {
+		for k, v := range rename {
+			if image.MatchTag(step.Docker.Image, k) {
+				img := image.Trim(step.Docker.Image) + ":" + v
+				step.Docker.Image = image.Expand(img)
+			}
+		}
+	}
+
+	// executes user-defined transformations before the
+	// final specification is returned.
+	if c.TransformFunc != nil {
+		c.TransformFunc(spec)
+	}
+
+	return spec
+}
+
+// return a .git-credentials file. If the user-defined
+// function is nil, a nil credentials file is returned.
+func (c *Compiler) gitCredentials() []byte {
+	if c.GitCredentialsFunc != nil {
+		return c.GitCredentialsFunc()
+	}
+	return nil
+}
+
+// return a .netrc file. If the user-defined function is
+// nil, a nil netrc file is returned.
+func (c *Compiler) netrc() []byte {
+	if c.NetrcFunc != nil {
+		return c.NetrcFunc()
+	}
+	return nil
+}
+
+// return true if the step should be executed in privileged
+// mode. If the user-defined privileged function is nil,
+// a default value of false is returned.
+func (c *Compiler) privileged(container *yaml.Container) bool {
+	if c.PrivilegedFunc != nil {
+		return c.PrivilegedFunc(container)
+	}
+	return false
+}
+
+// return true if the step should be skipped. If the
+// user-defined skip function is nil, a defalt skip
+// function is used that always returns true (i.e. do not skip).
+func (c *Compiler) skip(container *yaml.Container) bool {
+	if c.SkipFunc != nil {
+		return c.SkipFunc(container)
+	}
+	return false
+}
+
+func (c *Compiler) setupWorkspace(spec *engine.Spec) {
+	if c.WorkspaceFunc != nil {
+		c.WorkspaceFunc(spec)
+		return
+	}
+	CreateWorkspace(spec)
+	return
+}
+
+func (c *Compiler) setupWorkspaceMount(step *engine.Step, base, path, full string) {
+	if c.WorkspaceMountFunc != nil {
+		c.WorkspaceMountFunc(step, base, path, full)
+		return
+	}
+	MountWorkspace(step, base, path, full)
+}

yaml/compiler/convert.go

@@ -0,0 +1,76 @@
+package compiler
+
+import (
+	"strings"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/drone/drone-yaml/yaml"
+)
+
+func toIgnoreErr(from *yaml.Container) bool {
+	return strings.EqualFold(from.Failure, "ignore")
+}
+
+func toPorts(from *yaml.Container) []*engine.Port {
+	var ports []*engine.Port
+	for _, port := range from.Ports {
+		ports = append(ports, toPort(port))
+	}
+	return ports
+}
+
+func toPort(from *yaml.Port) *engine.Port {
+	return &engine.Port{
+		Port:     from.Port,
+		Host:     from.Host,
+		Protocol: from.Protocol,
+	}
+}
+
+func toPullPolicy(from *yaml.Container) engine.PullPolicy {
+	switch strings.ToLower(from.Pull) {
+	case "always":
+		return engine.PullAlways
+	case "if-not-exists":
+		return engine.PullIfNotExists
+	case "never":
+		return engine.PullNever
+	default:
+		return engine.PullDefault
+	}
+}
+
+func toRunPolicy(from *yaml.Container) engine.RunPolicy {
+	onFailure := from.When.Status.Match("failure") && len(from.When.Status.Include) > 0
+	onSuccess := from.When.Status.Match("success")
+	switch {
+	case onFailure && onSuccess:
+		return engine.RunAlways
+	case onFailure:
+		return engine.RunOnFailure
+	case onSuccess:
+		return engine.RunOnSuccess
+	default:
+		return engine.RunNever
+	}
+}
+
+func toResources(from *yaml.Container) *engine.Resources {
+	if from.Resources == nil {
+		return nil
+	}
+	return &engine.Resources{
+		Limits:   toResourceObject(from.Resources.Limits),
+		Requests: toResourceObject(from.Resources.Requests),
+	}
+}
+
+func toResourceObject(from *yaml.ResourceObject) *engine.ResourceObject {
+	if from == nil {
+		return nil
+	}
+	return &engine.ResourceObject{
+		CPU:    int64(from.CPU),
+		Memory: int64(from.Memory),
+	}
+}

yaml/compiler/convert_test.go

@@ -0,0 +1,154 @@
+package compiler
+
+import (
+	"testing"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/drone/drone-yaml/yaml"
+	"github.com/google/go-cmp/cmp"
+)
+
+func Test_toIgnoreErr(t *testing.T) {
+	tests := []struct {
+		mode string
+		want bool
+	}{
+		{"Ignore", true},
+		{"ignore", true},
+		{"fail", false},
+	}
+	for _, test := range tests {
+		from := &yaml.Container{Failure: test.mode}
+		if toIgnoreErr(from) != test.want {
+			t.Errorf("unexpected ignore error for %s", test.mode)
+		}
+	}
+}
+
+func Test_toPullPolicy(t *testing.T) {
+	tests := []struct {
+		mode string
+		want engine.PullPolicy
+	}{
+		{"", engine.PullDefault},
+		{"always", engine.PullAlways},
+		{"if-not-exists", engine.PullIfNotExists},
+		{"never", engine.PullNever},
+	}
+	for _, test := range tests {
+		from := &yaml.Container{Pull: test.mode}
+		if toPullPolicy(from) != test.want {
+			t.Errorf("unexpected pull policy for %s", test.mode)
+		}
+	}
+}
+
+func Test_toRunPolicy(t *testing.T) {
+	tests := []struct {
+		cond yaml.Condition
+		want engine.RunPolicy
+	}{
+		{yaml.Condition{}, engine.RunOnSuccess},
+		{yaml.Condition{Include: []string{"success"}}, engine.RunOnSuccess},
+		{yaml.Condition{Include: []string{"failure"}}, engine.RunOnFailure},
+		{yaml.Condition{Include: []string{"success", "failure"}}, engine.RunAlways},
+		{yaml.Condition{Exclude: []string{"success", "failure"}}, engine.RunNever},
+	}
+	for _, test := range tests {
+		from := &yaml.Container{When: yaml.Conditions{Status: test.cond}}
+		if toRunPolicy(from) != test.want {
+			t.Errorf("unexpected pull policy for incude: %s, exclude: %s", test.cond.Include, test.cond.Exclude)
+		}
+	}
+}
+
+func Test_toPorts(t *testing.T) {
+	from := &yaml.Container{
+		Ports: []*yaml.Port{
+			{
+				Port:     80,
+				Host:     8080,
+				Protocol: "TCP",
+			},
+			{
+				Port:     80,
+				Host:     0,
+				Protocol: "",
+			},
+		},
+	}
+	a := toPorts(from)
+	b := []*engine.Port{
+		{
+			Port:     80,
+			Host:     8080,
+			Protocol: "TCP",
+		},
+		{
+			Port:     80,
+			Host:     0,
+			Protocol: "",
+		},
+	}
+	if diff := cmp.Diff(a, b); diff != "" {
+		t.Errorf("Unexpected port conversion")
+		t.Log(diff)
+	}
+}
+
+func Test_toResources(t *testing.T) {
+	from := &yaml.Container{
+		Resources: nil,
+	}
+	if toResources(from) != nil {
+		t.Errorf("Expected nil resources")
+	}
+
+	// test what happens when limits are defined
+	// but reservations are nil.
+
+	from = &yaml.Container{
+		Resources: &yaml.Resources{
+			Limits: &yaml.ResourceObject{
+				Memory: yaml.BytesSize(1000),
+			},
+		},
+	}
+	a := toResources(from)
+	b := &engine.Resources{
+		Limits: &engine.ResourceObject{
+			Memory: 1000,
+		},
+	}
+	if diff := cmp.Diff(a, b); diff != "" {
+		t.Errorf("Unexpected resource conversion")
+		t.Log(diff)
+	}
+
+	// test what happens when reservation and limits
+	// are both provided.
+
+	from = &yaml.Container{
+		Resources: &yaml.Resources{
+			Limits: &yaml.ResourceObject{
+				Memory: yaml.BytesSize(1000),
+			},
+			Requests: &yaml.ResourceObject{
+				Memory: yaml.BytesSize(2000),
+			},
+		},
+	}
+	a = toResources(from)
+	b = &engine.Resources{
+		Limits: &engine.ResourceObject{
+			Memory: 1000,
+		},
+		Requests: &engine.ResourceObject{
+			Memory: 2000,
+		},
+	}
+	if diff := cmp.Diff(a, b); diff != "" {
+		t.Errorf("Unexpected resource conversion")
+		t.Log(diff)
+	}
+}

yaml/compiler/dind.go

@@ -0,0 +1,37 @@
+package compiler
+
+import (
+	"github.com/drone/drone-yaml/yaml"
+	"github.com/drone/drone-yaml/yaml/compiler/image"
+)
+
+// DindFunc is a helper function that returns true
+// if a container image (specifically a plugin) is
+// a whitelisted dind container and should be executed
+// in privileged mode.
+func DindFunc(images []string) func(*yaml.Container) bool {
+	return func(container *yaml.Container) bool {
+		// privileged-by-default containers are only
+		// enabled for plugins steps that do not define
+		// commands, command, or entrypoint.
+		if len(container.Commands) > 0 {
+			return false
+		}
+		if len(container.Command) > 0 {
+			return false
+		}
+		if len(container.Entrypoint) > 0 {
+			return false
+		}
+		// if the container image matches any image
+		// in the whitelist, return true.
+		for _, img := range images {
+			a := img
+			b := container.Image
+			if image.Match(a, b) {
+				return true
+			}
+		}
+		return false
+	}
+}

yaml/compiler/dind_test.go

@@ -0,0 +1,63 @@
+package compiler
+
+import (
+	"testing"
+
+	"github.com/drone/drone-yaml/yaml"
+)
+
+func TestDind(t *testing.T) {
+	tests := []struct {
+		container  *yaml.Container
+		privileged bool
+	}{
+		{
+			container:  &yaml.Container{Image: "plugins/docker"},
+			privileged: true,
+		},
+		{
+			container:  &yaml.Container{Image: "plugins/docker:latest"},
+			privileged: true,
+		},
+		{
+			container:  &yaml.Container{Image: "plugins/docker:1"},
+			privileged: true,
+		},
+		// no match
+		{
+			container:  &yaml.Container{Image: "golang"},
+			privileged: false,
+		},
+		// dind containers cannot set entrypoint or command
+		{
+			container: &yaml.Container{
+				Image:   "plugins/docker",
+				Command: []string{"docker run ..."},
+			},
+			privileged: false,
+		},
+		{
+			container: &yaml.Container{
+				Image:      "plugins/docker",
+				Entrypoint: []string{"docker run ..."},
+			},
+			privileged: false,
+		},
+		// dind containers cannot set commands
+		{
+			container: &yaml.Container{
+				Image:    "plugins/docker",
+				Commands: []string{"docker run ..."},
+			},
+			privileged: false,
+		},
+	}
+	for i, test := range tests {
+		images := []string{"plugins/docker", "plugins/ecr"}
+		privileged := DindFunc(images)(test.container)
+		if privileged != test.privileged {
+			t.Errorf("Want privileged %v at index %d", test.privileged, i)
+		}
+	}
+
+}

yaml/compiler/encode.go

@@ -0,0 +1,53 @@
+package compiler
+
+import (
+	"encoding/base64"
+	"strconv"
+	"strings"
+
+	json "github.com/ghodss/yaml"
+	"gopkg.in/yaml.v2"
+)
+
+// helper funciton encodes an interface value as a string.
+// this function assumes all types were unmarshaled by the
+// yaml.v2 library. The yaml.v2 package only supports a
+// subset of primative types.
+func encode(v interface{}) string {
+	switch v := v.(type) {
+	case string:
+		return v
+	case bool:
+		return strconv.FormatBool(v)
+	case int:
+		return strconv.Itoa(v)
+	case float64:
+		return strconv.FormatFloat(v, 'g', -1, 64)
+	case []byte:
+		return base64.StdEncoding.EncodeToString(v)
+	case []interface{}:
+		return encodeSlice(v)
+	default:
+		return encodeMap(v)
+	}
+}
+
+// helper function encodes a parameter in map format.
+func encodeMap(v interface{}) string {
+	yml, _ := yaml.Marshal(v)
+	out, _ := json.YAMLToJSON(yml)
+	return string(out)
+}
+
+// helper function encodes a parameter in slice format.
+func encodeSlice(v interface{}) string {
+	out, _ := yaml.Marshal(v)
+
+	in := []string{}
+	err := yaml.Unmarshal(out, &in)
+	if err == nil {
+		return strings.Join(in, ",")
+	}
+	out, _ = json.YAMLToJSON(out)
+	return string(out)
+}

yaml/compiler/encode_test.go

@@ -0,0 +1,59 @@
+package compiler
+
+import "testing"
+
+func TestEncode(t *testing.T) {
+	testdatum := []struct {
+		data interface{}
+		text string
+	}{
+		{
+			data: "foo",
+			text: "foo",
+		},
+		{
+			data: true,
+			text: "true",
+		},
+		{
+			data: 42,
+			text: "42",
+		},
+		{
+			data: float64(42.424242),
+			text: "42.424242",
+		},
+		{
+			data: []interface{}{"foo", "bar", "baz"},
+			text: "foo,bar,baz",
+		},
+		{
+			data: []interface{}{1, 1, 2, 3, 5, 8},
+			text: "1,1,2,3,5,8",
+		},
+		{
+			data: []byte("foo"),
+			text: "Zm9v",
+		},
+		{
+			data: []interface{}{
+				struct {
+					Name string `json:"name"`
+				}{
+					Name: "john",
+				},
+			},
+			text: `[{"name":"john"}]`,
+		},
+		{
+			data: map[interface{}]interface{}{"foo": "bar"},
+			text: `{"foo":"bar"}`,
+		},
+	}
+
+	for _, testdata := range testdatum {
+		if got, want := encode(testdata.data), testdata.text; got != want {
+			t.Errorf("Want interface{} encoded to %q, got %q", want, got)
+		}
+	}
+}

yaml/compiler/image/image.go

@@ -0,0 +1,67 @@
+package image
+
+import "github.com/docker/distribution/reference"
+
+// Trim returns the short image name without tag.
+func Trim(name string) string {
+	ref, err := reference.ParseAnyReference(name)
+	if err != nil {
+		return name
+	}
+	named, err := reference.ParseNamed(ref.String())
+	if err != nil {
+		return name
+	}
+	named = reference.TrimNamed(named)
+	return reference.FamiliarName(named)
+}
+
+// Expand returns the fully qualified image name.
+func Expand(name string) string {
+	ref, err := reference.ParseAnyReference(name)
+	if err != nil {
+		return name
+	}
+	named, err := reference.ParseNamed(ref.String())
+	if err != nil {
+		return name
+	}
+	named = reference.TagNameOnly(named)
+	return named.String()
+}
+
+// Match returns true if the image name matches
+// an image in the list. Note the image tag is not used
+// in the matching logic.
+func Match(from string, to ...string) bool {
+	from = Trim(from)
+	for _, match := range to {
+		if from == Trim(match) {
+			return true
+		}
+	}
+	return false
+}
+
+// MatchTag returns true if the image name matches
+// an image in the list, including the tag.
+func MatchTag(a, b string) bool {
+	return Expand(a) == Expand(b)
+}
+
+// MatchHostname returns true if the image hostname
+// matches the specified hostname.
+func MatchHostname(image, hostname string) bool {
+	ref, err := reference.ParseAnyReference(image)
+	if err != nil {
+		return false
+	}
+	named, err := reference.ParseNamed(ref.String())
+	if err != nil {
+		return false
+	}
+	if hostname == "index.docker.io" {
+		hostname = "docker.io"
+	}
+	return reference.Domain(named) == hostname
+}

yaml/compiler/image/image_test.go

@@ -0,0 +1,295 @@
+package image
+
+import "testing"
+
+func Test_trimImage(t *testing.T) {
+	testdata := []struct {
+		from string
+		want string
+	}{
+		{
+			from: "golang",
+			want: "golang",
+		},
+		{
+			from: "golang:latest",
+			want: "golang",
+		},
+		{
+			from: "golang:1.0.0",
+			want: "golang",
+		},
+		{
+			from: "library/golang",
+			want: "golang",
+		},
+		{
+			from: "library/golang:latest",
+			want: "golang",
+		},
+		{
+			from: "library/golang:1.0.0",
+			want: "golang",
+		},
+		{
+			from: "index.docker.io/library/golang:1.0.0",
+			want: "golang",
+		},
+		{
+			from: "docker.io/library/golang:1.0.0",
+			want: "golang",
+		},
+		{
+			from: "gcr.io/library/golang:1.0.0",
+			want: "gcr.io/library/golang",
+		},
+		// error cases, return input unmodified
+		{
+			from: "foo/bar?baz:boo",
+			want: "foo/bar?baz:boo",
+		},
+	}
+	for _, test := range testdata {
+		got, want := Trim(test.from), test.want
+		if got != want {
+			t.Errorf("Want image %q trimmed to %q, got %q", test.from, want, got)
+		}
+	}
+}
+
+func Test_expandImage(t *testing.T) {
+	testdata := []struct {
+		from string
+		want string
+	}{
+		{
+			from: "golang",
+			want: "docker.io/library/golang:latest",
+		},
+		{
+			from: "golang:latest",
+			want: "docker.io/library/golang:latest",
+		},
+		{
+			from: "golang:1.0.0",
+			want: "docker.io/library/golang:1.0.0",
+		},
+		{
+			from: "library/golang",
+			want: "docker.io/library/golang:latest",
+		},
+		{
+			from: "library/golang:latest",
+			want: "docker.io/library/golang:latest",
+		},
+		{
+			from: "library/golang:1.0.0",
+			want: "docker.io/library/golang:1.0.0",
+		},
+		{
+			from: "index.docker.io/library/golang:1.0.0",
+			want: "docker.io/library/golang:1.0.0",
+		},
+		{
+			from: "gcr.io/golang",
+			want: "gcr.io/golang:latest",
+		},
+		{
+			from: "gcr.io/golang:1.0.0",
+			want: "gcr.io/golang:1.0.0",
+		},
+		// error cases, return input unmodified
+		{
+			from: "foo/bar?baz:boo",
+			want: "foo/bar?baz:boo",
+		},
+	}
+	for _, test := range testdata {
+		got, want := Expand(test.from), test.want
+		if got != want {
+			t.Errorf("Want image %q expanded to %q, got %q", test.from, want, got)
+		}
+	}
+}
+
+func Test_matchImage(t *testing.T) {
+	testdata := []struct {
+		from, to string
+		want     bool
+	}{
+		{
+			from: "golang",
+			to:   "golang",
+			want: true,
+		},
+		{
+			from: "golang:latest",
+			to:   "golang",
+			want: true,
+		},
+		{
+			from: "library/golang:latest",
+			to:   "golang",
+			want: true,
+		},
+		{
+			from: "index.docker.io/library/golang:1.0.0",
+			to:   "golang",
+			want: true,
+		},
+		{
+			from: "golang",
+			to:   "golang:latest",
+			want: true,
+		},
+		{
+			from: "library/golang:latest",
+			to:   "library/golang",
+			want: true,
+		},
+		{
+			from: "gcr.io/golang",
+			to:   "gcr.io/golang",
+			want: true,
+		},
+		{
+			from: "gcr.io/golang:1.0.0",
+			to:   "gcr.io/golang",
+			want: true,
+		},
+		{
+			from: "gcr.io/golang:latest",
+			to:   "gcr.io/golang",
+			want: true,
+		},
+		{
+			from: "gcr.io/golang",
+			to:   "gcr.io/golang:latest",
+			want: true,
+		},
+		{
+			from: "golang",
+			to:   "library/golang",
+			want: true,
+		},
+		{
+			from: "golang",
+			to:   "gcr.io/project/golang",
+			want: false,
+		},
+		{
+			from: "golang",
+			to:   "gcr.io/library/golang",
+			want: false,
+		},
+		{
+			from: "golang",
+			to:   "gcr.io/golang",
+			want: false,
+		},
+	}
+	for _, test := range testdata {
+		got, want := Match(test.from, test.to), test.want
+		if got != want {
+			t.Errorf("Want image %q matching %q is %v", test.from, test.to, want)
+		}
+	}
+}
+
+func Test_matchHostname(t *testing.T) {
+	testdata := []struct {
+		image, hostname string
+		want            bool
+	}{
+		{
+			image:    "golang",
+			hostname: "docker.io",
+			want:     true,
+		},
+		{
+			image:    "golang:latest",
+			hostname: "docker.io",
+			want:     true,
+		},
+		{
+			image:    "golang:latest",
+			hostname: "index.docker.io",
+			want:     true,
+		},
+		{
+			image:    "library/golang:latest",
+			hostname: "docker.io",
+			want:     true,
+		},
+		{
+			image:    "docker.io/library/golang:1.0.0",
+			hostname: "docker.io",
+			want:     true,
+		},
+		{
+			image:    "gcr.io/golang",
+			hostname: "docker.io",
+			want:     false,
+		},
+		{
+			image:    "gcr.io/golang:1.0.0",
+			hostname: "gcr.io",
+			want:     true,
+		},
+		{
+			image:    "1.2.3.4:8000/golang:1.0.0",
+			hostname: "1.2.3.4:8000",
+			want:     true,
+		},
+		{
+			image:    "*&^%",
+			hostname: "1.2.3.4:8000",
+			want:     false,
+		},
+	}
+	for _, test := range testdata {
+		got, want := MatchHostname(test.image, test.hostname), test.want
+		if got != want {
+			t.Errorf("Want image %q matching hostname %q is %v", test.image, test.hostname, want)
+		}
+	}
+}
+
+func Test_matchTag(t *testing.T) {
+	testdata := []struct {
+		a, b string
+		want bool
+	}{
+		{
+			a:    "golang:1.0",
+			b:    "golang:1.0",
+			want: true,
+		},
+		{
+			a:    "golang",
+			b:    "golang:latest",
+			want: true,
+		},
+		{
+			a:    "docker.io/library/golang",
+			b:    "golang:latest",
+			want: true,
+		},
+		{
+			a:    "golang",
+			b:    "golang:1.0",
+			want: false,
+		},
+		{
+			a:    "golang:1.0",
+			b:    "golang:2.0",
+			want: false,
+		},
+	}
+	for _, test := range testdata {
+		got, want := MatchTag(test.a, test.b), test.want
+		if got != want {
+			t.Errorf("Want image %q matching image tag %q is %v", test.a, test.b, want)
+		}
+	}
+}

yaml/compiler/internal/rand/rand.go

@@ -0,0 +1,36 @@
+package rand
+
+import (
+	"crypto/rand"
+)
+
+var chars = []byte("abcdefghijklmnopqrstuvwxyz0123456789")
+
+// random string length
+const length = 32
+
+// String returns a string value.
+func String() string {
+	clen := len(chars)
+	maxrb := 255 - (256 % clen)
+	b := make([]byte, length)
+	r := make([]byte, length+(length/4)) // storage for random bytes.
+	i := 0
+	for {
+		if _, err := rand.Read(r); err != nil {
+			panic("rand: error reading random bytes")
+		}
+		for _, rb := range r {
+			c := int(rb)
+			if c > maxrb {
+				// Skip this number to avoid modulo bias.
+				continue
+			}
+			b[i] = chars[c%clen]
+			i++
+			if i == length {
+				return string(b)
+			}
+		}
+	}
+}

yaml/compiler/script_posix.go

@@ -0,0 +1,70 @@
+package compiler
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/drone/drone-yaml/yaml"
+	"github.com/drone/drone-yaml/yaml/compiler/internal/rand"
+)
+
+func setupScript(spec *engine.Spec, dst *engine.Step, src *yaml.Container) {
+	var buf bytes.Buffer
+	for _, command := range src.Commands {
+		escaped := fmt.Sprintf("%q", command)
+		escaped = strings.Replace(escaped, "$", `\$`, -1)
+		buf.WriteString(fmt.Sprintf(
+			traceScript,
+			escaped,
+			command,
+		))
+	}
+	script := fmt.Sprintf(
+		buildScript,
+		buf.String(),
+	)
+	spec.Files = append(spec.Files, &engine.File{
+		Metadata: engine.Metadata{
+			UID:       rand.String(),
+			Namespace: spec.Metadata.Namespace,
+			Name:      src.Name,
+		},
+		Data: []byte(script),
+	})
+	dst.Files = append(dst.Files, &engine.FileMount{
+		Name: src.Name,
+		Path: "/usr/drone/bin/init",
+		Mode: 0777,
+	})
+
+	dst.Docker.Command = []string{"/bin/sh"}
+	dst.Docker.Args = []string{"/usr/drone/bin/init"}
+}
+
+// buildScript is a helper script this is added to the build
+// to prepare the environment and execute the build commands.
+const buildScript = `
+if [ -n "$CI_NETRC_MACHINE" ]; then
+cat <<EOF > $HOME/.netrc
+machine $CI_NETRC_MACHINE
+login $CI_NETRC_USERNAME
+password $CI_NETRC_PASSWORD
+EOF
+chmod 0600 $HOME/.netrc
+fi
+unset CI_NETRC_USERNAME
+unset CI_NETRC_PASSWORD
+unset DRONE_NETRC_USERNAME
+unset DRONE_NETRC_PASSWORD
+set -e
+%s
+`
+
+// traceScript is a helper script that is added to
+// the build script to trace a command.
+const traceScript = `
+echo + %s
+%s
+`

yaml/compiler/script_posix_test.go

@@ -0,0 +1 @@
+package compiler

yaml/compiler/script_win.go

@@ -0,0 +1,58 @@
+package compiler
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"strings"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/drone/drone-yaml/yaml"
+)
+
+func setupScriptWin(spec *engine.Spec, dst *engine.Step, src *yaml.Container) {
+	var buf bytes.Buffer
+	for _, command := range src.Commands {
+		escaped := fmt.Sprintf("%q", command)
+		escaped = strings.Replace(escaped, "$", `\$`, -1)
+		buf.WriteString(fmt.Sprintf(
+			traceScriptWin,
+			escaped,
+			command,
+		))
+	}
+	script := fmt.Sprintf(
+		buildScriptWin,
+		buf.String(),
+	)
+	dst.Docker.Command = []string{"powershell", "-noprofile", "-noninteractive", "-command"}
+	dst.Docker.Args = []string{"[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Env:CI_SCRIPT)) | iex"}
+	dst.Envs["CI_SCRIPT"] = base64.StdEncoding.EncodeToString([]byte(script))
+	dst.Envs["SHELL"] = "powershell.exe"
+}
+
+// buildScriptWin is a helper script this is added to the build
+// to prepare the environment and execute the build commands.
+const buildScriptWin = `
+if ($Env:CI_NETRC_MACHINE) {
+@"
+machine $Env:CI_NETRC_MACHINE
+login $Env:CI_NETRC_USERNAME
+password $Env:CI_NETRC_PASSWORD
+"@ > (Join-Path $Env:USERPROFILE '_netrc');
+}
+[Environment]::SetEnvironmentVariable("CI_NETRC_USERNAME", $null);
+[Environment]::SetEnvironmentVariable("CI_NETRC_PASSWORD", $null);
+[Environment]::SetEnvironmentVariable("DRONE_NETRC_USERNAME", $null);
+[Environment]::SetEnvironmentVariable("DRONE_NETRC_PASSWORD", $null);
+[Environment]::SetEnvironmentVariable("CI_SCRIPT", $null);
+$ErrorActionPreference = 'Stop';
+%s
+`
+
+// traceScriptWin is a helper script that is added to
+// the build script to trace a command.
+const traceScriptWin = `
+Write-Output ('+ %s');
+& %s; if ($LASTEXITCODE -ne 0) {exit $LASTEXITCODE}
+`

yaml/compiler/script_win_test.go

@@ -0,0 +1 @@
+package compiler

yaml/compiler/skip.go

@@ -0,0 +1,37 @@
+package compiler
+
+import "github.com/drone/drone-yaml/yaml"
+
+// SkipData provides build metadata use to determine if a
+// pipeline step should be skipped.
+type SkipData struct {
+	Branch   string
+	Event    string
+	Instance string
+	Ref      string
+	Repo     string
+	Target   string
+}
+
+// SkipFunc returns a function that can be used to skip
+// individual pipeline steps based on build metadata.
+func SkipFunc(data SkipData) func(*yaml.Container) bool {
+	return func(container *yaml.Container) bool {
+		switch {
+		case !container.When.Branch.Match(data.Branch):
+			return true
+		case !container.When.Event.Match(data.Event):
+			return true
+		case !container.When.Instance.Match(data.Instance):
+			return true
+		case !container.When.Ref.Match(data.Ref):
+			return true
+		case !container.When.Repo.Match(data.Repo):
+			return true
+		case !container.When.Target.Match(data.Target):
+			return true
+		default:
+			return false
+		}
+	}
+}

yaml/compiler/skip_test.go

@@ -0,0 +1,105 @@
+package compiler
+
+import (
+	"testing"
+
+	"github.com/drone/drone-yaml/yaml"
+)
+
+func TestSkipFunc(t *testing.T) {
+	tests := []struct {
+		data SkipData
+		when yaml.Conditions
+		want bool
+	}{
+		//
+		// test branch conditions
+		//
+		{
+			data: SkipData{Branch: "master"},
+			when: yaml.Conditions{Branch: yaml.Condition{Include: []string{"master"}}},
+			want: false,
+		},
+		{
+			data: SkipData{Branch: "master"},
+			when: yaml.Conditions{Branch: yaml.Condition{Exclude: []string{"master"}}},
+			want: true,
+		},
+		//
+		// test event conditions
+		//
+		{
+			data: SkipData{Event: "push"},
+			when: yaml.Conditions{Event: yaml.Condition{Include: []string{"push"}}},
+			want: false,
+		},
+
+		{
+			data: SkipData{Event: "push"},
+			when: yaml.Conditions{Event: yaml.Condition{Exclude: []string{"push"}}},
+			want: true,
+		},
+		//
+		// test instance conditions
+		//
+		{
+			data: SkipData{Instance: "drone.company.com"},
+			when: yaml.Conditions{Instance: yaml.Condition{Include: []string{"drone.company.com"}}},
+			want: false,
+		},
+
+		{
+			data: SkipData{Instance: "drone.company.com"},
+			when: yaml.Conditions{Instance: yaml.Condition{Exclude: []string{"drone.company.com"}}},
+			want: true,
+		},
+		//
+		// test ref conditions
+		//
+		{
+			data: SkipData{Ref: "refs/heads/master"},
+			when: yaml.Conditions{Ref: yaml.Condition{Include: []string{"refs/heads/*"}}},
+			want: false,
+		},
+
+		{
+			data: SkipData{Ref: "refs/heads/master"},
+			when: yaml.Conditions{Ref: yaml.Condition{Exclude: []string{"refs/heads/*"}}},
+			want: true,
+		},
+		//
+		// test repo conditions
+		//
+		{
+			data: SkipData{Repo: "octocat/hello-world"},
+			when: yaml.Conditions{Repo: yaml.Condition{Include: []string{"octocat/hello-world"}}},
+			want: false,
+		},
+
+		{
+			data: SkipData{Repo: "octocat/hello-world"},
+			when: yaml.Conditions{Repo: yaml.Condition{Exclude: []string{"octocat/hello-world"}}},
+			want: true,
+		},
+		//
+		// test target conditions
+		//
+		{
+			data: SkipData{Target: "prod"},
+			when: yaml.Conditions{Target: yaml.Condition{Include: []string{"prod"}}},
+			want: false,
+		},
+		{
+			data: SkipData{Target: "prod"},
+			when: yaml.Conditions{Target: yaml.Condition{Exclude: []string{"prod"}}},
+			want: true,
+		},
+	}
+	for i, test := range tests {
+		container := &yaml.Container{When: test.when}
+		got := SkipFunc(test.data)(container)
+		if got != test.want {
+			t.Errorf("Want skip %v at index %d", test.want, i)
+		}
+	}
+}

yaml/compiler/step.go

@@ -0,0 +1,180 @@
+package compiler
+
+import (
+	"strings"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/drone/drone-yaml/yaml"
+	"github.com/drone/drone-yaml/yaml/compiler/image"
+	"github.com/drone/drone-yaml/yaml/compiler/internal/rand"
+)
+
+func createStep(spec *engine.Spec, src *yaml.Container) *engine.Step {
+	dst := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:       rand.String(),
+			Name:      src.Name,
+			Namespace: spec.Metadata.Namespace,
+			Labels: map[string]string{
+				"io.drone.step.name": src.Name,
+			},
+		},
+		Detach:    src.Detach,
+		DependsOn: src.DependsOn,
+		Devices:   nil,
+		Docker: &engine.DockerStep{
+			Args:       src.Command,
+			Command:    src.Entrypoint,
+			DNS:        src.DNS,
+			DNSSearch:  src.DNSSearch,
+			ExtraHosts: src.ExtraHosts,
+			Image:      image.Expand(src.Image),
+			Networks:   nil, // set in compiler.go
+			Ports:      toPorts(src),
+			Privileged: src.Privileged,
+			PullPolicy: toPullPolicy(src),
+		},
+		Envs:         map[string]string{},
+		Files:        nil, // set below
+		IgnoreErr:    toIgnoreErr(src),
+		IgnoreStderr: false,
+		IgnoreStdout: false,
+		Resources:    toResources(src),
+		RunPolicy:    toRunPolicy(src),
+		Secrets:      nil, // set below
+		Volumes:      nil, // set below
+		WorkingDir:   "",  // set in compiler.go
+	}
+
+	// if the user is running a service container with
+	// no custom commands, we should revert back to the
+	// user-defined working directory, which may be empty.
+	if dst.Detach && len(src.Commands) == 0 {
+		dst.WorkingDir = src.WorkingDir
+	}
+
+	// appends the volumes to the container def.
+	for _, vol := range src.Volumes {
+		// the user should never be able to directly
+		// mount the docker socket. This should be
+		// restricted by the linter, but we place this
+		// check here just to be safe.
+		if vol.Name == "_docker_socket" {
+			continue
+		}
+		mount := &engine.VolumeMount{
+			Name: vol.Name,
+			Path: vol.MountPath,
+		}
+		dst.Volumes = append(dst.Volumes, mount)
+	}
+
+	// appends the environment variables to the
+	// container definition.
+	for key, value := range src.Environment {
+		if value.Secret != "" {
+			sec := &engine.SecretVar{
+				Name: value.Secret,
+				Env:  key,
+			}
+			dst.Secrets = append(dst.Secrets, sec)
+		} else {
+			dst.Envs[key] = value.Value
+		}
+	}
+
+	// appends the settings variables to the
+	// container definition.
+	for key, value := range src.Settings {
+		// all settings are passed to the plugin env
+		// variables, prefixed with PLUGIN_
+		key = "PLUGIN_" + strings.ToUpper(key)
+
+		// if the setting parameter is sources from the
+		// secret we create a secret enviornment variable.
+		if value.Secret != "" {
+			sec := &engine.SecretVar{
+				Name: value.Secret,
+				Env:  key,
+			}
+			dst.Secrets = append(dst.Secrets, sec)
+		} else {
+			// else if the setting parameter is opaque
+			// we inject as a string-encoded environment
+			// variable.
+			dst.Envs[key] = encode(value.Value)
+		}
+	}
+
+	// if the step specifies shell commands we generate a
+	// script. The script is copied to the container at
+	// runtime (or mounted as a config map) and then executed
+	// as the entrypoint.
+	if len(src.Commands) > 0 {
+		switch spec.Platform.OS {
+		case "windows":
+			setupScriptWin(spec, dst, src)
+		default:
+			setupScript(spec, dst, src)
+		}
+	}
+
+	return dst
+}
+
+func createBuildStep(spec *engine.Spec, src *yaml.Container) *engine.Step {
+	dst := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:       rand.String(),
+			Name:      src.Name,
+			Namespace: spec.Metadata.Namespace,
+			Labels: map[string]string{
+				"io.drone.step.name": src.Name,
+			},
+		},
+		Detach:    src.Detach,
+		DependsOn: src.DependsOn,
+		Devices:   nil,
+		Docker: &engine.DockerStep{
+			Args:       []string{"--build"},
+			DNS:        src.DNS,
+			Image:      "drone/docker",
+			PullPolicy: engine.PullIfNotExists,
+		},
+		Envs:         map[string]string{},
+		IgnoreErr:    toIgnoreErr(src),
+		IgnoreStderr: false,
+		IgnoreStdout: false,
+		Resources:    toResources(src),
+		RunPolicy:    toRunPolicy(src),
+	}
+
+	// if v := src.Build.Args; len(v) > 0 {
+	// 	dst.Envs["DOCKER_BUILD_ARGS"] = strings.Join(v, ",")
+	// }
+	if v := src.Build.CacheFrom; len(v) > 0 {
+		dst.Envs["DOCKER_BUILD_CACHE_FROM"] = strings.Join(v, ",")
+	}
+	// if len(src.Build.Labels) > 0 {
+	// 	dst.Envs["DOCKER_BUILD_LABELS"] = strings.Join(v, ",")
+	// }
+	if v := src.Build.Dockerfile; v != "" {
+		dst.Envs["DOCKER_BUILD_DOCKERFILE"] = v
+
+	}
+	if v := src.Build.Context; v != "" {
+		dst.Envs["DOCKER_BUILD_CONTEXT"] = v
+	}
+	if v := src.Build.Image; v != "" {
+		alias := image.Trim(v) + ":" + dst.Metadata.UID
+		dst.Envs["DOCKER_BUILD_IMAGE"] = image.Expand(v)
+		dst.Envs["DOCKER_BUILD_IMAGE_ALIAS"] = image.Expand(alias)
+	}
+
+	dst.Volumes = append(dst.Volumes, &engine.VolumeMount{
+		Name: "_docker_socket",
+		Path: "/var/run/docker.sock",
+	})
+
+	return dst
+}

yaml/compiler/transform/auths.go

@@ -0,0 +1,29 @@
+package transform
+
+import "github.com/drone/drone-runtime/engine"
+
+// WithAuths is a transform function that adds a set
+// of global registry credentials to the container.
+func WithAuths(auths []*engine.DockerAuth) func(*engine.Spec) {
+	return func(spec *engine.Spec) {
+		for _, auth := range auths {
+			spec.Docker.Auths = append(spec.Docker.Auths, auth)
+		}
+	}
+}
+
+// AuthsFunc is a callback function used to request
+// registry credentials to pull private images.
+type AuthsFunc func() []*engine.DockerAuth
+
+// WithAuthsFunc is a transform function that provides
+// the sepcification with registry authentication
+// credentials via a callback function.
+func WithAuthsFunc(f AuthsFunc) func(*engine.Spec) {
+	return func(spec *engine.Spec) {
+		auths := f()
+		if len(auths) != 0 {
+			spec.Docker.Auths = append(spec.Docker.Auths, auths...)
+		}
+	}
+}

yaml/compiler/transform/auths_test.go

@@ -0,0 +1,49 @@
+package transform
+
+import (
+	"testing"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestWithAuths(t *testing.T) {
+	spec := &engine.Spec{
+		Steps:  []*engine.Step{},
+		Docker: &engine.DockerConfig{},
+	}
+	auths := []*engine.DockerAuth{
+		{
+			Address:  "docker.io",
+			Username: "octocat",
+			Password: "correct-horse-battery-staple",
+		},
+	}
+	WithAuths(auths)(spec)
+	if diff := cmp.Diff(auths, spec.Docker.Auths); diff != "" {
+		t.Errorf("Unexpected auths transform")
+		t.Log(diff)
+	}
+}
+
+func TestWithAuthsFunc(t *testing.T) {
+	spec := &engine.Spec{
+		Steps:  []*engine.Step{},
+		Docker: &engine.DockerConfig{},
+	}
+	auths := []*engine.DockerAuth{
+		{
+			Address:  "docker.io",
+			Username: "octocat",
+			Password: "correct-horse-battery-staple",
+		},
+	}
+	fn := func() []*engine.DockerAuth {
+		return auths
+	}
+	WithAuthsFunc(fn)(spec)
+	if diff := cmp.Diff(auths, spec.Docker.Auths); diff != "" {
+		t.Errorf("Unexpected auths transform")
+		t.Log(diff)
+	}
+}

yaml/compiler/transform/combine.go

@@ -0,0 +1,13 @@
+package transform
+
+import "github.com/drone/drone-runtime/engine"
+
+// Combine is a transform function that combines
+// one or many transform functions.
+func Combine(fns ...func(*engine.Spec)) func(*engine.Spec) {
+	return func(spec *engine.Spec) {
+		for _, fn := range fns {
+			fn(spec)
+		}
+	}
+}

yaml/compiler/transform/combine_test.go

@@ -0,0 +1,33 @@
+package transform
+
+import (
+	"testing"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestCombine(t *testing.T) {
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:  "1",
+			Name: "build",
+		},
+		Envs: map[string]string{},
+	}
+	spec := &engine.Spec{
+		Steps: []*engine.Step{step},
+	}
+	Combine(
+		WithEnviron(map[string]string{"GOOS": "linux"}),
+		WithEnviron(map[string]string{"GOARCH": "amd64"}),
+	)(spec)
+	want := map[string]string{
+		"GOOS":   "linux",
+		"GOARCH": "amd64",
+	}
+	if diff := cmp.Diff(want, step.Envs); diff != "" {
+		t.Errorf("Unexpected transform")
+		t.Log(diff)
+	}
+}

yaml/compiler/transform/env.go

@@ -0,0 +1,15 @@
+package transform
+
+import "github.com/drone/drone-runtime/engine"
+
+// WithEnviron is a transform function that adds a set
+// of environment variables to each container.
+func WithEnviron(envs map[string]string) func(*engine.Spec) {
+	return func(spec *engine.Spec) {
+		for key, value := range envs {
+			for _, step := range spec.Steps {
+				step.Envs[key] = value
+			}
+		}
+	}
+}

yaml/compiler/transform/env_test.go

@@ -0,0 +1,30 @@
+package transform
+
+import (
+	"testing"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestWithEnviron(t *testing.T) {
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:  "1",
+			Name: "build",
+		},
+		Envs: map[string]string{},
+	}
+	spec := &engine.Spec{
+		Steps: []*engine.Step{step},
+	}
+	envs := map[string]string{
+		"GOOS":   "linux",
+		"GOARCH": "amd64",
+	}
+	WithEnviron(envs)(spec)
+	if diff := cmp.Diff(envs, step.Envs); diff != "" {
+		t.Errorf("Unexpected transform")
+		t.Log(diff)
+	}
+}

yaml/compiler/transform/filter.go

@@ -0,0 +1,72 @@
+package transform
+
+import "github.com/drone/drone-runtime/engine"
+
+// Include is a transform function that limits the
+// pipeline execution to a whitelist of named steps.
+func Include(names []string) func(*engine.Spec) {
+	set := map[string]struct{}{}
+	for _, name := range names {
+		set[name] = struct{}{}
+	}
+	return func(spec *engine.Spec) {
+		if len(names) == 0 {
+			return
+		}
+		for _, step := range spec.Steps {
+			if step.Metadata.Name == "clone" {
+				continue
+			}
+			_, ok := set[step.Metadata.Name]
+			if !ok {
+				// if the step is not included in the
+				// whitelist the run policy is set to never.
+				step.RunPolicy = engine.RunNever
+			}
+		}
+	}
+}
+
+// Exclude is a transform function that limits the
+// pipeline execution to a whitelist of named steps.
+func Exclude(names []string) func(*engine.Spec) {
+	set := map[string]struct{}{}
+	for _, name := range names {
+		set[name] = struct{}{}
+	}
+	return func(spec *engine.Spec) {
+		if len(names) == 0 {
+			return
+		}
+		for _, step := range spec.Steps {
+			if step.Metadata.Name == "clone" {
+				continue
+			}
+			_, ok := set[step.Metadata.Name]
+			if ok {
+				// if the step is included in the blacklist
+				// the run policy is set to never.
+				step.RunPolicy = engine.RunNever
+			}
+		}
+	}
+}
+
+// ResumeAt is a transform function that modifies the
+// exuction to resume at a named step.
+func ResumeAt(name string) func(*engine.Spec) {
+	return func(spec *engine.Spec) {
+		if name == "" {
+			return
+		}
+		for _, step := range spec.Steps {
+			if step.Metadata.Name == name {
+				return
+			}
+			if step.Metadata.Name == "clone" {
+				continue
+			}
+			step.RunPolicy = engine.RunNever
+		}
+	}
+}

yaml/compiler/transform/filter_test.go

@@ -0,0 +1,139 @@
+package transform
+
+import (
+	"testing"
+
+	"github.com/drone/drone-runtime/engine"
+)
+
+func TestInclude(t *testing.T) {
+	step1 := &engine.Step{
+		Metadata:  engine.Metadata{Name: "clone"},
+		RunPolicy: engine.RunOnSuccess,
+	}
+	step2 := &engine.Step{
+		Metadata:  engine.Metadata{Name: "build"},
+		RunPolicy: engine.RunOnSuccess,
+	}
+	step3 := &engine.Step{
+		Metadata:  engine.Metadata{Name: "test"},
+		RunPolicy: engine.RunOnSuccess,
+	}
+	spec := &engine.Spec{
+		Metadata: engine.Metadata{},
+		Steps:    []*engine.Step{step1, step2, step3},
+	}
+	Include([]string{"test"})(spec)
+	if got, want := step1.RunPolicy, engine.RunOnSuccess; got != want {
+		t.Errorf("Want run policy %s got %s", want, got)
+	}
+	if got, want := step2.RunPolicy, engine.RunNever; got != want {
+		t.Errorf("Want run policy %s got %s", want, got)
+	}
+	if got, want := step3.RunPolicy, engine.RunOnSuccess; got != want {
+		t.Errorf("Want run policy %s got %s", want, got)
+	}
+}
+
+func TestInclude_Empty(t *testing.T) {
+	step := &engine.Step{
+		Metadata:  engine.Metadata{},
+		RunPolicy: engine.RunOnSuccess,
+	}
+	spec := &engine.Spec{
+		Metadata: engine.Metadata{},
+		Steps:    []*engine.Step{step},
+	}
+	Include(nil)(spec)
+	if got, want := step.RunPolicy, engine.RunOnSuccess; got != want {
+		t.Errorf("Want run policy %s got %s", want, got)
+	}
+}
+
+func TestExclude(t *testing.T) {
+	step1 := &engine.Step{
+		Metadata:  engine.Metadata{Name: "clone"},
+		RunPolicy: engine.RunOnSuccess,
+	}
+	step2 := &engine.Step{
+		Metadata:  engine.Metadata{Name: "build"},
+		RunPolicy: engine.RunOnSuccess,
+	}
+	step3 := &engine.Step{
+		Metadata:  engine.Metadata{Name: "test"},
+		RunPolicy: engine.RunOnSuccess,
+	}
+	spec := &engine.Spec{
+		Metadata: engine.Metadata{},
+		Steps:    []*engine.Step{step1, step2, step3},
+	}
+	Exclude([]string{"test"})(spec)
+	if got, want := step1.RunPolicy, engine.RunOnSuccess; got != want {
+		t.Errorf("Want run policy %s got %s", want, got)
+	}
+	if got, want := step2.RunPolicy, engine.RunOnSuccess; got != want {
+		t.Errorf("Want run policy %s got %s", want, got)
+	}
+	if got, want := step3.RunPolicy, engine.RunNever; got != want {
+		t.Errorf("Want run policy %s got %s", want, got)
+	}
+}
+
+func TestExclude_Empty(t *testing.T) {
+	step := &engine.Step{
+		Metadata:  engine.Metadata{},
+		RunPolicy: engine.RunOnSuccess,
+	}
+	spec := &engine.Spec{
+		Metadata: engine.Metadata{},
+		Steps:    []*engine.Step{step},
+	}
+	Exclude(nil)(spec)
+	if got, want := step.RunPolicy, engine.RunOnSuccess; got != want {
+		t.Errorf("Want run policy %s got %s", want, got)
+	}
+}
+
+func TestResumeAt(t *testing.T) {
+	step1 := &engine.Step{
+		Metadata:  engine.Metadata{Name: "clone"},
+		RunPolicy: engine.RunOnSuccess,
+	}
+	step2 := &engine.Step{
+		Metadata:  engine.Metadata{Name: "build"},
+		RunPolicy: engine.RunOnSuccess,
+	}
+	step3 := &engine.Step{
+		Metadata:  engine.Metadata{Name: "test"},
+		RunPolicy: engine.RunOnSuccess,
+	}
+	spec := &engine.Spec{
+		Metadata: engine.Metadata{},
+		Steps:    []*engine.Step{step1, step2, step3},
+	}
+	ResumeAt("test")(spec)
+	if got, want := step1.RunPolicy, engine.RunOnSuccess; got != want {
+		t.Errorf("Want run policy %s got %s", want, got)
+	}
+	if got, want := step2.RunPolicy, engine.RunNever; got != want {
+		t.Errorf("Want run policy %s got %s", want, got)
+	}
+	if got, want := step3.RunPolicy, engine.RunOnSuccess; got != want {
+		t.Errorf("Want run policy %s got %s", want, got)
+	}
+}
+
+func TestResume_Empty(t *testing.T) {
+	step := &engine.Step{
+		Metadata:  engine.Metadata{},
+		RunPolicy: engine.RunOnSuccess,
+	}
+	spec := &engine.Spec{
+		Metadata: engine.Metadata{},
+		Steps:    []*engine.Step{step},
+	}
+	ResumeAt("")(spec)
+	if got, want := step.RunPolicy, engine.RunOnSuccess; got != want {
+		t.Errorf("Want run policy %s got %s", want, got)
+	}
+}

yaml/compiler/transform/labels.go

@@ -0,0 +1,23 @@
+package transform
+
+import "github.com/drone/drone-runtime/engine"
+
+// WithLables is a transform function that adds a set
+// of labels to each resource.
+func WithLables(labels map[string]string) func(*engine.Spec) {
+	return func(spec *engine.Spec) {
+		for k, v := range labels {
+			spec.Metadata.Labels[k] = v
+		}
+		for _, resource := range spec.Docker.Volumes {
+			for k, v := range labels {
+				resource.Metadata.Labels[k] = v
+			}
+		}
+		for _, resource := range spec.Steps {
+			for k, v := range labels {
+				resource.Metadata.Labels[k] = v
+			}
+		}
+	}
+}

yaml/compiler/transform/labels_test.go

@@ -0,0 +1,48 @@
+package transform
+
+import (
+	"testing"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestWithLabels(t *testing.T) {
+	volume := &engine.Volume{
+		Metadata: engine.Metadata{
+			Labels: map[string]string{},
+		},
+	}
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			Labels: map[string]string{},
+		},
+		Envs: map[string]string{},
+	}
+	spec := &engine.Spec{
+		Metadata: engine.Metadata{
+			Labels: map[string]string{},
+		},
+		Steps: []*engine.Step{step},
+		Docker: &engine.DockerConfig{
+			Volumes: []*engine.Volume{volume},
+		},
+	}
+	labels := map[string]string{
+		"io.drone.build.number": "1",
+		"io.drone.build.event":  "push",
+	}
+	WithLables(labels)(spec)
+	if diff := cmp.Diff(labels, spec.Metadata.Labels); diff != "" {
+		t.Errorf("Unexpected spec labels")
+		t.Log(diff)
+	}
+	if diff := cmp.Diff(labels, step.Metadata.Labels); diff != "" {
+		t.Errorf("Unexpected step labels")
+		t.Log(diff)
+	}
+	if diff := cmp.Diff(labels, volume.Metadata.Labels); diff != "" {
+		t.Errorf("Unexpected volume labels")
+		t.Log(diff)
+	}
+}

yaml/compiler/transform/limits.go

@@ -0,0 +1,29 @@
+package transform
+
+import "github.com/drone/drone-runtime/engine"
+// WithLimits is a transform function that applies
+// resource limits to the container processes.
+func WithLimits(memlimit int64, cpulimit int64) func(*engine.Spec) {
+	return func(spec *engine.Spec) {
+		// if no limits are defined exit immediately.
+		if memlimit == 0 && cpulimit == 0 {
+			return
+		}
+		// otherwise apply the resource limits to every
+		// step in the runtime spec.
+		for _, step := range spec.Steps {
+			if step.Resources == nil {
+				step.Resources = &engine.Resources{}
+			}
+			if step.Resources.Limits == nil {
+				step.Resources.Limits = &engine.ResourceObject{}
+			}
+			if memlimit != 0 {
+				step.Resources.Limits.Memory = memlimit
+			}
+			if cpulimit != 0 {
+				step.Resources.Limits.CPU = cpulimit * 1000
+			}
+		}
+	}
+}

yaml/compiler/transform/limits_test.go

@@ -0,0 +1,84 @@
+package transform
+
+import (
+	"testing"
+
+	"github.com/drone/drone-runtime/engine"
+)
+
+func TestWithLimits(t *testing.T) {
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:  "1",
+			Name: "build",
+		},
+		Docker: &engine.DockerStep{},
+	}
+	spec := &engine.Spec{
+		Steps: []*engine.Step{step},
+	}
+	WithLimits(1, 2)(spec)
+	if got, want := step.Resources.Limits.Memory, int64(1); got != want {
+		t.Errorf("Want memory limit %v, got %v", want, got)
+	}
+	if got, want := step.Resources.Limits.CPU, int64(2000); got != want {
+		t.Errorf("Want cpu limit %v, got %v", want, got)
+	}
+}
+
+func TestWithMemory(t *testing.T) {
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:  "1",
+			Name: "build",
+		},
+		Docker: &engine.DockerStep{},
+	}
+	spec := &engine.Spec{
+		Steps: []*engine.Step{step},
+	}
+	WithLimits(1, 0)(spec)
+	if got, want := step.Resources.Limits.Memory, int64(1); got != want {
+		t.Errorf("Want memory limit %v, got %v", want, got)
+	}
+	if got, want := step.Resources.Limits.CPU, int64(0); got != want {
+		t.Errorf("Want cpu limit %v, got %v", want, got)
+	}
+}
+
+func TestWithCPU(t *testing.T) {
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:  "1",
+			Name: "build",
+		},
+		Docker: &engine.DockerStep{},
+	}
+	spec := &engine.Spec{
+		Steps: []*engine.Step{step},
+	}
+	WithLimits(0, 3)(spec)
+	if got, want := step.Resources.Limits.Memory, int64(0); got != want {
+		t.Errorf("Want memory limit %v, got %v", want, got)
+	}
+	if got, want := step.Resources.Limits.CPU, int64(3000); got != want {
+		t.Errorf("Want cpu limit %v, got %v", want, got)
+	}
+}
+
+func TestWithNoLimits(t *testing.T) {
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:  "1",
+			Name: "build",
+		},
+		Docker: &engine.DockerStep{},
+	}
+	spec := &engine.Spec{
+		Steps: []*engine.Step{step},
+	}
+	WithLimits(0, 0)(spec)
+	if step.Resources != nil {
+		t.Errorf("Expect no limits applied")
+	}
+}

yaml/compiler/transform/netrc.go

@@ -0,0 +1,73 @@
+package transform
+
+import (
+	"fmt"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/drone/drone-yaml/yaml/compiler/internal/rand"
+)
+
+const (
+	netrcName = ".netrc"
+	netrcPath = "/var/run/drone/.netrc"
+	netrcMode = 0777
+)
+
+const disableNetrcMount = true
+
+// WithNetrc is a helper function that creates a netrc file
+// and mounts the file to all container steps.
+func WithNetrc(machine, username, password string) func(*engine.Spec) {
+	return func(spec *engine.Spec) {
+		if username == "" || password == "" {
+			return
+		}
+
+		// TODO(bradrydzewski) temporarily disable mounting
+		// the netrc file due to issues with kubernetes
+		// compatibility.
+		if disableNetrcMount == false {
+			// Currently file mounts don't seem to work in Windows so environment
+			// variables are used instead
+			// FIXME: https://github.com/drone/drone-yaml/issues/20
+			if spec.Platform.OS != "windows" {
+				netrc := generateNetrc(machine, username, password)
+				spec.Files = append(spec.Files, &engine.File{
+					Metadata: engine.Metadata{
+						UID:       rand.String(),
+						Name:      netrcName,
+						Namespace: spec.Metadata.Namespace,
+					},
+					Data: []byte(netrc),
+				})
+				for _, step := range spec.Steps {
+					step.Files = append(step.Files, &engine.FileMount{
+						Name: netrcName,
+						Path: netrcPath,
+						Mode: netrcMode,
+					})
+				}
+			}
+		}
+
+		// TODO(bradrydzewski) these should only be injected
+		// if the file is not mounted, if OS == Windows.
+		for _, step := range spec.Steps {
+			if step.Envs == nil {
+				step.Envs = map[string]string{}
+			}
+			step.Envs["CI_NETRC_MACHINE"] = machine
+			step.Envs["CI_NETRC_USERNAME"] = username
+			step.Envs["CI_NETRC_PASSWORD"] = password
+
+			step.Envs["DRONE_NETRC_MACHINE"] = machine
+			step.Envs["DRONE_NETRC_USERNAME"] = username
+			step.Envs["DRONE_NETRC_PASSWORD"] = password
+		}
+	}
+}
+
+func generateNetrc(machine, username, password string) string {
+	return fmt.Sprintf("machine %s login %s password %s",
+		machine, username, password)
+}

yaml/compiler/transform/netrc_test.go

@@ -0,0 +1,78 @@
+package transform
+
+import (
+	"testing"
+
+	"github.com/drone/drone-runtime/engine"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+)
+
+var ignoreMetadata = cmpopts.IgnoreFields(
+	engine.Metadata{}, "UID")
+
+func TestWithNetrc(t *testing.T) {
+	if true {
+		t.Skipf("mounting the netrc is temporarily disabled")
+		return
+	}
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:  "1",
+			Name: "build",
+		},
+	}
+	spec := &engine.Spec{
+		Metadata: engine.Metadata{
+			UID:       "acdj0yjqv7uh5hidveg0ggr42x8oj67b",
+			Namespace: "pivqfthg1c9hy83ylht1sxx4nygjc7tk",
+		},
+		Steps: []*engine.Step{step},
+	}
+	WithNetrc("@machine", "@username", "@password")(spec)
+	if len(step.Files) == 0 {
+		t.Errorf("File mount not added to step")
+		return
+	}
+	if len(spec.Files) == 0 {
+		t.Errorf("File not declared in spec")
+		return
+	}
+	file := &engine.File{
+		Metadata: engine.Metadata{
+			Name:      ".netrc",
+			Namespace: "pivqfthg1c9hy83ylht1sxx4nygjc7tk",
+		},
+		Data: []byte("machine @machine login @username password @password"),
+	}
+	if diff := cmp.Diff(file, spec.Files[0], ignoreMetadata); diff != "" {
+		t.Errorf("Unexpected file declaration")
+		t.Log(diff)
+	}
+
+	fileMount := &engine.FileMount{Name: ".netrc", Path: "/root/.netrc", Mode: 0600}
+	if diff := cmp.Diff(fileMount, step.Files[0], ignoreMetadata); diff != "" {
+		t.Errorf("Unexpected file mount")
+		t.Log(diff)
+	}
+}
+
+func TestWithEmptyNetrc(t *testing.T) {
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:  "1",
+			Name: "build",
+		},
+	}
+	spec := &engine.Spec{
+		Steps: []*engine.Step{step},
+	}
+	WithNetrc("@machine", "", "")(spec)
+	if len(spec.Files) != 0 {
+		t.Errorf("Unexpected file declaration")
+	}
+	if len(step.Files) != 0 {
+		t.Errorf("Unexpected file mount")
+	}
+}

yaml/compiler/transform/network.go

@@ -0,0 +1,14 @@
+package transform
+
+import "github.com/drone/drone-runtime/engine"
+
+// WithNetworks is a transform function that attaches a
+// list of user-defined Docker networks to each step.
+func WithNetworks(networks []string) func(*engine.Spec) {
+	return func(spec *engine.Spec) {
+		for _, step := range spec.Steps {
+			step.Docker.Networks = append(
+				step.Docker.Networks, networks...)
+		}
+	}
+}

yaml/compiler/transform/network_test.go

@@ -0,0 +1,29 @@
+package transform
+
+import (
+	"testing"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestWithNetwork(t *testing.T) {
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:  "1",
+			Name: "build",
+		},
+		Docker: &engine.DockerStep{
+			Networks: nil,
+		},
+	}
+	spec := &engine.Spec{
+		Steps: []*engine.Step{step},
+	}
+	nets := []string{"a", "b"}
+	WithNetworks(nets)(spec)
+	if diff := cmp.Diff(nets, step.Docker.Networks); diff != "" {
+		t.Errorf("Unexpected transform")
+		t.Log(diff)
+	}
+}

yaml/compiler/transform/proxy.go

@@ -0,0 +1,39 @@
+package transform
+
+import (
+	"os"
+	"strings"
+
+	"github.com/drone/drone-runtime/engine"
+)
+
+// WithProxy is a transform function that adds the
+// http_proxy environment variables to every container.
+func WithProxy() func(*engine.Spec) {
+	environ := map[string]string{}
+	if value := getenv("no_proxy"); value != "" {
+		environ["no_proxy"] = value
+		environ["NO_PROXY"] = value
+	}
+	if value := getenv("http_proxy"); value != "" {
+		environ["http_proxy"] = value
+		environ["HTTP_PROXY"] = value
+	}
+	if value := getenv("https_proxy"); value != "" {
+		environ["https_proxy"] = value
+		environ["HTTPS_PROXY"] = value
+	}
+	return WithEnviron(environ)
+}
+
+func getenv(name string) (value string) {
+	name = strings.ToUpper(name)
+	if value := os.Getenv(name); value != "" {
+		return value
+	}
+	name = strings.ToLower(name)
+	if value := os.Getenv(name); value != "" {
+		return value
+	}
+	return
+}

yaml/compiler/transform/proxy_test.go

@@ -0,0 +1,52 @@
+package transform
+
+import (
+	"os"
+	"testing"
+
+	"github.com/drone/drone-runtime/engine"
+)
+
+func TestWithProxy(t *testing.T) {
+	var (
+		noProxy    = getenv("no_proxy")
+		httpProxy  = getenv("https_proxy")
+		httpsProxy = getenv("https_proxy")
+	)
+	defer func() {
+		os.Setenv("no_proxy", noProxy)
+		os.Setenv("NO_PROXY", noProxy)
+		os.Setenv("http_proxy", httpProxy)
+		os.Setenv("HTTP_PROXY", httpProxy)
+		os.Setenv("HTTPS_PROXY", httpsProxy)
+		os.Setenv("https_proxy", httpsProxy)
+	}()
+
+	testdata := map[string]string{
+		"NO_PROXY":    "http://dummy.no.proxy",
+		"http_proxy":  "http://dummy.http.proxy",
+		"https_proxy": "http://dummy.https.proxy",
+	}
+
+	for k, v := range testdata {
+		os.Setenv(k, v)
+	}
+
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:  "1",
+			Name: "build",
+		},
+		Envs: map[string]string{},
+	}
+	spec := &engine.Spec{
+		Steps: []*engine.Step{step},
+	}
+	WithProxy()(spec)
+	for k, v := range testdata {
+		step := spec.Steps[0]
+		if step.Envs[k] != v {
+			t.Errorf("Expect proxy varaible %s=%q, got %q", k, v, step.Envs[k])
+		}
+	}
+}

yaml/compiler/transform/secret.go

@@ -0,0 +1,62 @@
+package transform
+
+import (
+	"github.com/drone/drone-runtime/engine"
+	"github.com/drone/drone-yaml/yaml/compiler/internal/rand"
+)
+
+// WithSecrets is a transform function that adds a set
+// of global secrets to the container.
+func WithSecrets(secrets map[string]string) func(*engine.Spec) {
+	return func(spec *engine.Spec) {
+		for key, value := range secrets {
+			spec.Secrets = append(spec.Secrets,
+				&engine.Secret{
+					Metadata: engine.Metadata{
+						UID:       rand.String(),
+						Name:      key,
+						Namespace: spec.Metadata.Namespace,
+					},
+					Data: value,
+				},
+			)
+		}
+	}
+}
+
+// SecretFunc is a callback function used to request
+// named secret, required by a pipeline step.
+type SecretFunc func(string) *engine.Secret
+
+// WithSecretFunc is a transform function that resolves
+// all named secrets through a callback function, and
+// adds the secrets to the specification.
+func WithSecretFunc(f SecretFunc) func(*engine.Spec) {
+	return func(spec *engine.Spec) {
+		// first we get a unique list of all secrets
+		// used by the specification.
+		set := map[string]struct{}{}
+		for _, step := range spec.Steps {
+			// if we know the step is not going to run,
+			// we can ignore any secrets that it requires.
+			if step.RunPolicy == engine.RunNever {
+				continue
+			}
+			for _, v := range step.Secrets {
+				set[v.Name] = struct{}{}
+			}
+		}
+
+		// next we use the callback function to
+		// get the value for each secret, and append
+		// to the specification.
+		for name := range set {
+			secret := f(name)
+			if secret != nil {
+				secret.Metadata.UID = rand.String()
+				secret.Metadata.Namespace = spec.Metadata.Namespace
+				spec.Secrets = append(spec.Secrets, secret)
+			}
+		}
+	}
+}

yaml/compiler/transform/secret_test.go

@@ -0,0 +1,107 @@
+package transform
+
+import (
+	"testing"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestWithSecret(t *testing.T) {
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:  "1",
+			Name: "build",
+		},
+		Envs: map[string]string{},
+	}
+	spec := &engine.Spec{
+		Metadata: engine.Metadata{
+			UID:       "acdj0yjqv7uh5hidveg0ggr42x8oj67b",
+			Namespace: "pivqfthg1c9hy83ylht1sxx4nygjc7tk",
+		},
+		Steps: []*engine.Step{step},
+	}
+	secrets := map[string]string{
+		"password": "correct-horse-battery-staple",
+	}
+	WithSecrets(secrets)(spec)
+
+	want := []*engine.Secret{
+		{
+			Metadata: engine.Metadata{
+				Name:      "password",
+				Namespace: "pivqfthg1c9hy83ylht1sxx4nygjc7tk",
+			},
+			Data: "correct-horse-battery-staple",
+		},
+	}
+	if diff := cmp.Diff(want, spec.Secrets, ignoreMetadata); diff != "" {
+		t.Errorf("Unexpected secret transform")
+		t.Log(diff)
+	}
+}
+
+func TestWithSecretFunc(t *testing.T) {
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:  "1",
+			Name: "build",
+		},
+		Envs: map[string]string{},
+		Secrets: []*engine.SecretVar{
+			{
+				Name: "password",
+				Env:  "PASSWORD",
+			},
+		},
+	}
+	spec := &engine.Spec{
+		Metadata: engine.Metadata{
+			UID:       "acdj0yjqv7uh5hidveg0ggr42x8oj67b",
+			Namespace: "pivqfthg1c9hy83ylht1sxx4nygjc7tk",
+		},
+		Steps: []*engine.Step{
+			step,
+			// this is a step that requests a secret
+			// but should be skipped.
+			{
+				RunPolicy: engine.RunNever,
+				Secrets: []*engine.SecretVar{
+					{
+						Name: "github_token",
+						Env:  "GITHUB_TOKEN",
+					},
+				},
+			},
+		},
+	}
+
+	fn := func(name string) *engine.Secret {
+		if name == "github_token" {
+			t.Errorf("Requested secret for skipped step")
+			return nil
+		}
+		return &engine.Secret{
+			Metadata: engine.Metadata{
+				Name: "password",
+			},
+			Data: "correct-horse-battery-staple",
+		}
+	}
+	WithSecretFunc(fn)(spec)
+
+	want := []*engine.Secret{
+		{
+			Metadata: engine.Metadata{
+				Name:      "password",
+				Namespace: "pivqfthg1c9hy83ylht1sxx4nygjc7tk",
+			},
+			Data: "correct-horse-battery-staple",
+		},
+	}
+	if diff := cmp.Diff(want, spec.Secrets, ignoreMetadata); diff != "" {
+		t.Errorf("Unexpected secret transform")
+		t.Log(diff)
+	}
+}

yaml/compiler/transform/volume.go

@@ -0,0 +1,53 @@
+package transform
+
+import (
+	"strings"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/drone/drone-yaml/yaml/compiler/internal/rand"
+)
+
+// WithVolumes is a transform function that adds a set
+// of global volumes to the container.
+func WithVolumes(volumes map[string]string) func(*engine.Spec) {
+	return func(spec *engine.Spec) {
+		for key, value := range volumes {
+			volume := &engine.Volume{
+				Metadata: engine.Metadata{
+					UID:       rand.String(),
+					Name:      rand.String(),
+					Namespace: spec.Metadata.Name,
+					Labels:    map[string]string{},
+				},
+				HostPath: &engine.VolumeHostPath{
+					Path: key,
+				},
+			}
+			spec.Docker.Volumes = append(spec.Docker.Volumes, volume)
+			for _, step := range spec.Steps {
+				mount := &engine.VolumeMount{
+					Name: volume.Metadata.Name,
+					Path: value,
+				}
+				step.Volumes = append(step.Volumes, mount)
+			}
+		}
+	}
+}
+
+// WithVolumeSlice is a transform function that adds a set
+// of global volumes to the container that are defined in
+// --volume=host:container format.
+func WithVolumeSlice(volumes []string) func(*engine.Spec) {
+	to := map[string]string{}
+	for _, s := range volumes {
+		parts := strings.Split(s, ":")
+		if len(parts) != 2 {
+			continue
+		}
+		key := parts[0]
+		val := parts[1]
+		to[key] = val
+	}
+	return WithVolumes(to)
+}

yaml/compiler/transform/volume_test.go

@@ -0,0 +1,69 @@
+package transform
+
+import (
+	"testing"
+
+	"github.com/drone/drone-runtime/engine"
+)
+
+func TestWithVolumes(t *testing.T) {
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:  "1",
+			Name: "build",
+		},
+		Docker: &engine.DockerStep{
+			Networks: nil,
+		},
+	}
+	spec := &engine.Spec{
+		Docker: &engine.DockerConfig{},
+		Steps:  []*engine.Step{step},
+	}
+	vols := map[string]string{"/path/on/host": "/path/in/container"}
+	WithVolumes(vols)(spec)
+
+	if len(step.Volumes) == 0 {
+		t.Error("Expected volume added to container")
+	}
+	if got, want := step.Volumes[0].Path, "/path/in/container"; got != want {
+		t.Errorf("Want mount path %s, got %s", want, got)
+	}
+	if len(spec.Docker.Volumes) == 0 {
+		t.Error("Expected volume added to spec")
+	}
+	if got, want := spec.Docker.Volumes[0].HostPath.Path, "/path/on/host"; got != want {
+		t.Errorf("Want host mount path %s, got %s", want, got)
+	}
+}
+
+func TestWithVolumeSlice(t *testing.T) {
+	step := &engine.Step{
+		Metadata: engine.Metadata{
+			UID:  "1",
+			Name: "build",
+		},
+		Docker: &engine.DockerStep{
+			Networks: nil,
+		},
+	}
+	spec := &engine.Spec{
+		Docker: &engine.DockerConfig{},
+		Steps:  []*engine.Step{step},
+	}
+	vols := []string{"/path/on/host:/path/in/container"}
+	WithVolumeSlice(vols)(spec)
+
+	if len(step.Volumes) == 0 {
+		t.Error("Expected volume added to container")
+	}
+	if got, want := step.Volumes[0].Path, "/path/in/container"; got != want {
+		t.Errorf("Want mount path %s, got %s", want, got)
+	}
+	if len(spec.Docker.Volumes) == 0 {
+		t.Error("Expected volume added to spec")
+	}
+	if got, want := spec.Docker.Volumes[0].HostPath.Path, "/path/on/host"; got != want {
+		t.Errorf("Want host mount path %s, got %s", want, got)
+	}
+}

yaml/compiler/workspace.go

@@ -0,0 +1,144 @@
+package compiler
+
+import (
+	unixpath "path"
+	"strings"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/drone/drone-yaml/yaml"
+	"github.com/drone/drone-yaml/yaml/compiler/internal/rand"
+)
+
+const (
+	workspacePath     = "/drone/src"
+	workspaceName     = "workspace"
+	workspaceHostName = "host"
+)
+
+func setupWorkingDir(src *yaml.Container, dst *engine.Step, path string) {
+	// if the working directory is already set
+	// do not alter.
+	if dst.WorkingDir != "" {
+		return
+	}
+	// if the user is running the container as a
+	// service (detached mode) with no commands, we
+	// should use the default working directory.
+	if dst.Detach && len(src.Commands) == 0 {
+		return
+	}
+	// else set the working directory.
+	dst.WorkingDir = path
+}
+
+// helper function appends the workspace base and
+// path to the step's list of environment variables.
+func setupWorkspaceEnv(step *engine.Step, base, path, full string) {
+	step.Envs["DRONE_WORKSPACE_BASE"] = base
+	step.Envs["DRONE_WORKSPACE_PATH"] = path
+	step.Envs["DRONE_WORKSPACE"] = full
+	step.Envs["CI_WORKSPACE_BASE"] = base
+	step.Envs["CI_WORKSPACE_PATH"] = path
+	step.Envs["CI_WORKSPACE"] = full
+}
+
+// helper function converts the path to a valid windows
+// path, including the default C drive.
+func toWindowsDrive(s string) string {
+	return "c:" + toWindowsPath(s)
+}
+
+// helper function converts the path to a valid windows
+// path, replacing backslashes with forward slashes.
+func toWindowsPath(s string) string {
+	return strings.Replace(s, "/", "\\", -1)
+}
+
+//
+//
+//
+
+func createWorkspace(from *yaml.Pipeline) (base, path, full string) {
+	base = from.Workspace.Base
+	path = from.Workspace.Path
+	if base == "" {
+		base = workspacePath
+	}
+	full = unixpath.Join(base, path)
+
+	if from.Platform.OS == "windows" {
+		base = toWindowsDrive(base)
+		path = toWindowsPath(path)
+		full = toWindowsDrive(full)
+	}
+	return base, path, full
+}
+
+//
+//
+//
+
+// CreateWorkspace creates the workspace volume as
+// an empty directory mount.
+func CreateWorkspace(spec *engine.Spec) {
+	spec.Docker.Volumes = append(spec.Docker.Volumes,
+		&engine.Volume{
+			Metadata: engine.Metadata{
+				UID:       rand.String(),
+				Name:      workspaceName,
+				Namespace: spec.Metadata.Namespace,
+				Labels:    map[string]string{},
+			},
+			EmptyDir: &engine.VolumeEmptyDir{},
+		},
+	)
+}
+
+// CreateHostWorkspace returns a WorkspaceFunc that
+// mounts a host machine volume as the pipeline
+// workspace.
+func CreateHostWorkspace(workdir string) func(*engine.Spec) {
+	return func(spec *engine.Spec) {
+		CreateWorkspace(spec)
+		spec.Docker.Volumes = append(
+			spec.Docker.Volumes,
+			&engine.Volume{
+				Metadata: engine.Metadata{
+					UID:  rand.String(),
+					Name: workspaceHostName,
+				},
+				HostPath: &engine.VolumeHostPath{
+					Path: workdir,
+				},
+			},
+		)
+	}
+}
+
+//
+//
+//
+
+// MountWorkspace is a WorkspaceFunc that mounts the
+// default workspace volume to the pipeline step.
+func MountWorkspace(step *engine.Step, base, path, full string) {
+	step.Volumes = append(step.Volumes, &engine.VolumeMount{
+		Name: workspaceName,
+		Path: base,
+	})
+}
+
+// MountHostWorkspace is a WorkspaceFunc that mounts
+// the default workspace and host volume to the pipeline.
+func MountHostWorkspace(step *engine.Step, base, path, full string) {
+	step.Volumes = append(step.Volumes, &engine.VolumeMount{
+		Name: workspaceHostName,
+		Path: full,
+	})
+	if path != "" {
+		step.Volumes = append(step.Volumes, &engine.VolumeMount{
+			Name: workspaceName,
+			Path: base,
+		})
+	}
+}

yaml/compiler/workspace_test.go

@@ -0,0 +1,144 @@
+package compiler
+
+import (
+	"testing"
+
+	"github.com/drone/drone-runtime/engine"
+	"github.com/drone/drone-yaml/yaml"
+)
+
+func TestSetupWorkspace(t *testing.T) {
+	tests := []struct {
+		path string
+		src  *yaml.Container
+		dst  *engine.Step
+		want string
+	}{
+		{
+			path: "/drone/src",
+			src:  &yaml.Container{},
+			dst:  &engine.Step{},
+			want: "/drone/src",
+		},
+		// do not override the user-defined working dir.
+		{
+			path: "/drone/src",
+			src:  &yaml.Container{},
+			dst:  &engine.Step{WorkingDir: "/foo"},
+			want: "/foo",
+		},
+		// do not override the default working directory
+		// for service containers with no commands.
+		{
+			path: "/drone/src",
+			src:  &yaml.Container{},
+			dst:  &engine.Step{Detach: true},
+			want: "",
+		},
+		// overrides the default working directory
+		// for service containers with commands.
+		{
+			path: "/drone/src",
+			src:  &yaml.Container{Commands: []string{"whoami"}},
+			dst:  &engine.Step{Detach: true},
+			want: "/drone/src",
+		},
+	}
+	for _, test := range tests {
+		setupWorkingDir(test.src, test.dst, test.path)
+		if got, want := test.dst.WorkingDir, test.want; got != want {
+			t.Errorf("Want working_dir %s, got %s", want, got)
+		}
+	}
+}
+
+func TestToWindows(t *testing.T) {
+	got := toWindowsDrive("/go/src/github.com/octocat/hello-world")
+	want := "c:\\go\\src\\github.com\\octocat\\hello-world"
+	if got != want {
+		t.Errorf("Want windows drive %q, got %q", want, got)
+	}
+}
+
+func TestCreateWorkspace(t *testing.T) {
+	tests := []struct {
+		from *yaml.Pipeline
+		base string
+		path string
+		full string
+	}{
+		{
+			from: &yaml.Pipeline{
+				Workspace: yaml.Workspace{
+					Base: "",
+					Path: "",
+				},
+			},
+			base: "/drone/src",
+			path: "",
+			full: "/drone/src",
+		},
+		{
+			from: &yaml.Pipeline{
+				Workspace: yaml.Workspace{
+					Base: "",
+					Path: "",
+				},
+				Platform: yaml.Platform{
+					OS: "windows",
+				},
+			},
+			base: "c:\\drone\\src",
+			path: "",
+			full: "c:\\drone\\src",
+		},
+		{
+			from: &yaml.Pipeline{
+				Workspace: yaml.Workspace{
+					Base: "/drone",
+					Path: "src",
+				},
+			},
+			base: "/drone",
+			path: "src",
+			full: "/drone/src",
+		},
+		{
+			from: &yaml.Pipeline{
+				Workspace: yaml.Workspace{
+					Base: "/drone",
+					Path: "src",
+				},
+				Platform: yaml.Platform{
+					OS: "windows",
+				},
+			},
+			base: "c:\\drone",
+			path: "src",
+			full: "c:\\drone\\src",
+		},
+		{
+			from: &yaml.Pipeline{
+				Workspace: yaml.Workspace{
+					Base: "/foo",
+					Path: "bar",
+				},
+			},
+			base: "/foo",
+			path: "bar",
+			full: "/foo/bar",
+		},
+	}
+	for _, test := range tests {
+		base, path, full := createWorkspace(test.from)
+		if got, want := test.base, base; got != want {
+			t.Errorf("Want workspace base %s, got %s", want, got)
+		}
+		if got, want := test.path, path; got != want {
+			t.Errorf("Want workspace path %s, got %s", want, got)
+		}
+		if got, want := test.full, full; got != want {
+			t.Errorf("Want workspace %s, got %s", want, got)
+		}
+	}
+}

yaml/cond.go

@@ -0,0 +1,85 @@
+package yaml
+
+import filepath "github.com/bmatcuk/doublestar"
+
+// Conditions defines a group of conditions.
+type Conditions struct {
+	Ref      Condition `json:"ref,omitempty"`
+	Repo     Condition `json:"repo,omitempty"`
+	Instance Condition `json:"instance,omitempty"`
+	Target   Condition `json:"target,omitempty"`
+	Event    Condition `json:"event,omitempty"`
+	Branch   Condition `json:"branch,omitempty"`
+	Status   Condition `json:"status,omitempty"`
+	Paths    Condition `json:"paths,omitempty"`
+}
+
+// Condition defines a runtime condition.
+type Condition struct {
+	Include []string `yaml:"include,omitempty" json:"include,omitempty"`
+	Exclude []string `yaml:"exclude,omitempty" json:"exclude,omitempty"`
+}
+
+// Match returns true if the string matches the include
+// patterns and does not match any of the exclude patterns.
+func (c *Condition) Match(v string) bool {
+	if c.Excludes(v) {
+		return false
+	}
+	if c.Includes(v) {
+		return true
+	}
+	if len(c.Include) == 0 {
+		return true
+	}
+	return false
+}
+
+// Includes returns true if the string matches the include
+// patterns.
+func (c *Condition) Includes(v string) bool {
+	for _, pattern := range c.Include {
+		if ok, _ := filepath.Match(pattern, v); ok {
+			return true
+		}
+	}
+	return false
+}
+
+// Excludes returns true if the string matches the exclude
+// patterns.
+func (c *Condition) Excludes(v string) bool {
+	for _, pattern := range c.Exclude {
+		if ok, _ := filepath.Match(pattern, v); ok {
+			return true
+		}
+	}
+	return false
+}
+
+// UnmarshalYAML implements yml unmarshalling.
+func (c *Condition) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var out1 string
+	var out2 []string
+	var out3 = struct {
+		Include []string
+		Exclude []string
+	}{}
+
+	err := unmarshal(&out1)
+	if err == nil {
+		c.Include = []string{out1}
+		return nil
+	}
+
+	unmarshal(&out2)
+	unmarshal(&out3)
+
+	c.Exclude = out3.Exclude
+	c.Include = append(
+		out3.Include,
+		out2...,
+	)
+
+	return nil
+}

yaml/cond_test.go

@@ -0,0 +1,166 @@
+package yaml
+
+import (
+	"testing"
+
+	"gopkg.in/yaml.v2"
+)
+
+func TestConstraintMatch(t *testing.T) {
+	testdata := []struct {
+		conf string
+		with string
+		want bool
+	}{
+		// string value
+		{
+			conf: "master",
+			with: "develop",
+			want: false,
+		},
+		{
+			conf: "master",
+			with: "master",
+			want: true,
+		},
+		{
+			conf: "feature/*",
+			with: "feature/foo",
+			want: true,
+		},
+		// slice value
+		{
+			conf: "[ master, feature/* ]",
+			with: "develop",
+			want: false,
+		},
+		{
+			conf: "[ master, feature/* ]",
+			with: "master",
+			want: true,
+		},
+		{
+			conf: "[ master, feature/* ]",
+			with: "feature/foo",
+			want: true,
+		},
+		// includes block
+		{
+			conf: "include: [ master ]",
+			with: "develop",
+			want: false,
+		},
+		{
+			conf: "include: [ master] ",
+			with: "master",
+			want: true,
+		},
+		{
+			conf: "include: [ feature/* ]",
+			with: "master",
+			want: false,
+		},
+		{
+			conf: "include: [ feature/* ]",
+			with: "feature/foo",
+			want: true,
+		},
+		{
+			conf: "include: [ master, feature/* ]",
+			with: "develop",
+			want: false,
+		},
+		{
+			conf: "include: [ master, feature/* ]",
+			with: "master",
+			want: true,
+		},
+		{
+			conf: "include: [ master, feature/* ]",
+			with: "feature/foo",
+			want: true,
+		},
+		// excludes block
+		{
+			conf: "exclude: [ master ]",
+			with: "develop",
+			want: true,
+		},
+		{
+			conf: "exclude: [ master ]",
+			with: "master",
+			want: false,
+		},
+		{
+			conf: "exclude: [ feature/* ]",
+			with: "master",
+			want: true,
+		},
+		{
+			conf: "exclude: [ feature/* ]",
+			with: "feature/foo",
+			want: false,
+		},
+		{
+			conf: "exclude: [ master, develop ]",
+			with: "master",
+			want: false,
+		},
+		{
+			conf: "exclude: [ feature/*, bar ]",
+			with: "master",
+			want: true,
+		},
+		{
+			conf: "exclude: [ feature/*, bar ]",
+			with: "feature/foo",
+			want: false,
+		},
+		// include and exclude blocks
+		{
+			conf: "{ include: [ master, feature/* ], exclude: [ develop ] }",
+			with: "master",
+			want: true,
+		},
+		{
+			conf: "{ include: [ master, feature/* ], exclude: [ feature/bar ] }",
+			with: "feature/bar",
+			want: false,
+		},
+		{
+			conf: "{ include: [ master, feature/* ], exclude: [ master, develop ] }",
+			with: "master",
+			want: false,
+		},
+		// empty blocks
+		{
+			conf: "",
+			with: "master",
+			want: true,
+		},
+		// double star
+		{
+			conf: "foo/**",
+			with: "foo/bar/baz/qux",
+			want: true,
+		},
+		{
+			conf: "foo/**/qux",
+			with: "foo/bar/baz/qux",
+			want: true,
+		},
+	}
+	for _, test := range testdata {
+		c := parseCondition(test.conf)
+		got, want := c.Match(test.with), test.want
+		if got != want {
+			t.Errorf("Expect %q matches %q is %v", test.with, test.conf, want)
+		}
+	}
+}
+
+func parseCondition(s string) *Condition {
+	c := &Condition{}
+	yaml.Unmarshal([]byte(s), c)
+	return c
+}

yaml/converter/bitbucket/config.go

@@ -0,0 +1,104 @@
+package bitbucket
+
+import (
+	"path"
+	"strings"
+)
+
+type (
+	// Config defines the pipeline configuration.
+	Config struct {
+		// Image specifies the Docker image with
+		// which we run your builds.
+		Image string
+
+		// Clone defines the depth of Git clones
+		// for all pipelines.
+		Clone struct {
+			Depth int
+		}
+
+		// Pipeline defines the pipeline configuration
+		// which includes a list of all steps for default,
+		// tag, and branch-specific execution.
+		Pipelines struct {
+			Default  Stage
+			Tags     map[string]Stage
+			Branches map[string]Stage
+		}
+
+		Definitions struct {
+			Services map[string]*Step
+			Caches   map[string]string
+		}
+	}
+
+	// Stage contains a list of steps executed
+	// for a specific branch or tag.
+	Stage struct {
+		Name  string
+		Steps []*Step
+	}
+
+	// Step defines a build execution unit.
+	Step struct {
+		// Name of the pipeline step.
+		Name string
+
+		// Image specifies the Docker image with
+		// which we run your builds.
+		Image string
+
+		// Script contains the list of bash commands
+		// that are executed in sequence.
+		Script []string
+
+		// Variables provides environment variables
+		// passed to the container at runtime.
+		Variables map[string]string
+
+		// Artifacts defines files that are to be
+		// snapshotted and shared with the subsequent
+		// step. This is not used, because Drone uses
+		// a shared volume to share artifacts.
+		Artifacts []string
+	}
+)
+
+// Pipeline returns the pipeline stage that best matches the branch
+// and ref. If there is no matching pipeline specific to the branch
+// or tag, the default pipeline is returned.
+func (c *Config) Pipeline(ref string) Stage {
+	// match pipeline by tag name
+	tag := strings.TrimPrefix(ref, "refs/tags/")
+	for pattern, pipeline := range c.Pipelines.Tags {
+		if ok, _ := path.Match(pattern, tag); ok {
+			return pipeline
+		}
+	}
+	// match pipeline by branch name
+	branch := strings.TrimPrefix(ref, "refs/heads/")
+	for pattern, pipeline := range c.Pipelines.Branches {
+		if ok, _ := path.Match(pattern, branch); ok {
+			return pipeline
+		}
+	}
+	// use default
+	return c.Pipelines.Default
+}
+
+// UnmarshalYAML implements custom parsing for the stage section of the yaml
+// to cleanup the structure a bit.
+func (s *Stage) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	in := []struct {
+		Step *Step
+	}{}
+	err := unmarshal(&in)
+	if err != nil {
+		return err
+	}
+	for _, step := range in {
+		s.Steps = append(s.Steps, step.Step)
+	}
+	return nil
+}

yaml/converter/bitbucket/config_test.go

@@ -0,0 +1 @@
+package bitbucket

yaml/converter/bitbucket/convert.go

@@ -0,0 +1,82 @@
+package bitbucket
+
+import (
+	"bytes"
+	"fmt"
+
+	droneyaml "github.com/drone/drone-yaml/yaml"
+	"github.com/drone/drone-yaml/yaml/pretty"
+
+	"gopkg.in/yaml.v2"
+)
+
+// Convert converts the yaml configuration file from
+// the legacy format to the 1.0+ format.
+func Convert(b []byte, ref string) ([]byte, error) {
+	config := new(Config)
+	err := yaml.Unmarshal(b, config)
+	if err != nil {
+		return nil, err
+	}
+
+	// TODO (bradrydzewski) to correctly choose
+	// the pipeline we need to pass the branch
+	// and ref.
+	stage := config.Pipeline(ref)
+
+	pipeline := &droneyaml.Pipeline{}
+	pipeline.Name = "default"
+	pipeline.Kind = "pipeline"
+
+	//
+	// clone
+	//
+
+	pipeline.Clone.Depth = config.Clone.Depth
+
+	//
+	// steps
+	//
+
+	for i, from := range stage.Steps {
+		to := toContainer(from)
+		// defaults to the global image if the
+		// step does not define an image.
+		if to.Image == "" {
+			to.Image = config.Image
+		}
+		if to.Name == "" {
+			to.Name = fmt.Sprintf("step_%d", i)
+		}
+		pipeline.Steps = append(pipeline.Steps, to)
+	}
+
+	//
+	// services
+	//
+
+	for name, from := range config.Definitions.Services {
+		to := toContainer(from)
+		to.Name = name
+		pipeline.Services = append(pipeline.Services, to)
+	}
+
+	//
+	// wrap the pipeline in the manifest
+	//
+
+	manifest := &droneyaml.Manifest{}
+	manifest.Resources = append(manifest.Resources, pipeline)
+
+	buf := new(bytes.Buffer)
+	pretty.Print(buf, manifest)
+	return buf.Bytes(), nil
+}
+
+func toContainer(from *Step) *droneyaml.Container {
+	return &droneyaml.Container{
+		Name:     from.Name,
+		Image:    from.Image,
+		Commands: from.Script,
+	}
+}

yaml/converter/bitbucket/convert_test.go

@@ -0,0 +1,46 @@
+package bitbucket
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+)
+
+func TestConvert(t *testing.T) {
+	tests := []struct {
+		before, after, ref string
+	}{
+		{
+			before: "testdata/sample1.yaml",
+			after:  "testdata/sample1.yaml.golden",
+			ref:    "refs/heads/master",
+		},
+		{
+			before: "testdata/sample2.yaml",
+			after:  "testdata/sample2.yaml.golden",
+			ref:    "refs/heads/feature/foo",
+		},
+	}
+
+	for _, test := range tests {
+		a, err := ioutil.ReadFile(test.before)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		b, err := ioutil.ReadFile(test.after)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		c, err := Convert([]byte(a), test.ref)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		if bytes.Equal(b, c) == false {
+			t.Errorf("Unexpected yaml conversion of %s", test.before)
+			t.Log(string(c))
+		}
+	}
+}

yaml/converter/bitbucket/testdata/sample1.yaml

@@ -0,0 +1,32 @@
+pipelines:
+  default:
+    - step:
+        name: Build and test
+        image: node:8.5.0
+        caches:
+          - node
+        script:
+          - npm install
+          - npm test
+          - npm build
+        artifacts:
+          - dist/**
+    - step:
+        name: Integration test
+        image: node:8.5.0
+        caches:
+          - node
+        services:
+          - postgres
+        script:
+          - npm run integration-test
+    - step:
+        name: Deploy to beanstalk
+        image: python:3.5.1
+        script:
+          - python deploy-to-beanstalk.py
+ 
+definitions:
+  services:
+    postgres:
+      image: postgres:9.6.4

yaml/converter/bitbucket/testdata/sample1.yaml.golden

@@ -0,0 +1,31 @@
+---
+kind: pipeline
+name: default
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: Build and test
+  image: node:8.5.0
+  commands:
+  - npm install
+  - npm test
+  - npm build
+
+- name: Integration test
+  image: node:8.5.0
+  commands:
+  - npm run integration-test
+
+- name: Deploy to beanstalk
+  image: python:3.5.1
+  commands:
+  - python deploy-to-beanstalk.py
+
+services:
+- name: postgres
+  image: postgres:9.6.4
+
+...

yaml/converter/bitbucket/testdata/sample2.yaml

@@ -0,0 +1,40 @@
+pipelines:
+  branches:
+    feature/*:
+      - step:
+          name: Test
+          image: node:latest
+          script:
+            - npm install
+            - npm test
+  default:
+    - step:
+        name: Build and test
+        image: node:8.5.0
+        caches:
+          - node
+        script:
+          - npm install
+          - npm test
+          - npm build
+        artifacts:
+          - dist/**
+    - step:
+        name: Integration test
+        image: node:8.5.0
+        caches:
+          - node
+        services:
+          - postgres
+        script:
+          - npm run integration-test
+    - step:
+        name: Deploy to beanstalk
+        image: python:3.5.1
+        script:
+          - python deploy-to-beanstalk.py
+ 
+definitions:
+  services:
+    postgres:
+      image: postgres:9.6.4

yaml/converter/bitbucket/testdata/sample2.yaml.golden

@@ -0,0 +1,20 @@
+---
+kind: pipeline
+name: default
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: Test
+  image: node:latest
+  commands:
+  - npm install
+  - npm test
+
+services:
+- name: postgres
+  image: postgres:9.6.4
+
+...

yaml/converter/circleci/config.go

@@ -0,0 +1,76 @@
+package circleci
+
+type (
+	// Config defines the pipeline configuration.
+	Config struct {
+		// Version specifies the yaml configuration
+		// file version.
+		Version string
+
+		// Jobs defines a list of pipeline jobs.
+		Jobs []*Job
+
+		// Workflows are used to orchestrate jobs.
+		Workflows struct {
+			Version string
+			List    map[string]*Workflow `yaml:",inline"`
+		}
+	}
+
+	// Workflow ochestrates one or more jobs.
+	Workflow struct {
+		Jobs []string
+	}
+
+	// Job defines a pipeline job.
+	Job struct {
+		// Name of the stage.
+		Name string
+
+		// Docker configures a Docker executor.
+		Docker Docker
+
+		// Environment variables passed to the executor.
+		Environment map[string]string
+
+		// Steps configures the Job steps.
+		Steps map[string]Step
+
+		// Branches limits execution by branch.
+		Branches []struct {
+			Only   []string
+			Ignore []string
+		}
+	}
+
+	// Step defines a build execution unit.
+	Step struct {
+		Run                Run
+		AddSSHKeys         map[string]interface{} `yaml:"add_ssh_keys"`
+		AttachWorkspace    map[string]interface{} `yaml:"attach_workspace"`
+		Checkout           map[string]interface{} `yaml:"checkout"`
+		Deploy             map[string]interface{} `yaml:"deploy"`
+		PersistToWorkspace map[string]interface{} `yaml:"persist_to_workspace"`
+		RestoreCache       map[string]interface{} `yaml:"restore_cache"`
+		SaveCache          map[string]interface{} `yaml:"save_cache"`
+		SetupRemoteDocker  map[string]interface{} `yaml:"setup_remote_docker"`
+		StoreArtifacts     map[string]interface{} `yaml:"store_artifacts"`
+		StoreTestResults   map[string]interface{} `yaml:"store_test_results"`
+	}
+)
+
+// // UnmarshalYAML implements custom parsing for the stage section of the yaml
+// // to cleanup the structure a bit.
+// func (s *Stage) UnmarshalYAML(unmarshal func(interface{}) error) error {
+// 	in := []struct {
+// 		Step *Step
+// 	}{}
+// 	err := unmarshal(&in)
+// 	if err != nil {
+// 		return err
+// 	}
+// 	for _, step := range in {
+// 		s.Steps = append(s.Steps, step.Step)
+// 	}
+// 	return nil
+// }

yaml/converter/circleci/docker.go

@@ -0,0 +1,28 @@
+package circleci
+
+// Docker configures a Docker executor.
+type Docker struct {
+	// Image is the Docker image name.
+	Image string
+
+	// Name is the Docker container hostname.
+	Name string
+
+	// Entrypoint is the Docker container entrypoint.
+	Entrypoint []string
+
+	// Command is the Docker container command.
+	Command []string
+
+	// User is user that runs the Docker entrypoint.
+	User string
+
+	// Environment variables passed to the container.
+	Environment map[string]string
+
+	// Auth credentials to pull private images.
+	Auth map[string]string
+
+	// Auth credentials to pull private ECR images.
+	AWSAuth map[string]string `yaml:"aws_auth"`
+}

yaml/converter/circleci/docker_test.go

@@ -0,0 +1 @@
+package circleci

yaml/converter/circleci/run.go

@@ -0,0 +1,34 @@
+package circleci
+
+import "time"
+
+// Run defines a command
+type Run struct {
+	// Name of the command
+	Name string
+
+	// Command run in the shell.
+	Command string
+
+	// Shell to use to execute the command.
+	Shell string
+
+	// Workiring Directory in which the command
+	// is run.
+	WorkingDir string `yaml:"working_directory"`
+
+	// Command is run in the background.
+	Background bool `yaml:"background"`
+
+	// Amount of time the command can run with
+	// no output before being canceled.
+	NoOutputTimeout time.Duration `yaml:"no_output_timeout"`
+
+	// Environment variables set when running
+	// the command in the shell.
+	Environment map[string]string
+
+	// Defines when the command should be executed.
+	// Values are always, on_success, and on_fail.
+	When string
+}

yaml/converter/circleci/run_test.go

@@ -0,0 +1,11 @@
+package circleci
+
+const testRun = `
+- run:
+    name: test
+    command: go test
+`
+
+const testRunShort = `
+- run: go test
+`

yaml/converter/circleci/testdata/sample1.yml

@@ -0,0 +1,22 @@
+version: 2
+jobs:
+  backend:
+    docker:
+      - image: golang:1.8
+    steps:
+      - checkout
+      - run: go build
+      - run: go test
+  frontend:
+    docker:
+      - image: node:latest
+    steps:
+      - checkout
+      - run: npm install
+      - run: npm test
+workflows:
+  version: 2
+  default:
+    jobs:
+      - backend
+      - frontend

yaml/converter/convert.go

@@ -0,0 +1,51 @@
+package converter
+
+import (
+	"github.com/drone/drone-yaml/yaml/converter/bitbucket"
+	"github.com/drone/drone-yaml/yaml/converter/gitlab"
+	"github.com/drone/drone-yaml/yaml/converter/legacy"
+)
+
+// Metadata provides additional metadata used to
+// convert the configuration file format.
+type Metadata struct {
+	// Filename of the configuration file, helps
+	// determine the yaml configuration format.
+	Filename string
+
+	// Ref of the commit use to choose the correct
+	// pipeline if the configuration format defines
+	// multiple pipelines (like Bitbucket)
+	Ref string
+}
+
+// Convert converts the yaml configuration file from
+// the legacy format to the 1.0+ format.
+func Convert(d []byte, m Metadata) ([]byte, error) {
+	switch m.Filename {
+	case "bitbucket-pipelines.yml":
+		return bitbucket.Convert(d, m.Ref)
+	case "circle.yml", ".circleci/config.yml":
+		// TODO(bradrydzewski)
+	case ".gitlab-ci.yml":
+		return gitlab.Convert(d)
+	case ".travis.yml":
+		// TODO(bradrydzewski)
+	}
+	// if the filename does not match any external
+	// systems we check to see if the configuration
+	// file is a legacy (pre 1.0) .drone.yml format.
+	if legacy.Match(d) {
+		return legacy.Convert(d)
+	}
+	// else return the unmodified configuration
+	// back to the caller.
+	return d, nil
+}
+
+// ConvertString converts the yaml configuration file from
+// the legacy format to the 1.0+ format.
+func ConvertString(s string, m Metadata) (string, error) {
+	b, err := Convert([]byte(s), m)
+	return string(b), err
+}

yaml/converter/convert_test.go

@@ -0,0 +1 @@
+package converter

yaml/converter/gitlab/config.go

@@ -0,0 +1,125 @@
+package gitlab
+
+import (
+	"github.com/drone/drone-yaml/yaml/converter/internal"
+)
+
+type (
+	// Config defines the pipeline configuration.
+	Config struct {
+		// Image specifies the Docker image with
+		// which we run your builds.
+		Image Image
+
+		// Stages is used to group steps into stages,
+		// where each stage is executed sequentially.
+		Stages []string
+
+		// Services is used to define a set of services
+		// that should be started and linked to each
+		// step in the pipeline.
+		Services []*Image
+
+		// Variables is used to customize execution,
+		// such as the clone strategy.
+		Variables map[string]string
+
+		// Before contains the list of bash commands
+		// that are executed in sequence before the
+		// first job.
+		Before internal.StringSlice `yaml:"before_script"`
+
+		// After contains the list of bash commands
+		// that are executed in sequence after the
+		// last job.
+		After internal.StringSlice `yaml:"after_script"`
+
+		// Jobs is used to define individual units
+		// of execution that make up a stage.
+		Jobs map[string]*Job `yaml:",inline"`
+	}
+
+	// Job defines a build execution unit.
+	Job struct {
+		// Name of the pipeline step.
+		Name string
+
+		// Stage is the name of the stage.
+		Stage string
+
+		// Image specifies the Docker image with
+		// which we run your builds.
+		Image Image
+
+		// Script contains the list of bash commands
+		// that are executed in sequence.
+		Script internal.StringSlice
+
+		// Before contains the list of bash commands
+		// that are executed in sequence before the
+		// primary script.
+		Before internal.StringSlice `yaml:"before_script"`
+
+		// After contains the list of bash commands
+		// that are executed in sequence after the
+		// primary script.
+		After internal.StringSlice `yaml:"after_script"`
+
+		// Services defines a set of services linked
+		// to the job.
+		Services []*Image
+
+		// Only defines the names of branches and tags
+		// for which the job will run.
+		Only internal.StringSlice
+
+		// Except defines the names of branches and tags
+		// for which the job will not run.
+		Except internal.StringSlice
+
+		// Variables is used to customize execution,
+		// such as the clone strategy.
+		Variables map[string]string
+
+		// Allow job to fail. Failed job doesn’t contribute
+		// to commit status
+		AllowFailure bool
+
+		// Define when to run job. Can be on_success, on_failure,
+		// always or manual
+		When internal.StringSlice
+	}
+
+	// Image defines a Docker image.
+	Image struct {
+		Name       string
+		Entrypoint []string
+		Command    []string
+		Alias      string
+	}
+)
+
+// UnmarshalYAML implements custom parsing for an Image.
+func (i *Image) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var name string
+	err := unmarshal(&name)
+	if err == nil {
+		i.Name = name
+		return nil
+	}
+	data := struct {
+		Name       string
+		Entrypoint internal.StringSlice
+		Command    internal.StringSlice
+		Alias      string
+	}{}
+	err = unmarshal(&data)
+	if err != nil {
+		return err
+	}
+	i.Name = data.Name
+	i.Entrypoint = data.Entrypoint
+	i.Command = data.Command
+	i.Alias = data.Alias
+	return nil
+}

yaml/converter/gitlab/convert.go

@@ -0,0 +1,95 @@
+package gitlab
+
+import (
+	"bytes"
+
+	droneyaml "github.com/drone/drone-yaml/yaml"
+	"github.com/drone/drone-yaml/yaml/pretty"
+
+	"github.com/gosimple/slug"
+	"gopkg.in/yaml.v2"
+)
+
+// Convert converts the yaml configuration file from
+// the legacy format to the 1.0+ format.
+func Convert(b []byte) ([]byte, error) {
+	config := new(Config)
+	err := yaml.Unmarshal(b, config)
+	if err != nil {
+		return nil, err
+	}
+	manifest := &droneyaml.Manifest{}
+
+	// if no stages are defined, we create a single,
+	// default stage that will be used for all jobs.
+	if len(config.Stages) == 0 {
+		for name, job := range config.Jobs {
+			config.Stages = append(config.Stages, name)
+			job.Stage = name
+		}
+	}
+
+	// create a new pipeline for each stage.
+	var prevstage string
+	for _, stage := range config.Stages {
+		pipeline := &droneyaml.Pipeline{}
+		pipeline.Name = stage
+		pipeline.Kind = droneyaml.KindPipeline
+		manifest.Resources = append(manifest.Resources, pipeline)
+		for name, job := range config.Jobs {
+			if job.Stage != stage {
+				continue
+			}
+			cmds := []string(config.Before)
+			cmds = append(cmds, []string(job.Before)...)
+			cmds = append(cmds, []string(job.Script)...)
+			cmds = append(cmds, []string(job.After)...)
+			cmds = append(cmds, []string(config.After)...)
+
+			step := &droneyaml.Container{
+				Name:       name,
+				Image:      job.Image.Name,
+				Command:    job.Image.Command,
+				Entrypoint: job.Image.Entrypoint,
+				Commands:   cmds,
+			}
+
+			if job.AllowFailure {
+				step.Failure = "ignore"
+			}
+
+			if step.Image == "" {
+				step.Image = config.Image.Name
+			}
+			// TODO: handle Services
+			// TODO: handle Only
+			// TODO: handle Except
+			// TODO: handle Variables
+			// TODO: handle When
+
+			pipeline.Steps = append(pipeline.Steps, step)
+		}
+
+		for _, step := range config.Services {
+			step := &droneyaml.Container{
+				Name:       step.Alias,
+				Image:      step.Name,
+				Command:    step.Command,
+				Entrypoint: step.Entrypoint,
+			}
+			if step.Name == "" {
+				step.Name = slug.Make(step.Image)
+			}
+			pipeline.Services = append(pipeline.Services, step)
+		}
+
+		if prevstage != "" {
+			pipeline.DependsOn = []string{prevstage}
+		}
+		prevstage = stage
+	}
+
+	buf := new(bytes.Buffer)
+	pretty.Print(buf, manifest)
+	return buf.Bytes(), nil
+}

yaml/converter/gitlab/convert_test.go

@@ -0,0 +1,57 @@
+package gitlab
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/sergi/go-diff/diffmatchpatch"
+)
+
+func TestConvert(t *testing.T) {
+	tests := []struct {
+		before, after, ref string
+	}{
+		{
+			before: "testdata/example1.yml",
+			after:  "testdata/example1.yml.golden",
+		},
+		{
+			before: "testdata/example2.yml",
+			after:  "testdata/example2.yml.golden",
+		},
+		{
+			before: "testdata/example3.yml",
+			after:  "testdata/example3.yml.golden",
+		},
+		{
+			before: "testdata/example4.yml",
+			after:  "testdata/example4.yml.golden",
+		},
+	}
+
+	for _, test := range tests {
+		a, err := ioutil.ReadFile(test.before)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		b, err := ioutil.ReadFile(test.after)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		c, err := Convert([]byte(a))
+		if err != nil {
+			t.Error(err)
+			return
+		}
+
+		if bytes.Equal(b, c) == false {
+			t.Errorf("Unexpected yaml conversion of %s", test.before)
+			dmp := diffmatchpatch.New()
+			diffs := dmp.DiffMain(string(b), string(c), false)
+			t.Log(dmp.DiffCleanupSemantic(diffs))
+		}
+	}
+}

yaml/converter/gitlab/testdata/example1.yml

@@ -0,0 +1,11 @@
+image: ruby:2.2
+
+services:
+  - postgres:9.3
+
+before_script:
+  - bundle install
+
+test:
+  script:
+  - bundle exec rake spec

yaml/converter/gitlab/testdata/example1.yml.golden

@@ -0,0 +1,20 @@
+---
+kind: pipeline
+name: test
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: test
+  image: ruby:2.2
+  commands:
+  - bundle install
+  - bundle exec rake spec
+
+services:
+- name: postgres-9-3
+  image: postgres:9.3
+
+...

yaml/converter/gitlab/testdata/example2.yml

@@ -0,0 +1,16 @@
+before_script:
+  - bundle install
+
+test2.1:
+  image: ruby:2.1
+  services:
+  - postgres:9.3
+  script:
+  - bundle exec rake spec
+
+test2.2:
+  image: ruby:2.2
+  services:
+  - postgres:9.4
+  script:
+  - bundle exec rake spec

yaml/converter/gitlab/testdata/example2.yml.golden

@@ -0,0 +1,34 @@
+---
+kind: pipeline
+name: test2.1
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: test2.1
+  image: ruby:2.1
+  commands:
+  - bundle install
+  - bundle exec rake spec
+
+---
+kind: pipeline
+name: test2.2
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: test2.2
+  image: ruby:2.2
+  commands:
+  - bundle install
+  - bundle exec rake spec
+
+depends_on:
+- test2.1
+
+...

yaml/converter/gitlab/testdata/example3.yml

@@ -0,0 +1,16 @@
+image:
+  name: ruby:2.2
+  entrypoint: ["/bin/bash"]
+
+services:
+- name: my-postgres:9.4
+  alias: db-postgres
+  entrypoint: ["/usr/local/bin/db-postgres"]
+  command: ["start"]
+
+before_script:
+- bundle install
+
+test:
+  script:
+  - bundle exec rake spec

yaml/converter/gitlab/testdata/example3.yml.golden

@@ -0,0 +1,24 @@
+---
+kind: pipeline
+name: test
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: test
+  image: ruby:2.2
+  commands:
+  - bundle install
+  - bundle exec rake spec
+
+services:
+- name: db-postgres
+  image: my-postgres:9.4
+  entrypoint:
+  - /usr/local/bin/db-postgres
+  command:
+  - start
+
+...

yaml/converter/gitlab/testdata/example4.yml

@@ -0,0 +1,22 @@
+stages:
+  - build
+  - test
+  - deploy
+
+image: ruby:2.2
+
+job 1:
+  stage: build
+  script: make build dependencies
+
+job 2:
+  stage: build
+  script: make build artifacts
+
+job 3:
+  stage: test
+  script: make test
+
+job 4:
+  stage: deploy
+  script: make deploy

yaml/converter/gitlab/testdata/example4.yml.golden

@@ -0,0 +1,54 @@
+---
+kind: pipeline
+name: build
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: job 1
+  image: ruby:2.2
+  commands:
+  - make build dependencies
+
+- name: job 2
+  image: ruby:2.2
+  commands:
+  - make build artifacts
+
+---
+kind: pipeline
+name: test
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: job 3
+  image: ruby:2.2
+  commands:
+  - make test
+
+depends_on:
+- build
+
+---
+kind: pipeline
+name: deploy
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: job 4
+  image: ruby:2.2
+  commands:
+  - make deploy
+
+depends_on:
+- test
+
+...

yaml/converter/internal/string_slice.go

@@ -0,0 +1,20 @@
+package internal
+
+// StringSlice represents a slice of strings or a string.
+type StringSlice []string
+
+// UnmarshalYAML implements the Unmarshaller interface.
+func (s *StringSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var stringType string
+	if err := unmarshal(&stringType); err == nil {
+		*s = []string{stringType}
+		return nil
+	}
+
+	var sliceType []string
+	if err := unmarshal(&sliceType); err != nil {
+		return err
+	}
+	*s = sliceType
+	return nil
+}

yaml/converter/internal/string_slice_test.go

@@ -0,0 +1,45 @@
+package internal
+
+import (
+	"reflect"
+	"testing"
+
+	"gopkg.in/yaml.v2"
+)
+
+func TestStringSlice(t *testing.T) {
+	var tests = []struct {
+		yaml string
+		want []string
+	}{
+		{
+			yaml: "hello world",
+			want: []string{"hello world"},
+		},
+		{
+			yaml: "[ hello, world ]",
+			want: []string{"hello", "world"},
+		},
+		{
+			yaml: "42",
+			want: []string{"42"},
+		},
+	}
+
+	for _, test := range tests {
+		var got StringSlice
+
+		if err := yaml.Unmarshal([]byte(test.yaml), &got); err != nil {
+			t.Error(err)
+		}
+
+		if !reflect.DeepEqual([]string(got), test.want) {
+			t.Errorf("Got slice %v want %v", got, test.want)
+		}
+	}
+
+	var got StringSlice
+	if err := yaml.Unmarshal([]byte("{}"), &got); err == nil {
+		t.Errorf("Want error unmarshaling invalid string or slice value.")
+	}
+}

yaml/converter/legacy/convert.go

@@ -0,0 +1,9 @@
+package legacy
+
+import "github.com/drone/drone-yaml/yaml/converter/legacy/internal"
+
+// Convert converts the yaml configuration file from
+// the legacy format to the 1.0+ format.
+func Convert(d []byte) ([]byte, error) {
+	return yaml.Convert(d)
+}

yaml/converter/legacy/internal/config.go

@@ -0,0 +1,247 @@
+package yaml
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+
+	droneyaml "github.com/drone/drone-yaml/yaml"
+	"github.com/drone/drone-yaml/yaml/pretty"
+
+	"gopkg.in/yaml.v2"
+)
+
+// Config provides the high-level configuration.
+type Config struct {
+	Workspace struct {
+		Base string
+		Path string
+	}
+	Clone    Containers
+	Pipeline Containers
+	Services Containers
+	Branches Constraint
+	Secrets  map[string]struct {
+		Driver     string
+		DriverOpts map[string]string `yaml:"driver_opts"`
+		Path       string
+		Vault      string
+	}
+}
+
+// Convert converts the yaml configuration file from
+// the legacy format to the 1.0+ format.
+func Convert(d []byte) ([]byte, error) {
+	from := new(Config)
+	err := yaml.Unmarshal(d, from)
+	if err != nil {
+		return nil, err
+	}
+
+	pipeline := &droneyaml.Pipeline{}
+	pipeline.Name = "default"
+	pipeline.Kind = "pipeline"
+
+	pipeline.Workspace.Base = from.Workspace.Base
+	pipeline.Workspace.Path = from.Workspace.Path
+
+	if len(from.Clone.Containers) != 0 {
+		pipeline.Clone.Disable = true
+		for _, container := range from.Clone.Containers {
+			pipeline.Steps = append(pipeline.Steps,
+				toContainer(container),
+			)
+		}
+	}
+
+	for _, container := range from.Services.Containers {
+		pipeline.Services = append(pipeline.Services,
+			toContainer(container),
+		)
+	}
+
+	for _, container := range from.Pipeline.Containers {
+		pipeline.Steps = append(pipeline.Steps,
+			toContainer(container),
+		)
+	}
+	pipeline.Volumes = toVolumes(from)
+	pipeline.Trigger.Branch.Include = from.Branches.Include
+	pipeline.Trigger.Branch.Exclude = from.Branches.Exclude
+
+	manifest := &droneyaml.Manifest{}
+	manifest.Resources = append(manifest.Resources, pipeline)
+
+	secrets := toSecrets(from)
+	if secrets != nil {
+		manifest.Resources = append(manifest.Resources, secrets)
+	}
+
+	buf := new(bytes.Buffer)
+	pretty.Print(buf, manifest)
+	return buf.Bytes(), nil
+}
+
+func toContainer(from *Container) *droneyaml.Container {
+	return &droneyaml.Container{
+		Name:        from.Name,
+		Image:       from.Image,
+		Detach:      from.Detached,
+		Command:     from.Command,
+		Commands:    from.Commands,
+		DNS:         from.DNS,
+		DNSSearch:   from.DNSSearch,
+		Entrypoint:  from.Entrypoint,
+		Environment: toEnvironment(from),
+		ExtraHosts:  from.ExtraHosts,
+		Pull:        toPullPolicy(from.Pull),
+		Privileged:  from.Privileged,
+		Settings:    toSettings(from.Vargs),
+		Volumes:     toVolumeMounts(from.Volumes),
+		When:        toConditions(from.Constraints),
+	}
+}
+
+// helper function converts the legacy constraint syntax
+// to the new condition syntax.
+func toConditions(from Constraints) droneyaml.Conditions {
+	return droneyaml.Conditions{
+		Ref: droneyaml.Condition{
+			Include: from.Ref.Include,
+			Exclude: from.Ref.Exclude,
+		},
+		Repo: droneyaml.Condition{
+			Include: from.Repo.Include,
+			Exclude: from.Repo.Exclude,
+		},
+		Instance: droneyaml.Condition{
+			Include: from.Instance.Include,
+			Exclude: from.Instance.Exclude,
+		},
+		Target: droneyaml.Condition{
+			Include: from.Environment.Include,
+			Exclude: from.Environment.Exclude,
+		},
+		Event: droneyaml.Condition{
+			Include: from.Event.Include,
+			Exclude: from.Event.Exclude,
+		},
+		Branch: droneyaml.Condition{
+			Include: from.Branch.Include,
+			Exclude: from.Branch.Exclude,
+		},
+		Status: droneyaml.Condition{
+			Include: from.Status.Include,
+			Exclude: from.Status.Exclude,
+		},
+	}
+}
+
+// helper function converts the legacy environment syntax
+// to the new environment syntax.
+func toEnvironment(from *Container) map[string]*droneyaml.Variable {
+	envs := map[string]*droneyaml.Variable{}
+	for key, val := range from.Environment.Map {
+		envs[key] = &droneyaml.Variable{
+			Value: val,
+		}
+	}
+	for _, val := range from.Secrets.Secrets {
+		name := strings.ToUpper(val.Target)
+		envs[name] = &droneyaml.Variable{
+			Secret: val.Source,
+		}
+	}
+	return envs
+}
+
+// helper function converts the legacy image pull syntax
+// to the new pull policy syntax.
+func toPullPolicy(pull bool) string {
+	switch pull {
+	case true:
+		return "always"
+	default:
+		return "default"
+	}
+}
+
+// helper function converts the legacy secret syntax to the
+// new secret variable syntax.
+func toSecrets(from *Config) *droneyaml.Secret {
+	secret := &droneyaml.Secret{}
+	secret.Kind = "secret"
+	secret.Type = "general"
+	secret.External = map[string]droneyaml.ExternalData{}
+	for key, val := range from.Secrets {
+		external := droneyaml.ExternalData{}
+		if val.Driver == "vault" {
+			if val.DriverOpts != nil {
+				external.Path = val.DriverOpts["path"]
+				external.Name = val.DriverOpts["key"]
+			}
+		} else if val.Path != "" {
+			external.Path = val.Path
+		} else {
+			external.Path = val.Vault
+		}
+		secret.External[key] = external
+	}
+	if len(secret.External) == 0 {
+		return nil
+	}
+	return secret
+}
+
+// helper function converts the legacy vargs syntax to the
+// new environment syntax.
+func toSettings(from map[string]interface{}) map[string]*droneyaml.Parameter {
+	params := map[string]*droneyaml.Parameter{}
+	for key, val := range from {
+		params[key] = &droneyaml.Parameter{
+			Value: val,
+		}
+	}
+	return params
+}
+
+// helper function converts the legacy volume syntax
+// to the new volume mount syntax.
+func toVolumeMounts(from []*Volume) []*droneyaml.VolumeMount {
+	to := []*droneyaml.VolumeMount{}
+	for _, v := range from {
+		to = append(to, &droneyaml.VolumeMount{
+			Name:      fmt.Sprintf("%x", v.Source),
+			MountPath: v.Destination,
+		})
+	}
+	return to
+}
+
+// helper function converts the legacy volume syntax
+// to the new volume mount syntax.
+func toVolumes(from *Config) []*droneyaml.Volume {
+	set := map[string]struct{}{}
+	to := []*droneyaml.Volume{}
+
+	containers := []*Container{}
+	containers = append(containers, from.Pipeline.Containers...)
+	containers = append(containers, from.Services.Containers...)
+
+	for _, container := range containers {
+		for _, v := range container.Volumes {
+			name := fmt.Sprintf("%x", v.Source)
+			if _, ok := set[name]; ok {
+				continue
+			}
+			set[name] = struct{}{}
+			to = append(to, &droneyaml.Volume{
+				Name: name,
+				HostPath: &droneyaml.VolumeHostPath{
+					Path: v.Source,
+				},
+			})
+		}
+	}
+	return to
+}

yaml/converter/legacy/internal/config_test.go

@@ -0,0 +1,52 @@
+package yaml
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+)
+
+func TestConvert(t *testing.T) {
+	tests := []struct {
+		before, after string
+	}{
+		{
+			before: "testdata/simple.yml",
+			after:  "testdata/simple.yml.golden",
+		},
+		{
+			before: "testdata/vault_1.yml",
+			after:  "testdata/vault_1.yml.golden",
+		},
+		{
+			before: "testdata/vault_2.yml",
+			after:  "testdata/vault_2.yml.golden",
+		},
+		{
+			before: "testdata/vault_3.yml",
+			after:  "testdata/vault_3.yml.golden",
+		},
+	}
+
+	for _, test := range tests {
+		a, err := ioutil.ReadFile(test.before)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		b, err := ioutil.ReadFile(test.after)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		c, err := Convert(a)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		if bytes.Equal(b, c) == false {
+			t.Errorf("Unexpected yaml conversion of %s", test.before)
+			t.Log(string(c))
+		}
+	}
+}

yaml/converter/legacy/internal/constraint.go

@@ -0,0 +1,69 @@
+package yaml
+
+type (
+	// Constraints defines a set of runtime constraints.
+	Constraints struct {
+		Ref         Constraint
+		Repo        Constraint
+		Instance    Constraint
+		Environment Constraint
+		Event       Constraint
+		Branch      Constraint
+		Status      Constraint
+	}
+
+	// Constraint defines a runtime constraint.
+	Constraint struct {
+		Include []string
+		Exclude []string
+	}
+
+	// ConstraintMap defines a runtime constraint map.
+	ConstraintMap struct {
+		Include map[string]string
+		Exclude map[string]string
+	}
+)
+
+// UnmarshalYAML unmarshals the constraint.
+func (c *Constraint) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var out1 = struct {
+		Include StringSlice
+		Exclude StringSlice
+	}{}
+
+	var out2 StringSlice
+
+	unmarshal(&out1)
+	unmarshal(&out2)
+
+	c.Exclude = out1.Exclude
+	c.Include = append(
+		out1.Include,
+		out2...,
+	)
+	return nil
+}
+
+// UnmarshalYAML unmarshals the constraint map.
+func (c *ConstraintMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	out1 := struct {
+		Include map[string]string
+		Exclude map[string]string
+	}{
+		Include: map[string]string{},
+		Exclude: map[string]string{},
+	}
+
+	out2 := map[string]string{}
+
+	unmarshal(&out1)
+	unmarshal(&out2)
+
+	c.Include = out1.Include
+	c.Exclude = out1.Exclude
+	for k, v := range out2 {
+		c.Include[k] = v
+	}
+	return nil
+}

yaml/converter/legacy/internal/container.go

@@ -0,0 +1,58 @@
+package yaml
+
+import (
+	"fmt"
+
+	"gopkg.in/yaml.v2"
+)
+
+type (
+	// Containers represents an ordered list of containers.
+	Containers struct {
+		Containers []*Container
+	}
+
+	// Container represents a Docker container.
+	Container struct {
+		Command     StringSlice            `yaml:"command,omitempty"`
+		Commands    StringSlice            `yaml:"commands,omitempty"`
+		Detached    bool                   `yaml:"detach,omitempty"`
+		Devices     []string               `yaml:"devices,omitempty"`
+		ErrIgnore   bool                   `yaml:"allow_failure,omitempty"`
+		Tmpfs       []string               `yaml:"tmpfs,omitempty"`
+		DNS         StringSlice            `yaml:"dns,omitempty"`
+		DNSSearch   StringSlice            `yaml:"dns_search,omitempty"`
+		Entrypoint  StringSlice            `yaml:"entrypoint,omitempty"`
+		Environment SliceMap               `yaml:"environment,omitempty"`
+		ExtraHosts  []string               `yaml:"extra_hosts,omitempty"`
+		Image       string                 `yaml:"image,omitempty"`
+		Name        string                 `yaml:"name,omitempty"`
+		Privileged  bool                   `yaml:"privileged,omitempty"`
+		Pull        bool                   `yaml:"pull,omitempty"`
+		Shell       string                 `yaml:"shell,omitempty"`
+		Volumes     []*Volume              `yaml:"volumes,omitempty"`
+		Secrets     Secrets                `yaml:"secrets,omitempty"`
+		Constraints Constraints            `yaml:"when,omitempty"`
+		Vargs       map[string]interface{} `yaml:",inline"`
+	}
+)
+
+// UnmarshalYAML implements the Unmarshaller interface.
+func (c *Containers) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	slice := yaml.MapSlice{}
+	if err := unmarshal(&slice); err != nil {
+		return err
+	}
+
+	for _, s := range slice {
+		container := Container{}
+		out, _ := yaml.Marshal(s.Value)
+
+		if err := yaml.Unmarshal(out, &container); err != nil {
+			return err
+		}
+		container.Name = fmt.Sprintf("%v", s.Key)
+		c.Containers = append(c.Containers, &container)
+	}
+	return nil
+}

yaml/converter/legacy/internal/container_test.go

@@ -0,0 +1 @@
+package yaml

yaml/converter/legacy/internal/secret.go

@@ -0,0 +1,30 @@
+package yaml
+
+type (
+	// Secrets represents a list of container secrets.
+	Secrets struct {
+		Secrets []*Secret
+	}
+
+	// Secret represents a container secret.
+	Secret struct {
+		Source string
+		Target string
+	}
+)
+
+// UnmarshalYAML implements the Unmarshaller interface.
+func (s *Secrets) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var strslice []string
+	err := unmarshal(&strslice)
+	if err == nil {
+		for _, str := range strslice {
+			s.Secrets = append(s.Secrets, &Secret{
+				Source: str,
+				Target: str,
+			})
+		}
+		return nil
+	}
+	return unmarshal(&s.Secrets)
+}

yaml/converter/legacy/internal/secret_test.go

@@ -0,0 +1,62 @@
+package yaml
+
+import (
+	"reflect"
+	"testing"
+
+	"gopkg.in/yaml.v2"
+)
+
+func TestUnmarshalSecrets(t *testing.T) {
+	testdata := []struct {
+		from string
+		want []*Secret
+	}{
+		{
+			from: "[ mysql_username, mysql_password]",
+			want: []*Secret{
+				{
+					Source: "mysql_username",
+					Target: "mysql_username",
+				},
+				{
+					Source: "mysql_password",
+					Target: "mysql_password",
+				},
+			},
+		},
+		{
+			from: "[ { source: mysql_prod_username, target: mysql_username } ]",
+			want: []*Secret{
+				{
+					Source: "mysql_prod_username",
+					Target: "mysql_username",
+				},
+			},
+		},
+		{
+			from: "[ { source: mysql_prod_username, target: mysql_username }, { source: redis_username, target: redis_username } ]",
+			want: []*Secret{
+				{
+					Source: "mysql_prod_username",
+					Target: "mysql_username",
+				},
+				{
+					Source: "redis_username",
+					Target: "redis_username",
+				},
+			},
+		},
+	}
+
+	for _, test := range testdata {
+		in := []byte(test.from)
+		got := Secrets{}
+		err := yaml.Unmarshal(in, &got)
+		if err != nil {
+			t.Error(err)
+		} else if !reflect.DeepEqual(test.want, got.Secrets) {
+			t.Errorf("got secret %v want %v", got.Secrets, test.want)
+		}
+	}
+}

yaml/converter/legacy/internal/slice_map.go

@@ -0,0 +1,32 @@
+package yaml
+
+import "strings"
+
+// SliceMap represents a slice or map of key pairs.
+type SliceMap struct {
+	Map map[string]string
+}
+
+// UnmarshalYAML implements custom Yaml unmarshaling.
+func (s *SliceMap) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	s.Map = map[string]string{}
+	err := unmarshal(&s.Map)
+	if err == nil {
+		return nil
+	}
+
+	var slice []string
+	err = unmarshal(&slice)
+	if err != nil {
+		return err
+	}
+	for _, v := range slice {
+		parts := strings.SplitN(v, "=", 2)
+		if len(parts) == 2 {
+			key := parts[0]
+			val := parts[1]
+			s.Map[key] = val
+		}
+	}
+	return nil
+}

yaml/converter/legacy/internal/slice_map_test.go

@@ -0,0 +1,41 @@
+package yaml
+
+import (
+	"reflect"
+	"testing"
+
+	"gopkg.in/yaml.v2"
+)
+
+func TestMapSlice(t *testing.T) {
+	var tests = []struct {
+		yaml string
+		want map[string]string
+	}{
+		{
+			yaml: "[ foo=bar, baz=qux ]",
+			want: map[string]string{"foo": "bar", "baz": "qux"},
+		},
+		{
+			yaml: "{ foo: bar, baz: qux }",
+			want: map[string]string{"foo": "bar", "baz": "qux"},
+		},
+	}
+
+	for _, test := range tests {
+		var got SliceMap
+
+		if err := yaml.Unmarshal([]byte(test.yaml), &got); err != nil {
+			t.Error(err)
+		}
+
+		if !reflect.DeepEqual(got.Map, test.want) {
+			t.Errorf("Got map %v want %v", got, test.want)
+		}
+	}
+
+	var got SliceMap
+	if err := yaml.Unmarshal([]byte("1"), &got); err == nil {
+		t.Errorf("Want error unmarshaling invalid map value.")
+	}
+}

yaml/converter/legacy/internal/string_slice.go

@@ -0,0 +1,20 @@
+package yaml
+
+// StringSlice represents a slice of strings or a string.
+type StringSlice []string
+
+// UnmarshalYAML implements the Unmarshaller interface.
+func (s *StringSlice) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var stringType string
+	if err := unmarshal(&stringType); err == nil {
+		*s = []string{stringType}
+		return nil
+	}
+
+	var sliceType []string
+	if err := unmarshal(&sliceType); err != nil {
+		return err
+	}
+	*s = sliceType
+	return nil
+}

yaml/converter/legacy/internal/string_slice_test.go

@@ -0,0 +1,45 @@
+package yaml
+
+import (
+	"reflect"
+	"testing"
+
+	"gopkg.in/yaml.v2"
+)
+
+func TestStringSlice(t *testing.T) {
+	var tests = []struct {
+		yaml string
+		want []string
+	}{
+		{
+			yaml: "hello world",
+			want: []string{"hello world"},
+		},
+		{
+			yaml: "[ hello, world ]",
+			want: []string{"hello", "world"},
+		},
+		{
+			yaml: "42",
+			want: []string{"42"},
+		},
+	}
+
+	for _, test := range tests {
+		var got StringSlice
+
+		if err := yaml.Unmarshal([]byte(test.yaml), &got); err != nil {
+			t.Error(err)
+		}
+
+		if !reflect.DeepEqual([]string(got), test.want) {
+			t.Errorf("Got slice %v want %v", got, test.want)
+		}
+	}
+
+	var got StringSlice
+	if err := yaml.Unmarshal([]byte("{}"), &got); err == nil {
+		t.Errorf("Want error unmarshaling invalid string or slice value.")
+	}
+}

yaml/converter/legacy/internal/testdata/simple.yml

@@ -0,0 +1,48 @@
+workspace:
+  base: /go
+  path: src/github.com/octocat/hello-world
+
+pipeline:
+  build:
+    image: golang
+    commands:
+    - go get
+    - go build
+    volumes:
+    - /tmp/go:/go/bin
+    environment:
+    - GOOS=linux
+    - GOARCH=amd64
+
+  test:
+    image: golang:latest
+    volumes:
+    - /tmp/go:/go/bin
+    commands:
+    - go test -v
+  
+  docker:
+    image: plugins/docker
+    secrets:
+    - docker_username
+    - docker_password
+    repo: octocat/hello-world
+    when:
+      branch: master
+
+  slack:
+    image: plugins/slack
+    secrets:
+    - source: token
+      target: slack_token
+    channel: general
+
+services:
+  database:
+    image: mysql
+    environment:
+      MYSQL_USERNAME: foo
+      MYSQL_PASSWORD: bar
+
+branches:
+- master

yaml/converter/legacy/internal/testdata/simple.yml.golden

@@ -0,0 +1,76 @@
+---
+kind: pipeline
+name: default
+
+platform:
+  os: linux
+  arch: amd64
+
+workspace:
+  base: /go
+  path: src/github.com/octocat/hello-world
+
+steps:
+- name: build
+  pull: default
+  image: golang
+  commands:
+  - go get
+  - go build
+  environment:
+    GOARCH: amd64
+    GOOS: linux
+  volumes:
+  - name: 2f746d702f676f
+    path: /go/bin
+
+- name: test
+  pull: default
+  image: golang:latest
+  commands:
+  - go test -v
+  volumes:
+  - name: 2f746d702f676f
+    path: /go/bin
+
+- name: docker
+  pull: default
+  image: plugins/docker
+  settings:
+    repo: octocat/hello-world
+  environment:
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    DOCKER_USERNAME:
+      from_secret: docker_username
+  when:
+    branch:
+    - master
+
+- name: slack
+  pull: default
+  image: plugins/slack
+  settings:
+    channel: general
+  environment:
+    SLACK_TOKEN:
+      from_secret: token
+
+services:
+- name: database
+  pull: default
+  image: mysql
+  environment:
+    MYSQL_PASSWORD: bar
+    MYSQL_USERNAME: foo
+
+volumes:
+- name: 2f746d702f676f
+  host:
+    path: /tmp/go
+
+trigger:
+  branch:
+  - master
+
+...

yaml/converter/legacy/internal/testdata/vault_1.yml

@@ -0,0 +1,16 @@
+pipeline:
+  docker:
+    image: plugins/docker
+    secrets: [ docker_username, docker_password ]
+    repo: octocat/hello-world
+
+secrets:
+  docker_username:
+    driver: vault
+    driver_opts:
+      path: secret/docker/username
+  docker_password:
+    driver: vault
+    driver_opts:
+      path: secret/docker
+      key: password

yaml/converter/legacy/internal/testdata/vault_1.yml.golden

@@ -0,0 +1,31 @@
+---
+kind: pipeline
+name: default
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: docker
+  pull: default
+  image: plugins/docker
+  settings:
+    repo: octocat/hello-world
+  environment:
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+    DOCKER_USERNAME:
+      from_secret: docker_username
+
+---
+kind: secret
+type: general
+external_data:
+  docker_password:
+    path: secret/docker
+    name: password
+  docker_username:
+    path: secret/docker/username
+
+...

yaml/converter/legacy/internal/testdata/vault_2.yml

@@ -0,0 +1,11 @@
+pipeline:
+  docker:
+    image: plugins/docker
+    secrets: [ docker_username, docker_password ]
+    repo: octocat/hello-world
+
+secrets:
+  docker_username:
+    path: secret/docker/username
+  docker_password:
+    path: secret/docker/password