From 98eb77b4c58a783b3f2efce8fed4e5d9d1f09469 Mon Sep 17 00:00:00 2001 From: Brad Rydzewski Date: Tue, 22 Jan 2019 15:44:17 -0800 Subject: [PATCH] initial commit --- .drone.yml | 20 ++ .gitignore | 3 + README.md | 33 ++ main.go | 295 ++++++++++++++++ samples/complex.yml | 71 ++++ samples/simple.yml | 22 ++ yaml/build.go | 40 +++ yaml/build_test.go | 44 +++ yaml/compiler/clone.go | 75 ++++ yaml/compiler/clone_test.go | 103 ++++++ yaml/compiler/compiler.go | 303 +++++++++++++++++ yaml/compiler/convert.go | 76 +++++ yaml/compiler/convert_test.go | 154 +++++++++ yaml/compiler/dind.go | 37 ++ yaml/compiler/dind_test.go | 63 ++++ yaml/compiler/encode.go | 53 +++ yaml/compiler/encode_test.go | 59 ++++ yaml/compiler/image/image.go | 67 ++++ yaml/compiler/image/image_test.go | 295 ++++++++++++++++ yaml/compiler/internal/rand/rand.go | 36 ++ yaml/compiler/script_posix.go | 70 ++++ yaml/compiler/script_posix_test.go | 1 + yaml/compiler/script_win.go | 58 ++++ yaml/compiler/script_win_test.go | 1 + yaml/compiler/skip.go | 37 ++ yaml/compiler/skip_test.go | 105 ++++++ yaml/compiler/step.go | 180 ++++++++++ yaml/compiler/transform/auths.go | 29 ++ yaml/compiler/transform/auths_test.go | 49 +++ yaml/compiler/transform/combine.go | 13 + yaml/compiler/transform/combine_test.go | 33 ++ yaml/compiler/transform/env.go | 15 + yaml/compiler/transform/env_test.go | 30 ++ yaml/compiler/transform/filter.go | 72 ++++ yaml/compiler/transform/filter_test.go | 139 ++++++++ yaml/compiler/transform/labels.go | 23 ++ yaml/compiler/transform/labels_test.go | 48 +++ yaml/compiler/transform/limits.go | 29 ++ yaml/compiler/transform/limits_test.go | 84 +++++ yaml/compiler/transform/netrc.go | 73 ++++ yaml/compiler/transform/netrc_test.go | 78 +++++ yaml/compiler/transform/network.go | 14 + yaml/compiler/transform/network_test.go | 29 ++ yaml/compiler/transform/proxy.go | 39 +++ yaml/compiler/transform/proxy_test.go | 52 +++ yaml/compiler/transform/secret.go | 62 ++++ yaml/compiler/transform/secret_test.go | 107 ++++++ yaml/compiler/transform/volume.go | 53 +++ yaml/compiler/transform/volume_test.go | 69 ++++ yaml/compiler/workspace.go | 144 ++++++++ yaml/compiler/workspace_test.go | 144 ++++++++ yaml/cond.go | 85 +++++ yaml/cond_test.go | 166 +++++++++ yaml/converter/bitbucket/config.go | 104 ++++++ yaml/converter/bitbucket/config_test.go | 1 + yaml/converter/bitbucket/convert.go | 82 +++++ yaml/converter/bitbucket/convert_test.go | 46 +++ .../converter/bitbucket/testdata/sample1.yaml | 32 ++ .../bitbucket/testdata/sample1.yaml.golden | 31 ++ .../converter/bitbucket/testdata/sample2.yaml | 40 +++ .../bitbucket/testdata/sample2.yaml.golden | 20 ++ yaml/converter/circleci/config.go | 76 +++++ yaml/converter/circleci/docker.go | 28 ++ yaml/converter/circleci/docker_test.go | 1 + yaml/converter/circleci/run.go | 34 ++ yaml/converter/circleci/run_test.go | 11 + yaml/converter/circleci/testdata/sample1.yml | 22 ++ .../circleci/testdata/sample1.yml.golden | 0 yaml/converter/convert.go | 51 +++ yaml/converter/convert_test.go | 1 + yaml/converter/gitlab/config.go | 125 +++++++ yaml/converter/gitlab/convert.go | 95 ++++++ yaml/converter/gitlab/convert_test.go | 57 ++++ yaml/converter/gitlab/testdata/example1.yml | 11 + .../gitlab/testdata/example1.yml.golden | 20 ++ yaml/converter/gitlab/testdata/example2.yml | 16 + .../gitlab/testdata/example2.yml.golden | 34 ++ yaml/converter/gitlab/testdata/example3.yml | 16 + .../gitlab/testdata/example3.yml.golden | 24 ++ yaml/converter/gitlab/testdata/example4.yml | 22 ++ .../gitlab/testdata/example4.yml.golden | 54 +++ yaml/converter/internal/string_slice.go | 20 ++ yaml/converter/internal/string_slice_test.go | 45 +++ yaml/converter/legacy/convert.go | 9 + yaml/converter/legacy/internal/config.go | 247 ++++++++++++++ yaml/converter/legacy/internal/config_test.go | 52 +++ yaml/converter/legacy/internal/constraint.go | 69 ++++ yaml/converter/legacy/internal/container.go | 58 ++++ .../legacy/internal/container_test.go | 1 + yaml/converter/legacy/internal/secret.go | 30 ++ yaml/converter/legacy/internal/secret_test.go | 62 ++++ yaml/converter/legacy/internal/slice_map.go | 32 ++ .../legacy/internal/slice_map_test.go | 41 +++ .../converter/legacy/internal/string_slice.go | 20 ++ .../legacy/internal/string_slice_test.go | 45 +++ .../legacy/internal/testdata/simple.yml | 48 +++ .../internal/testdata/simple.yml.golden | 76 +++++ .../legacy/internal/testdata/vault_1.yml | 16 + .../internal/testdata/vault_1.yml.golden | 31 ++ .../legacy/internal/testdata/vault_2.yml | 11 + .../internal/testdata/vault_2.yml.golden | 30 ++ .../legacy/internal/testdata/vault_3.yml | 11 + .../internal/testdata/vault_3.yml.golden | 30 ++ yaml/converter/legacy/internal/volume.go | 29 ++ yaml/converter/legacy/internal/volume_test.go | 43 +++ yaml/converter/legacy/match.go | 14 + yaml/converter/legacy/match_test.go | 33 ++ yaml/converter/legacy/testdata/legacy.yml | 7 + yaml/cron.go | 44 +++ yaml/cron_test.go | 28 ++ yaml/env.go | 30 ++ yaml/env_test.go | 39 +++ yaml/linter/config.go | 65 ++++ yaml/linter/config_test.go | 79 +++++ yaml/linter/linter.go | 215 ++++++++++++ yaml/linter/linter_test.go | 246 ++++++++++++++ yaml/linter/testdata/duplicate_name.yml | 23 ++ yaml/linter/testdata/duplicate_step.yml | 16 + .../testdata/duplicate_step_service.yml | 14 + yaml/linter/testdata/invalid_arch.yml | 14 + yaml/linter/testdata/invalid_os.yml | 13 + yaml/linter/testdata/missing_build_image.yml | 8 + yaml/linter/testdata/missing_dep.yml | 34 ++ yaml/linter/testdata/missing_image.yml | 9 + yaml/linter/testdata/missing_name.yml | 9 + yaml/linter/testdata/pipeline_device.yml | 19 ++ yaml/linter/testdata/pipeline_dns.yml | 12 + yaml/linter/testdata/pipeline_dns_search.yml | 13 + yaml/linter/testdata/pipeline_extra_hosts.yml | 13 + yaml/linter/testdata/pipeline_port_host.yml | 17 + yaml/linter/testdata/pipeline_privileged.yml | 17 + .../testdata/pipeline_volume_invalid_name.yml | 13 + yaml/linter/testdata/service_device.yml | 19 ++ yaml/linter/testdata/service_port_host.yml | 17 + yaml/linter/testdata/simple.yml | 38 +++ yaml/linter/testdata/volume_empty_dir.yml | 20 ++ .../testdata/volume_empty_dir_memory.yml | 21 ++ yaml/linter/testdata/volume_host_path.yml | 21 ++ yaml/linter/testdata/volume_invalid_name.yml | 20 ++ yaml/manifest.go | 91 +++++ yaml/manifest_test.go | 40 +++ yaml/param.go | 31 ++ yaml/param_test.go | 39 +++ yaml/parse.go | 155 +++++++++ yaml/parse_test.go | 95 ++++++ yaml/pipeline.go | 137 ++++++++ yaml/pipeline_test.go | 14 + yaml/port.go | 30 ++ yaml/port_test.go | 45 +++ yaml/pretty/container.go | 219 ++++++++++++ yaml/pretty/container_test.go | 1 + yaml/pretty/cron.go | 41 +++ yaml/pretty/cron_test.go | 12 + yaml/pretty/pipeline.go | 265 +++++++++++++++ yaml/pretty/pipeline_test.go | 147 ++++++++ yaml/pretty/pretty.go | 29 ++ yaml/pretty/pretty_test.go | 43 +++ yaml/pretty/registry.go | 20 ++ yaml/pretty/registry_test.go | 12 + yaml/pretty/secret.go | 88 +++++ yaml/pretty/secret_test.go | 21 ++ yaml/pretty/signature.go | 15 + yaml/pretty/signature_test.go | 12 + yaml/pretty/testdata/cron.yml | 12 + yaml/pretty/testdata/cron.yml.golden | 11 + yaml/pretty/testdata/manifest.yml | 64 ++++ yaml/pretty/testdata/manifest.yml.golden | 65 ++++ yaml/pretty/testdata/pipeline.yml | 151 +++++++++ yaml/pretty/testdata/pipeline.yml.golden | 0 yaml/pretty/testdata/pipeline_build_long.yml | 21 ++ .../testdata/pipeline_build_long.yml.golden | 23 ++ yaml/pretty/testdata/pipeline_build_short.yml | 7 + .../testdata/pipeline_build_short.yml.golden | 13 + yaml/pretty/testdata/pipeline_clone_depth.yml | 13 + .../testdata/pipeline_clone_depth.yml.golden | 19 ++ .../testdata/pipeline_clone_disable.yml | 12 + .../pipeline_clone_disable.yml.golden | 18 + .../testdata/pipeline_clone_skip_verify.yml | 12 + .../pipeline_clone_skip_verify.yml.golden | 18 + yaml/pretty/testdata/pipeline_concurrency.yml | 12 + .../testdata/pipeline_concurrency.yml.golden | 18 + yaml/pretty/testdata/pipeline_depends.yml | 20 ++ .../testdata/pipeline_depends.yml.golden | 27 ++ yaml/pretty/testdata/pipeline_node.yml | 19 ++ yaml/pretty/testdata/pipeline_node.yml.golden | 26 ++ yaml/pretty/testdata/pipeline_ports.yml | 13 + .../pretty/testdata/pipeline_ports.yml.golden | 19 ++ yaml/pretty/testdata/pipeline_push.yml | 6 + yaml/pretty/testdata/pipeline_push.yml.golden | 13 + yaml/pretty/testdata/pipeline_resources.yml | 15 + .../testdata/pipeline_resources.yml.golden | 22 ++ yaml/pretty/testdata/pipeline_settings.yml | 14 + .../testdata/pipeline_settings.yml.golden | 21 ++ yaml/pretty/testdata/pipeline_trigger.yml | 20 ++ .../testdata/pipeline_trigger.yml.golden | 27 ++ yaml/pretty/testdata/pipeline_volumes.yml | 30 ++ .../testdata/pipeline_volumes.yml.golden | 36 ++ yaml/pretty/testdata/pipeline_workspace.yml | 13 + .../testdata/pipeline_workspace.yml.golden | 20 ++ yaml/pretty/testdata/registry.yml | 4 + yaml/pretty/testdata/registry.yml.golden | 8 + yaml/pretty/testdata/secret.yml | 6 + yaml/pretty/testdata/secret.yml.golden | 12 + yaml/pretty/testdata/secret_extern.yml | 10 + yaml/pretty/testdata/secret_extern.yml.golden | 12 + yaml/pretty/testdata/services.yml | 32 ++ yaml/pretty/testdata/services.yml.golden | 40 +++ yaml/pretty/testdata/signature.yml | 2 + yaml/pretty/testdata/signature.yml.golden | 5 + yaml/pretty/util.go | 78 +++++ yaml/pretty/util_test.go | 28 ++ yaml/pretty/writer.go | 319 ++++++++++++++++++ yaml/pretty/writer_test.go | 54 +++ yaml/push.go | 25 ++ yaml/push_test.go | 44 +++ yaml/registry.go | 30 ++ yaml/registry_test.go | 34 ++ yaml/secret.go | 37 ++ yaml/secret_test.go | 36 ++ yaml/signature.go | 29 ++ yaml/signature_test.go | 27 ++ yaml/signer/signer.go | 151 +++++++++ yaml/signer/signer_test.go | 142 ++++++++ yaml/signer/testdata/invalid_signature.yml | 22 ++ yaml/signer/testdata/missing_signature.yml | 12 + yaml/signer/testdata/signed.yml | 22 ++ yaml/testdata/cron.yml | 8 + yaml/testdata/cron.yml.golden | 13 + yaml/testdata/manifest.yml | 12 + yaml/testdata/manifest.yml.golden | 18 + yaml/testdata/pipeline.yml | 81 +++++ yaml/testdata/pipeline.yml.golden | 129 +++++++ yaml/testdata/registry.yml | 5 + yaml/testdata/registry.yml.golden | 9 + yaml/testdata/secret.yml | 7 + yaml/testdata/secret.yml.golden | 10 + yaml/testdata/signature.yml | 3 + yaml/testdata/signature.yml.golden | 6 + yaml/unit.go | 75 ++++ yaml/unit_test.go | 85 +++++ 240 files changed, 11424 insertions(+) create mode 100644 .drone.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 main.go create mode 100644 samples/complex.yml create mode 100644 samples/simple.yml create mode 100644 yaml/build.go create mode 100644 yaml/build_test.go create mode 100644 yaml/compiler/clone.go create mode 100644 yaml/compiler/clone_test.go create mode 100644 yaml/compiler/compiler.go create mode 100644 yaml/compiler/convert.go create mode 100644 yaml/compiler/convert_test.go create mode 100644 yaml/compiler/dind.go create mode 100644 yaml/compiler/dind_test.go create mode 100644 yaml/compiler/encode.go create mode 100644 yaml/compiler/encode_test.go create mode 100644 yaml/compiler/image/image.go create mode 100644 yaml/compiler/image/image_test.go create mode 100644 yaml/compiler/internal/rand/rand.go create mode 100644 yaml/compiler/script_posix.go create mode 100644 yaml/compiler/script_posix_test.go create mode 100644 yaml/compiler/script_win.go create mode 100644 yaml/compiler/script_win_test.go create mode 100644 yaml/compiler/skip.go create mode 100644 yaml/compiler/skip_test.go create mode 100644 yaml/compiler/step.go create mode 100644 yaml/compiler/transform/auths.go create mode 100644 yaml/compiler/transform/auths_test.go create mode 100644 yaml/compiler/transform/combine.go create mode 100644 yaml/compiler/transform/combine_test.go create mode 100644 yaml/compiler/transform/env.go create mode 100644 yaml/compiler/transform/env_test.go create mode 100644 yaml/compiler/transform/filter.go create mode 100644 yaml/compiler/transform/filter_test.go create mode 100644 yaml/compiler/transform/labels.go create mode 100644 yaml/compiler/transform/labels_test.go create mode 100644 yaml/compiler/transform/limits.go create mode 100644 yaml/compiler/transform/limits_test.go create mode 100644 yaml/compiler/transform/netrc.go create mode 100644 yaml/compiler/transform/netrc_test.go create mode 100644 yaml/compiler/transform/network.go create mode 100644 yaml/compiler/transform/network_test.go create mode 100644 yaml/compiler/transform/proxy.go create mode 100644 yaml/compiler/transform/proxy_test.go create mode 100644 yaml/compiler/transform/secret.go create mode 100644 yaml/compiler/transform/secret_test.go create mode 100644 yaml/compiler/transform/volume.go create mode 100644 yaml/compiler/transform/volume_test.go create mode 100644 yaml/compiler/workspace.go create mode 100644 yaml/compiler/workspace_test.go create mode 100644 yaml/cond.go create mode 100644 yaml/cond_test.go create mode 100644 yaml/converter/bitbucket/config.go create mode 100644 yaml/converter/bitbucket/config_test.go create mode 100644 yaml/converter/bitbucket/convert.go create mode 100644 yaml/converter/bitbucket/convert_test.go create mode 100644 yaml/converter/bitbucket/testdata/sample1.yaml create mode 100644 yaml/converter/bitbucket/testdata/sample1.yaml.golden create mode 100644 yaml/converter/bitbucket/testdata/sample2.yaml create mode 100644 yaml/converter/bitbucket/testdata/sample2.yaml.golden create mode 100644 yaml/converter/circleci/config.go create mode 100644 yaml/converter/circleci/docker.go create mode 100644 yaml/converter/circleci/docker_test.go create mode 100644 yaml/converter/circleci/run.go create mode 100644 yaml/converter/circleci/run_test.go create mode 100644 yaml/converter/circleci/testdata/sample1.yml create mode 100644 yaml/converter/circleci/testdata/sample1.yml.golden create mode 100644 yaml/converter/convert.go create mode 100644 yaml/converter/convert_test.go create mode 100644 yaml/converter/gitlab/config.go create mode 100644 yaml/converter/gitlab/convert.go create mode 100644 yaml/converter/gitlab/convert_test.go create mode 100644 yaml/converter/gitlab/testdata/example1.yml create mode 100644 yaml/converter/gitlab/testdata/example1.yml.golden create mode 100644 yaml/converter/gitlab/testdata/example2.yml create mode 100644 yaml/converter/gitlab/testdata/example2.yml.golden create mode 100644 yaml/converter/gitlab/testdata/example3.yml create mode 100644 yaml/converter/gitlab/testdata/example3.yml.golden create mode 100644 yaml/converter/gitlab/testdata/example4.yml create mode 100644 yaml/converter/gitlab/testdata/example4.yml.golden create mode 100644 yaml/converter/internal/string_slice.go create mode 100644 yaml/converter/internal/string_slice_test.go create mode 100644 yaml/converter/legacy/convert.go create mode 100644 yaml/converter/legacy/internal/config.go create mode 100644 yaml/converter/legacy/internal/config_test.go create mode 100644 yaml/converter/legacy/internal/constraint.go create mode 100644 yaml/converter/legacy/internal/container.go create mode 100644 yaml/converter/legacy/internal/container_test.go create mode 100644 yaml/converter/legacy/internal/secret.go create mode 100644 yaml/converter/legacy/internal/secret_test.go create mode 100644 yaml/converter/legacy/internal/slice_map.go create mode 100644 yaml/converter/legacy/internal/slice_map_test.go create mode 100644 yaml/converter/legacy/internal/string_slice.go create mode 100644 yaml/converter/legacy/internal/string_slice_test.go create mode 100644 yaml/converter/legacy/internal/testdata/simple.yml create mode 100644 yaml/converter/legacy/internal/testdata/simple.yml.golden create mode 100644 yaml/converter/legacy/internal/testdata/vault_1.yml create mode 100644 yaml/converter/legacy/internal/testdata/vault_1.yml.golden create mode 100644 yaml/converter/legacy/internal/testdata/vault_2.yml create mode 100644 yaml/converter/legacy/internal/testdata/vault_2.yml.golden create mode 100644 yaml/converter/legacy/internal/testdata/vault_3.yml create mode 100644 yaml/converter/legacy/internal/testdata/vault_3.yml.golden create mode 100644 yaml/converter/legacy/internal/volume.go create mode 100644 yaml/converter/legacy/internal/volume_test.go create mode 100644 yaml/converter/legacy/match.go create mode 100644 yaml/converter/legacy/match_test.go create mode 100644 yaml/converter/legacy/testdata/legacy.yml create mode 100644 yaml/cron.go create mode 100644 yaml/cron_test.go create mode 100644 yaml/env.go create mode 100644 yaml/env_test.go create mode 100644 yaml/linter/config.go create mode 100644 yaml/linter/config_test.go create mode 100644 yaml/linter/linter.go create mode 100644 yaml/linter/linter_test.go create mode 100644 yaml/linter/testdata/duplicate_name.yml create mode 100644 yaml/linter/testdata/duplicate_step.yml create mode 100644 yaml/linter/testdata/duplicate_step_service.yml create mode 100644 yaml/linter/testdata/invalid_arch.yml create mode 100644 yaml/linter/testdata/invalid_os.yml create mode 100644 yaml/linter/testdata/missing_build_image.yml create mode 100644 yaml/linter/testdata/missing_dep.yml create mode 100644 yaml/linter/testdata/missing_image.yml create mode 100644 yaml/linter/testdata/missing_name.yml create mode 100644 yaml/linter/testdata/pipeline_device.yml create mode 100644 yaml/linter/testdata/pipeline_dns.yml create mode 100644 yaml/linter/testdata/pipeline_dns_search.yml create mode 100644 yaml/linter/testdata/pipeline_extra_hosts.yml create mode 100644 yaml/linter/testdata/pipeline_port_host.yml create mode 100644 yaml/linter/testdata/pipeline_privileged.yml create mode 100644 yaml/linter/testdata/pipeline_volume_invalid_name.yml create mode 100644 yaml/linter/testdata/service_device.yml create mode 100644 yaml/linter/testdata/service_port_host.yml create mode 100644 yaml/linter/testdata/simple.yml create mode 100644 yaml/linter/testdata/volume_empty_dir.yml create mode 100644 yaml/linter/testdata/volume_empty_dir_memory.yml create mode 100644 yaml/linter/testdata/volume_host_path.yml create mode 100644 yaml/linter/testdata/volume_invalid_name.yml create mode 100644 yaml/manifest.go create mode 100644 yaml/manifest_test.go create mode 100644 yaml/param.go create mode 100644 yaml/param_test.go create mode 100644 yaml/parse.go create mode 100644 yaml/parse_test.go create mode 100644 yaml/pipeline.go create mode 100644 yaml/pipeline_test.go create mode 100644 yaml/port.go create mode 100644 yaml/port_test.go create mode 100644 yaml/pretty/container.go create mode 100644 yaml/pretty/container_test.go create mode 100644 yaml/pretty/cron.go create mode 100644 yaml/pretty/cron_test.go create mode 100644 yaml/pretty/pipeline.go create mode 100644 yaml/pretty/pipeline_test.go create mode 100644 yaml/pretty/pretty.go create mode 100644 yaml/pretty/pretty_test.go create mode 100644 yaml/pretty/registry.go create mode 100644 yaml/pretty/registry_test.go create mode 100644 yaml/pretty/secret.go create mode 100644 yaml/pretty/secret_test.go create mode 100644 yaml/pretty/signature.go create mode 100644 yaml/pretty/signature_test.go create mode 100644 yaml/pretty/testdata/cron.yml create mode 100644 yaml/pretty/testdata/cron.yml.golden create mode 100644 yaml/pretty/testdata/manifest.yml create mode 100644 yaml/pretty/testdata/manifest.yml.golden create mode 100644 yaml/pretty/testdata/pipeline.yml create mode 100644 yaml/pretty/testdata/pipeline.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_build_long.yml create mode 100644 yaml/pretty/testdata/pipeline_build_long.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_build_short.yml create mode 100644 yaml/pretty/testdata/pipeline_build_short.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_clone_depth.yml create mode 100644 yaml/pretty/testdata/pipeline_clone_depth.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_clone_disable.yml create mode 100644 yaml/pretty/testdata/pipeline_clone_disable.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_clone_skip_verify.yml create mode 100644 yaml/pretty/testdata/pipeline_clone_skip_verify.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_concurrency.yml create mode 100644 yaml/pretty/testdata/pipeline_concurrency.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_depends.yml create mode 100644 yaml/pretty/testdata/pipeline_depends.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_node.yml create mode 100644 yaml/pretty/testdata/pipeline_node.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_ports.yml create mode 100644 yaml/pretty/testdata/pipeline_ports.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_push.yml create mode 100644 yaml/pretty/testdata/pipeline_push.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_resources.yml create mode 100644 yaml/pretty/testdata/pipeline_resources.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_settings.yml create mode 100644 yaml/pretty/testdata/pipeline_settings.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_trigger.yml create mode 100644 yaml/pretty/testdata/pipeline_trigger.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_volumes.yml create mode 100644 yaml/pretty/testdata/pipeline_volumes.yml.golden create mode 100644 yaml/pretty/testdata/pipeline_workspace.yml create mode 100644 yaml/pretty/testdata/pipeline_workspace.yml.golden create mode 100644 yaml/pretty/testdata/registry.yml create mode 100644 yaml/pretty/testdata/registry.yml.golden create mode 100644 yaml/pretty/testdata/secret.yml create mode 100644 yaml/pretty/testdata/secret.yml.golden create mode 100644 yaml/pretty/testdata/secret_extern.yml create mode 100644 yaml/pretty/testdata/secret_extern.yml.golden create mode 100644 yaml/pretty/testdata/services.yml create mode 100644 yaml/pretty/testdata/services.yml.golden create mode 100644 yaml/pretty/testdata/signature.yml create mode 100644 yaml/pretty/testdata/signature.yml.golden create mode 100644 yaml/pretty/util.go create mode 100644 yaml/pretty/util_test.go create mode 100644 yaml/pretty/writer.go create mode 100644 yaml/pretty/writer_test.go create mode 100644 yaml/push.go create mode 100644 yaml/push_test.go create mode 100644 yaml/registry.go create mode 100644 yaml/registry_test.go create mode 100644 yaml/secret.go create mode 100644 yaml/secret_test.go create mode 100644 yaml/signature.go create mode 100644 yaml/signature_test.go create mode 100644 yaml/signer/signer.go create mode 100644 yaml/signer/signer_test.go create mode 100644 yaml/signer/testdata/invalid_signature.yml create mode 100644 yaml/signer/testdata/missing_signature.yml create mode 100644 yaml/signer/testdata/signed.yml create mode 100644 yaml/testdata/cron.yml create mode 100644 yaml/testdata/cron.yml.golden create mode 100644 yaml/testdata/manifest.yml create mode 100644 yaml/testdata/manifest.yml.golden create mode 100644 yaml/testdata/pipeline.yml create mode 100644 yaml/testdata/pipeline.yml.golden create mode 100644 yaml/testdata/registry.yml create mode 100644 yaml/testdata/registry.yml.golden create mode 100644 yaml/testdata/secret.yml create mode 100644 yaml/testdata/secret.yml.golden create mode 100644 yaml/testdata/signature.yml create mode 100644 yaml/testdata/signature.yml.golden create mode 100644 yaml/unit.go create mode 100644 yaml/unit_test.go diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..f99f2fa --- /dev/null +++ b/.drone.yml @@ -0,0 +1,20 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +workspace: + base: /go + path: src/github.com/drone/drone-yaml + +steps: +- name: test + image: golang + commands: + - go get -t -v ./... + - go test ./... + +... diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9d5fbf3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +NOTES* +*.out +*.env \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..90a067e --- /dev/null +++ b/README.md @@ -0,0 +1,33 @@ +Package yaml provides a parser, linter, formatter and compiler for the [drone](https://github.com/drone/drone) configuration file format. + +Lint the yaml file: + +```text +$ drone-yaml lint samples/simple.yml +``` + +Format the yaml file: + +```text +$ drone-yaml fmt samples/simple.yml +$ drone-yaml fmt samples/simple.yml --save +``` + +Sign the yaml file using a 32-bit secret key: + +```text +$ drone-yaml sign 642909eb4c3d47e33999235c0598353c samples/simple.yml +$ drone-yaml sign 642909eb4c3d47e33999235c0598353c samples/simple.yml --save +``` + +Verify the yaml file signature: + +```text +$ drone-yaml verify 642909eb4c3d47e33999235c0598353c samples/simple.yml +``` + +Compile the yaml file: + +```text +$ drone-yaml compile samples/simple.yml > samples/simple.json +``` diff --git a/main.go b/main.go new file mode 100644 index 0000000..ab4d76c --- /dev/null +++ b/main.go @@ -0,0 +1,295 @@ +package main + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "strings" + + "github.com/drone/drone-runtime/engine" + "github.com/drone/drone-yaml/yaml" + "github.com/drone/drone-yaml/yaml/compiler" + "github.com/drone/drone-yaml/yaml/compiler/transform" + "github.com/drone/drone-yaml/yaml/linter" + "github.com/drone/drone-yaml/yaml/pretty" + "github.com/drone/drone-yaml/yaml/signer" + + "gopkg.in/alecthomas/kingpin.v2" +) + +var ( + format = kingpin.Command("fmt", "format the yaml file") + formatPriv = format.Flag("privileged", "privileged mode").Short('p').Bool() + formatSave = format.Flag("save", "save result to source").Short('s').Bool() + formatFile = format.Arg("source", "source file location").Default(".drone.yml").File() + + lint = kingpin.Command("lint", "lint the yaml file") + lintPriv = lint.Flag("privileged", "privileged mode").Short('p').Bool() + lintFile = lint.Arg("source", "source file location").Default(".drone.yml").File() + + sign = kingpin.Command("sign", "sign the yaml file") + signKey = sign.Arg("key", "secret key").Required().String() + signFile = sign.Arg("source", "source file location").Default(".drone.yml").File() + signSave = sign.Flag("save", "save result to source").Short('s').Bool() + + verify = kingpin.Command("verify", "verify the yaml signature") + verifyKey = verify.Arg("key", "secret key").Required().String() + verifyFile = verify.Arg("source", "source file location").Default(".drone.yml").File() + + compile = kingpin.Command("compile", "compile the yaml file") + compileIn = compile.Arg("source", "source file location").Default(".drone.yml").File() + compileName = compile.Flag("name", "pipeline name").String() +) + +func main() { + switch kingpin.Parse() { + case format.FullCommand(): + kingpin.FatalIfError(runFormat(), "") + case lint.FullCommand(): + kingpin.FatalIfError(runLint(), "") + case sign.FullCommand(): + kingpin.FatalIfError(runSign(), "") + case verify.FullCommand(): + kingpin.FatalIfError(runVerify(), "") + case compile.FullCommand(): + kingpin.FatalIfError(runCompile(), "") + } +} + +func runFormat() error { + f := *formatFile + m, err := yaml.Parse(f) + if err != nil { + return err + } + + b := new(bytes.Buffer) + pretty.Print(b, m) + + if *formatSave { + return ioutil.WriteFile(f.Name(), b.Bytes(), 0644) + } + _, err = io.Copy(os.Stderr, b) + return err +} + +func runLint() error { + f := *lintFile + m, err := yaml.Parse(f) + if err != nil { + return err + } + for _, r := range m.Resources { + err := linter.Lint(r, *lintPriv) + if err != nil { + return err + } + } + return nil +} + +func runSign() error { + f := *signFile + d, err := ioutil.ReadAll(f) + if err != nil { + return err + } + + k := signer.KeyString(*signKey) + + if *signSave { + out, err := signer.SignUpdate(d, k) + if err != nil { + return err + } + return ioutil.WriteFile(f.Name(), out, 0644) + } + + hmac, err := signer.Sign(d, k) + if err != nil { + return err + } + fmt.Println(hmac) + return nil +} + +func runVerify() error { + f := *verifyFile + d, err := ioutil.ReadAll(f) + if err != nil { + return err + } + + k := signer.KeyString(*verifyKey) + ok, err := signer.Verify(d, k) + if err != nil { + return err + } else if !ok { + return errors.New("cannot verify yaml signature") + } + + fmt.Println("success: yaml signature verified") + return nil +} + +var ( + trusted = compile.Flag("trusted", "trusted mode").Bool() + labels = compile.Flag("label", "container labels").StringMap() + clone = compile.Flag("clone", "clone step").Bool() + volume = compile.Flag("volume", "attached volumes").StringMap() + network = compile.Flag("network", "attached networks").Strings() + environ = compile.Flag("env", "environment variable").StringMap() + dind = compile.Flag("dind", "dind images").Default("plugins/docker").Strings() + event = compile.Flag("event", "event type").PlaceHolder("").Enum("push", "pull_request", "tag", "deployment") + repo = compile.Flag("repo", "repository name").PlaceHolder("octocat/hello-world").String() + remote = compile.Flag("git-remote", "git remote url").PlaceHolder("https://github.com/octocat/hello-world.git").String() + branch = compile.Flag("git-branch", "git commit branch").PlaceHolder("master").String() + ref = compile.Flag("git-ref", "git commit ref").PlaceHolder("refs/heads/master").String() + sha = compile.Flag("git-sha", "git commit sha").String() + creds = compile.Flag("git-creds", "git credentials").URLList() + instance = compile.Flag("instance", "drone instance hostname").PlaceHolder("drone.company.com").String() + deploy = compile.Flag("deploy-to", "target deployment").PlaceHolder("production").String() + secrets = compile.Flag("secret", "secret variable").StringMap() + registries = compile.Flag("registry", "registry credentials").URLList() + username = compile.Flag("netrc-username", "netrc username").PlaceHolder("").String() + password = compile.Flag("netrc-password", "netrc password").PlaceHolder("x-oauth-basic").String() + machine = compile.Flag("netrc-machine", "netrc machine").PlaceHolder("github.com").String() + memlimit = compile.Flag("mem-limit", "memory limit").PlaceHolder("1GB").Bytes() + cpulimit = compile.Flag("cpu-limit", "cpu limit").PlaceHolder("2").Int64() +) + +func runCompile() error { + m, err := yaml.Parse(*compileIn) + if err != nil { + return err + } + + var p *yaml.Pipeline + for _, r := range m.Resources { + v, ok := r.(*yaml.Pipeline) + if !ok { + continue + } + if *compileName == "" || + *compileName == v.Name { + p = v + break + } + } + + if p == nil { + return errors.New("cannot find pipeline resource") + } + + // the user has the option to disable the git clone + // if the pipeline is being executed on the local + // codebase. + if *clone == false { + p.Clone.Disable = true + } + + var auths []*engine.DockerAuth + for _, uri := range *registries { + if uri.User == nil { + log.Fatalln("Expect registry format [user]:[password]@hostname") + } + password, ok := uri.User.Password() + if !ok { + log.Fatalln("Invalid or missing registry password") + } + auths = append(auths, &engine.DockerAuth{ + Address: uri.Host, + Username: uri.User.Username(), + Password: password, + }) + } + + comp := new(compiler.Compiler) + comp.GitCredentialsFunc = defaultCreds // TODO create compiler.GitCredentialsFunc and compiler.GlobalGitCredentialsFunc + comp.NetrcFunc = nil // TODO create compiler.NetrcFunc and compiler.GlobalNetrcFunc + comp.PrivilegedFunc = compiler.DindFunc(*dind) + comp.SkipFunc = compiler.SkipFunc( + compiler.SkipData{ + Branch: *branch, + Event: *event, + Instance: *instance, + Ref: *ref, + Repo: *repo, + Target: *deploy, + }, + ) + comp.TransformFunc = transform.Combine( + transform.WithAuths(auths), + transform.WithEnviron(*environ), + transform.WithEnviron(defaultEnvs()), + transform.WithLables(*labels), + transform.WithLimits(int64(*memlimit), int64(*cpulimit)), + transform.WithNetrc(*machine, *username, *password), + transform.WithNetworks(*network), + transform.WithProxy(), + transform.WithSecrets(*secrets), + transform.WithVolumes(*volume), + ) + compiled := comp.Compile(p) + + // // for drone-exec we will need to change the workspace + // // to a host volume mount, to the current working dir. + // for _, volume := range compiled.Docker.Volumes { + // if volume.Metadata.Name == "workspace" { + // volume.EmptyDir = nil + // volume.HostPath = &engine.VolumeHostPath{ + // Path: "", // pwd + // } + // break + // } + // } + // // then we need to change the base mount for every container + // // to use the workspace base + path. + // for _, container := range compiled.Steps { + // for _, volume := range container.Volumes { + // if volume.Name == "workspace" { + // volume.Path = container.Envs["DRONE_WORKSPACE"] + // } + // } + // } + + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(compiled) +} + +// helper function returns the git credential function, +// used to return a git credentials file. +func defaultCreds() []byte { + urls := *creds + if len(urls) == 0 { + return nil + } + var buf bytes.Buffer + for _, url := range urls { + buf.WriteString(url.String()) + buf.WriteByte('\n') + } + return buf.Bytes() +} + +// helper function returns the minimum required environment +// variables to clone a repository. All other environment +// variables should be passed via the --env flag. +func defaultEnvs() map[string]string { + envs := map[string]string{} + envs["DRONE_COMMIT_BRANCH"] = *branch + envs["DRONE_COMMIT_SHA"] = *sha + envs["DRONE_COMMIT_REF"] = *ref + envs["DRONE_REMOTE_URL"] = *remote + envs["DRONE_BUILD_EVENT"] = *event + if strings.HasPrefix(*ref, "refs/tags/") { + envs["DRONE_TAG"] = strings.TrimPrefix(*ref, "refs/tags/") + } + return envs +} diff --git a/samples/complex.yml b/samples/complex.yml new file mode 100644 index 0000000..e94c188 --- /dev/null +++ b/samples/complex.yml @@ -0,0 +1,71 @@ +kind: pipeline +name: build + +steps: +- name: backend + image: golang:1.11 + commands: + - go build + - go test -v + +- name: frontend + image: node + commands: + - npm install + - npm run test + - npm run lint + +services: +- name: redis + image: redis:latest + ports: + - 6379 + volumes: + - name: foo + path: /bar + +volumes: +- name: foo + temp: {} + +--- +kind: pipeline +name: notify + +steps: +- name: notify + image: plugins/slack + settings: + room: general + token: + $secret: token + +node: + disk: ssd + +depends_on: +- build + +--- +kind: cron +name: nightly +spec: + schedule: "1 * * * *" + branch: master + deployment: + target: production + +--- +kind: secret +type: encrypted + +data: + token: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + +--- +kind: registry +type: encrypted + +data: + index.drone.io: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + diff --git a/samples/simple.yml b/samples/simple.yml new file mode 100644 index 0000000..e9aa277 --- /dev/null +++ b/samples/simple.yml @@ -0,0 +1,22 @@ +kind: pipeline +name: default + +steps: +- name: backend + image: golang:1.11 + commands: + - go build + - go test -v + +- name: frontend + image: node + commands: + - npm install + - npm run test + - npm run lint + +services: +- name: redis + image: redis:latest + ports: + - 6379 diff --git a/yaml/build.go b/yaml/build.go new file mode 100644 index 0000000..73ab569 --- /dev/null +++ b/yaml/build.go @@ -0,0 +1,40 @@ +package yaml + +type ( + // Build configures a Docker build. + Build struct { + Args map[string]string `json:"args,omitempty"` + CacheFrom []string `json:"cache_from,omitempty" yaml:"cache_from"` + Context string `json:"context,omitempty"` + Dockerfile string `json:"dockerfile,omitempty"` + Image string `json:"image,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + } + + // build is a tempoary type used to unmarshal + // the Build struct when long format is used. + build struct { + Args map[string]string + CacheFrom []string `yaml:"cache_from"` + Context string + Dockerfile string + Image string + Labels map[string]string + } +) + +// UnmarshalYAML implements yaml unmarshalling. +func (b *Build) UnmarshalYAML(unmarshal func(interface{}) error) error { + d := new(build) + err := unmarshal(&d.Image) + if err != nil { + err = unmarshal(d) + } + b.Args = d.Args + b.CacheFrom = d.CacheFrom + b.Context = d.Context + b.Dockerfile = d.Dockerfile + b.Labels = d.Labels + b.Image = d.Image + return err +} diff --git a/yaml/build_test.go b/yaml/build_test.go new file mode 100644 index 0000000..4f39526 --- /dev/null +++ b/yaml/build_test.go @@ -0,0 +1,44 @@ +package yaml + +import ( + "testing" + + "gopkg.in/yaml.v2" +) + +func TestBuild(t *testing.T) { + tests := []struct { + yaml string + image string + }{ + { + yaml: "bar", + image: "bar", + }, + { + yaml: "{ image: foo }", + image: "foo", + }, + } + for _, test := range tests { + in := []byte(test.yaml) + out := new(Build) + err := yaml.Unmarshal(in, out) + if err != nil { + t.Error(err) + return + } + if got, want := out.Image, test.image; got != want { + t.Errorf("Want image %q, got %q", want, got) + } + } +} + +func TestBuildError(t *testing.T) { + in := []byte("[]") + out := new(Build) + err := yaml.Unmarshal(in, out) + if err == nil { + t.Errorf("Expect unmarshal error") + } +} diff --git a/yaml/compiler/clone.go b/yaml/compiler/clone.go new file mode 100644 index 0000000..f88c940 --- /dev/null +++ b/yaml/compiler/clone.go @@ -0,0 +1,75 @@ +package compiler + +import ( + "strconv" + + "github.com/drone/drone-runtime/engine" + "github.com/drone/drone-yaml/yaml" + "github.com/drone/drone-yaml/yaml/compiler/internal/rand" +) + +// default name of the clone step. +const cloneStepName = "clone" + +// helper function returns the preferred clone image +// based on the target architecture. +func cloneImage(src *yaml.Pipeline) string { + switch { + case src.Platform.OS == "linux" && src.Platform.Arch == "arm": + return "drone/git:linux-arm" + case src.Platform.OS == "linux" && src.Platform.Arch == "arm64": + return "drone/git:linux-arm64" + case src.Platform.OS == "windows": + return "drone/git:windows-1803" + default: + return "drone/git" + } +} + +// helper function configures the clone depth parameter, +// specific to the clone plugin. +// +// TODO(bradrydzewski) rename to setupCloneParams +func setupCloneDepth(src *yaml.Pipeline, dst *engine.Step) { + if depth := src.Clone.Depth; depth > 0 { + dst.Envs["PLUGIN_DEPTH"] = strconv.Itoa(depth) + } + if skipVerify := src.Clone.SkipVerify; skipVerify { + dst.Envs["GIT_SSL_NO_VERIFY"] = "true" + dst.Envs["PLUGIN_SKIP_VERIFY"] = "true" + } +} + +// helper function configures the .git-clone credentials +// file. The file is mounted into the container, pointed +// to by XDG_CONFIG_HOME +// see https://git-scm.com/docs/git-credential-store +func setupCloneCredentials(spec *engine.Spec, dst *engine.Step, data []byte) { + if len(data) == 0 { + return + } + // TODO(bradrydzewski) we may need to update the git + // clone plugin to configure the git credential store. + dst.Files = append(dst.Files, &engine.FileMount{ + Name: ".git-credentials", + Path: "/root/.git-credentials", + }) + spec.Files = append(spec.Files, &engine.File{ + Metadata: engine.Metadata{ + UID: rand.String(), + Namespace: spec.Metadata.Namespace, + Name: ".git-credentials", + }, + Data: data, + }) +} + +// helper function creates a default container configuration +// for the clone stage. The clone stage is automatically +// added to each pipeline. +func createClone(src *yaml.Pipeline) *yaml.Container { + return &yaml.Container{ + Name: cloneStepName, + Image: cloneImage(src), + } +} diff --git a/yaml/compiler/clone_test.go b/yaml/compiler/clone_test.go new file mode 100644 index 0000000..83d4013 --- /dev/null +++ b/yaml/compiler/clone_test.go @@ -0,0 +1,103 @@ +package compiler + +import ( + "testing" + + "github.com/drone/drone-runtime/engine" + "github.com/drone/drone-yaml/yaml" +) + +func TestCloneImage(t *testing.T) { + tests := []struct { + platform yaml.Platform + image string + }{ + { + platform: yaml.Platform{OS: "linux", Arch: "amd64"}, + image: "drone/git", + }, + { + platform: yaml.Platform{OS: "linux", Arch: "arm"}, + image: "drone/git:linux-arm", + }, + { + platform: yaml.Platform{OS: "linux", Arch: "arm64"}, + image: "drone/git:linux-arm64", + }, + { + platform: yaml.Platform{OS: "windows", Arch: "amd64"}, + image: "drone/git:windows-1803", + }, + { + platform: yaml.Platform{}, + image: "drone/git", + }, + } + for _, test := range tests { + pipeline := &yaml.Pipeline{Platform: test.platform} + image := cloneImage(pipeline) + if got, want := image, test.image; got != want { + t.Errorf("Want clone image %s, got %s", want, got) + } + } +} + +func TestSetupCloneDepth(t *testing.T) { + // test zero depth + src := &yaml.Pipeline{ + Clone: yaml.Clone{ + Depth: 0, + }, + } + dst := &engine.Step{ + Envs: map[string]string{}, + } + setupCloneDepth(src, dst) + if _, ok := dst.Envs["PLUGIN_DEPTH"]; ok { + t.Errorf("Expect depth ignored when zero value") + } + + // test non-zero depth + src = &yaml.Pipeline{ + Clone: yaml.Clone{ + Depth: 50, + }, + } + dst = &engine.Step{ + Envs: map[string]string{}, + } + setupCloneDepth(src, dst) + if got, want := dst.Envs["PLUGIN_DEPTH"], "50"; got != want { + t.Errorf("Expect depth %s, got %s", want, got) + } +} + +func TestSetupCloneSkipVerify(t *testing.T) { + // test zero depth + src := &yaml.Pipeline{ + Clone: yaml.Clone{ + SkipVerify: false, + }, + } + dst := &engine.Step{ + Envs: map[string]string{}, + } + setupCloneDepth(src, dst) + if _, ok := dst.Envs["PLUGIN_SKIP_VERIFY"]; ok { + t.Errorf("Expect skip verify not set") + } + + // test non-zero depth + src = &yaml.Pipeline{ + Clone: yaml.Clone{ + SkipVerify: true, + }, + } + dst = &engine.Step{ + Envs: map[string]string{}, + } + setupCloneDepth(src, dst) + if got, want := dst.Envs["PLUGIN_SKIP_VERIFY"], "true"; got != want { + t.Errorf("Expect skip verify %s, got %s", want, got) + } +} diff --git a/yaml/compiler/compiler.go b/yaml/compiler/compiler.go new file mode 100644 index 0000000..3912d3c --- /dev/null +++ b/yaml/compiler/compiler.go @@ -0,0 +1,303 @@ +package compiler + +import ( + "github.com/drone/drone-runtime/engine" + "github.com/drone/drone-yaml/yaml" + "github.com/drone/drone-yaml/yaml/compiler/image" + "github.com/drone/drone-yaml/yaml/compiler/internal/rand" +) + +// A Compiler compiles the pipeline configuration to an +// intermediate representation that can be executed by +// the Drone runtime engine. +type Compiler struct { + // GitCredentialsFunc returns a .git-credentials file + // that can be used by the default clone step to + // authenticate to the remote repository. + GitCredentialsFunc func() []byte + + // NetrcFunc returns a .netrc file that can be used by + // the default clone step to authenticate to the remote + // repository. + NetrcFunc func() []byte + + // PrivilegedFunc returns true if the container should + // be started in privileged mode. The intended use is + // for plugins that run Docker-in-Docker. This will be + // deprecated in a future release. + PrivilegedFunc func(*yaml.Container) bool + + // SkipFunc returns true if the step should be skipped. + // The skip function can be used to evaluate the when + // clause of each step, and return true if it should + // be skipped. + SkipFunc func(*yaml.Container) bool + + // TransformFunc can be used to modify the compiled + // output prior to completion. This can be useful when + // you need to programatically modify the output, + // set defaults, etc. + TransformFunc func(*engine.Spec) + + // WorkspaceFunc can be used to set the workspace volume + // that is created for the entire pipeline. The primary + // use case for this function is running local builds, + // where the workspace is mounted to the host machine + // working directory. + WorkspaceFunc func(*engine.Spec) + + // WorkspaceMountFunc can be used to override the default + // workspace volume mount. + WorkspaceMountFunc func(step *engine.Step, base, path, full string) +} + +// Compile returns an intermediate representation of the +// pipeline configuration that can be executed by the +// Drone runtime engine. +func (c *Compiler) Compile(from *yaml.Pipeline) *engine.Spec { + namespace := rand.String() + + isSerial := true + for _, step := range from.Steps { + if len(step.DependsOn) != 0 { + isSerial = false + break + } + } + + spec := &engine.Spec{ + Metadata: engine.Metadata{ + UID: namespace, + Name: namespace, + Namespace: namespace, + Labels: map[string]string{ + "io.drone.pipeline.name": from.Name, + "io.drone.pipeline.kind": from.Kind, + "io.drone.pipeline.type": from.Type, + }, + }, + Platform: engine.Platform{ + OS: from.Platform.OS, + Arch: from.Platform.Arch, + Version: from.Platform.Version, + Variant: from.Platform.Variant, + }, + Docker: &engine.DockerConfig{}, + Files: nil, + Secrets: nil, + } + + // create the default workspace path. If a container + // does not specify a working directory it defaults + // to the workspace path. + base, dir, workspace := createWorkspace(from) + + // create the default workspace volume definition. + // the volume will be mounted to each container in + // the pipeline. + c.setupWorkspace(spec) + + // for each volume defined in the yaml configuration + // file, convert to a runtime volume and append to the + // specification. + for _, from := range from.Volumes { + to := &engine.Volume{ + Metadata: engine.Metadata{ + UID: rand.String(), + Name: from.Name, + Namespace: namespace, + Labels: map[string]string{}, + }, + } + if from.EmptyDir != nil { + // if the yaml configuration specifies an empty + // directory volume (data volume) or an in-memory + // file system. + to.EmptyDir = &engine.VolumeEmptyDir{ + Medium: from.EmptyDir.Medium, + SizeLimit: int64(from.EmptyDir.SizeLimit), + } + } else if from.HostPath != nil { + // if the yaml configuration specifies a bind + // mount to the host machine. + to.HostPath = &engine.VolumeHostPath{ + Path: from.HostPath.Path, + } + } + spec.Docker.Volumes = append(spec.Docker.Volumes, to) + } + + if !from.Clone.Disable { + src := createClone(from) + dst := createStep(spec, src) + dst.Docker.PullPolicy = engine.PullIfNotExists + setupCloneDepth(from, dst) + setupCloneCredentials(spec, dst, c.gitCredentials()) + setupWorkingDir(src, dst, workspace) + setupWorkspaceEnv(dst, base, dir, workspace) + c.setupWorkspaceMount(dst, base, dir, workspace) + spec.Steps = append(spec.Steps, dst) + } + + // for each pipeline service defined in the yaml + // configuration file, convert to a runtime step + // and append to the specification. + for _, service := range from.Services { + step := createStep(spec, service) + // note that all services are automatically + // set to run in detached mode. + step.Detach = true + setupWorkingDir(service, step, workspace) + setupWorkspaceEnv(step, base, dir, workspace) + c.setupWorkspaceMount(step, base, dir, workspace) + // if the skip callback function returns true, + // modify the runtime step to never execute. + if c.skip(service) { + step.RunPolicy = engine.RunNever + } + // if the step is a plugin and should be executed + // in privileged mode, set the privileged flag. + if c.privileged(service) { + step.Docker.Privileged = true + } + // if the clone step is enabled, the service should + // not start until the clone step is complete. Add + // the clone step as a dependency in the graph. + if isSerial == false && from.Clone.Disable == false { + step.DependsOn = append(step.DependsOn, cloneStepName) + } + spec.Steps = append(spec.Steps, step) + } + + // rename will store a list of container names + // that should be mapped to their temporary alias. + rename := map[string]string{} + + // for each pipeline step defined in the yaml + // configuration file, convert to a runtime step + // and append to the specification. + for _, container := range from.Steps { + var step *engine.Step + switch { + case container.Build != nil: + step = createBuildStep(spec, container) + rename[container.Build.Image] = step.Metadata.UID + default: + step = createStep(spec, container) + } + setupWorkingDir(container, step, workspace) + setupWorkspaceEnv(step, base, dir, workspace) + c.setupWorkspaceMount(step, base, dir, workspace) + // if the skip callback function returns true, + // modify the runtime step to never execute. + if c.skip(container) { + step.RunPolicy = engine.RunNever + } + // if the step is a plugin and should be executed + // in privileged mode, set the privileged flag. + if c.privileged(container) { + step.Docker.Privileged = true + } + // if the clone step is enabled, the step should + // not start until the clone step is complete. If + // no dependencies are defined, at a minimum, the + // step depends on the initial clone step completing. + if isSerial == false && from.Clone.Disable == false && len(step.DependsOn) == 0 { + step.DependsOn = append(step.DependsOn, cloneStepName) + } + spec.Steps = append(spec.Steps, step) + } + + // if the pipeline includes any build and publish + // steps we should create an entry for the host + // machine docker socket. + if spec.Docker != nil && len(rename) > 0 { + v := &engine.Volume{ + Metadata: engine.Metadata{ + UID: rand.String(), + Name: "_docker_socket", + Namespace: namespace, + Labels: map[string]string{}, + }, + HostPath: &engine.VolumeHostPath{ + Path: "/var/run/docker.sock", + }, + } + spec.Docker.Volumes = append(spec.Docker.Volumes, v) + } + + // images created during the pipeline are assigned a + // random alias. All references to the origin image + // name must be changed to the alias. + for _, step := range spec.Steps { + for k, v := range rename { + if image.MatchTag(step.Docker.Image, k) { + img := image.Trim(step.Docker.Image) + ":" + v + step.Docker.Image = image.Expand(img) + } + } + } + + // executes user-defined transformations before the + // final specification is returned. + if c.TransformFunc != nil { + c.TransformFunc(spec) + } + + return spec +} + +// return a .git-credentials file. If the user-defined +// function is nil, a nil credentials file is returned. +func (c *Compiler) gitCredentials() []byte { + if c.GitCredentialsFunc != nil { + return c.GitCredentialsFunc() + } + return nil +} + +// return a .netrc file. If the user-defined function is +// nil, a nil netrc file is returned. +func (c *Compiler) netrc() []byte { + if c.NetrcFunc != nil { + return c.NetrcFunc() + } + return nil +} + +// return true if the step should be executed in privileged +// mode. If the user-defined privileged function is nil, +// a default value of false is returned. +func (c *Compiler) privileged(container *yaml.Container) bool { + if c.PrivilegedFunc != nil { + return c.PrivilegedFunc(container) + } + return false +} + +// return true if the step should be skipped. If the +// user-defined skip function is nil, a defalt skip +// function is used that always returns true (i.e. do not skip). +func (c *Compiler) skip(container *yaml.Container) bool { + if c.SkipFunc != nil { + return c.SkipFunc(container) + } + return false +} + +func (c *Compiler) setupWorkspace(spec *engine.Spec) { + if c.WorkspaceFunc != nil { + c.WorkspaceFunc(spec) + return + } + CreateWorkspace(spec) + return +} + +func (c *Compiler) setupWorkspaceMount(step *engine.Step, base, path, full string) { + if c.WorkspaceMountFunc != nil { + c.WorkspaceMountFunc(step, base, path, full) + return + } + MountWorkspace(step, base, path, full) +} diff --git a/yaml/compiler/convert.go b/yaml/compiler/convert.go new file mode 100644 index 0000000..f3ef05b --- /dev/null +++ b/yaml/compiler/convert.go @@ -0,0 +1,76 @@ +package compiler + +import ( + "strings" + + "github.com/drone/drone-runtime/engine" + "github.com/drone/drone-yaml/yaml" +) + +func toIgnoreErr(from *yaml.Container) bool { + return strings.EqualFold(from.Failure, "ignore") +} + +func toPorts(from *yaml.Container) []*engine.Port { + var ports []*engine.Port + for _, port := range from.Ports { + ports = append(ports, toPort(port)) + } + return ports +} + +func toPort(from *yaml.Port) *engine.Port { + return &engine.Port{ + Port: from.Port, + Host: from.Host, + Protocol: from.Protocol, + } +} + +func toPullPolicy(from *yaml.Container) engine.PullPolicy { + switch strings.ToLower(from.Pull) { + case "always": + return engine.PullAlways + case "if-not-exists": + return engine.PullIfNotExists + case "never": + return engine.PullNever + default: + return engine.PullDefault + } +} + +func toRunPolicy(from *yaml.Container) engine.RunPolicy { + onFailure := from.When.Status.Match("failure") && len(from.When.Status.Include) > 0 + onSuccess := from.When.Status.Match("success") + switch { + case onFailure && onSuccess: + return engine.RunAlways + case onFailure: + return engine.RunOnFailure + case onSuccess: + return engine.RunOnSuccess + default: + return engine.RunNever + } +} + +func toResources(from *yaml.Container) *engine.Resources { + if from.Resources == nil { + return nil + } + return &engine.Resources{ + Limits: toResourceObject(from.Resources.Limits), + Requests: toResourceObject(from.Resources.Requests), + } +} + +func toResourceObject(from *yaml.ResourceObject) *engine.ResourceObject { + if from == nil { + return nil + } + return &engine.ResourceObject{ + CPU: int64(from.CPU), + Memory: int64(from.Memory), + } +} diff --git a/yaml/compiler/convert_test.go b/yaml/compiler/convert_test.go new file mode 100644 index 0000000..d9f3fbb --- /dev/null +++ b/yaml/compiler/convert_test.go @@ -0,0 +1,154 @@ +package compiler + +import ( + "testing" + + "github.com/drone/drone-runtime/engine" + "github.com/drone/drone-yaml/yaml" + "github.com/google/go-cmp/cmp" +) + +func Test_toIgnoreErr(t *testing.T) { + tests := []struct { + mode string + want bool + }{ + {"Ignore", true}, + {"ignore", true}, + {"fail", false}, + } + for _, test := range tests { + from := &yaml.Container{Failure: test.mode} + if toIgnoreErr(from) != test.want { + t.Errorf("unexpected ignore error for %s", test.mode) + } + } +} + +func Test_toPullPolicy(t *testing.T) { + tests := []struct { + mode string + want engine.PullPolicy + }{ + {"", engine.PullDefault}, + {"always", engine.PullAlways}, + {"if-not-exists", engine.PullIfNotExists}, + {"never", engine.PullNever}, + } + for _, test := range tests { + from := &yaml.Container{Pull: test.mode} + if toPullPolicy(from) != test.want { + t.Errorf("unexpected pull policy for %s", test.mode) + } + } +} + +func Test_toRunPolicy(t *testing.T) { + tests := []struct { + cond yaml.Condition + want engine.RunPolicy + }{ + {yaml.Condition{}, engine.RunOnSuccess}, + {yaml.Condition{Include: []string{"success"}}, engine.RunOnSuccess}, + {yaml.Condition{Include: []string{"failure"}}, engine.RunOnFailure}, + {yaml.Condition{Include: []string{"success", "failure"}}, engine.RunAlways}, + {yaml.Condition{Exclude: []string{"success", "failure"}}, engine.RunNever}, + } + for _, test := range tests { + from := &yaml.Container{When: yaml.Conditions{Status: test.cond}} + if toRunPolicy(from) != test.want { + t.Errorf("unexpected pull policy for incude: %s, exclude: %s", test.cond.Include, test.cond.Exclude) + } + } +} + +func Test_toPorts(t *testing.T) { + from := &yaml.Container{ + Ports: []*yaml.Port{ + { + Port: 80, + Host: 8080, + Protocol: "TCP", + }, + { + Port: 80, + Host: 0, + Protocol: "", + }, + }, + } + a := toPorts(from) + b := []*engine.Port{ + { + Port: 80, + Host: 8080, + Protocol: "TCP", + }, + { + Port: 80, + Host: 0, + Protocol: "", + }, + } + if diff := cmp.Diff(a, b); diff != "" { + t.Errorf("Unexpected port conversion") + t.Log(diff) + } +} + +func Test_toResources(t *testing.T) { + from := &yaml.Container{ + Resources: nil, + } + if toResources(from) != nil { + t.Errorf("Expected nil resources") + } + + // test what happens when limits are defined + // but reservations are nil. + + from = &yaml.Container{ + Resources: &yaml.Resources{ + Limits: &yaml.ResourceObject{ + Memory: yaml.BytesSize(1000), + }, + }, + } + a := toResources(from) + b := &engine.Resources{ + Limits: &engine.ResourceObject{ + Memory: 1000, + }, + } + if diff := cmp.Diff(a, b); diff != "" { + t.Errorf("Unexpected resource conversion") + t.Log(diff) + } + + // test what happens when reservation and limits + // are both provided. + + from = &yaml.Container{ + Resources: &yaml.Resources{ + Limits: &yaml.ResourceObject{ + Memory: yaml.BytesSize(1000), + }, + Requests: &yaml.ResourceObject{ + Memory: yaml.BytesSize(2000), + }, + }, + } + a = toResources(from) + b = &engine.Resources{ + Limits: &engine.ResourceObject{ + Memory: 1000, + }, + Requests: &engine.ResourceObject{ + Memory: 2000, + }, + } + if diff := cmp.Diff(a, b); diff != "" { + t.Errorf("Unexpected resource conversion") + t.Log(diff) + } +} diff --git a/yaml/compiler/dind.go b/yaml/compiler/dind.go new file mode 100644 index 0000000..72f9c0a --- /dev/null +++ b/yaml/compiler/dind.go @@ -0,0 +1,37 @@ +package compiler + +import ( + "github.com/drone/drone-yaml/yaml" + "github.com/drone/drone-yaml/yaml/compiler/image" +) + +// DindFunc is a helper function that returns true +// if a container image (specifically a plugin) is +// a whitelisted dind container and should be executed +// in privileged mode. +func DindFunc(images []string) func(*yaml.Container) bool { + return func(container *yaml.Container) bool { + // privileged-by-default containers are only + // enabled for plugins steps that do not define + // commands, command, or entrypoint. + if len(container.Commands) > 0 { + return false + } + if len(container.Command) > 0 { + return false + } + if len(container.Entrypoint) > 0 { + return false + } + // if the container image matches any image + // in the whitelist, return true. + for _, img := range images { + a := img + b := container.Image + if image.Match(a, b) { + return true + } + } + return false + } +} diff --git a/yaml/compiler/dind_test.go b/yaml/compiler/dind_test.go new file mode 100644 index 0000000..707ace6 --- /dev/null +++ b/yaml/compiler/dind_test.go @@ -0,0 +1,63 @@ +package compiler + +import ( + "testing" + + "github.com/drone/drone-yaml/yaml" +) + +func TestDind(t *testing.T) { + tests := []struct { + container *yaml.Container + privileged bool + }{ + { + container: &yaml.Container{Image: "plugins/docker"}, + privileged: true, + }, + { + container: &yaml.Container{Image: "plugins/docker:latest"}, + privileged: true, + }, + { + container: &yaml.Container{Image: "plugins/docker:1"}, + privileged: true, + }, + // no match + { + container: &yaml.Container{Image: "golang"}, + privileged: false, + }, + // dind containers cannot set entrypoint or command + { + container: &yaml.Container{ + Image: "plugins/docker", + Command: []string{"docker run ..."}, + }, + privileged: false, + }, + { + container: &yaml.Container{ + Image: "plugins/docker", + Entrypoint: []string{"docker run ..."}, + }, + privileged: false, + }, + // dind containers cannot set commands + { + container: &yaml.Container{ + Image: "plugins/docker", + Commands: []string{"docker run ..."}, + }, + privileged: false, + }, + } + for i, test := range tests { + images := []string{"plugins/docker", "plugins/ecr"} + privileged := DindFunc(images)(test.container) + if privileged != test.privileged { + t.Errorf("Want privileged %v at index %d", test.privileged, i) + } + } + +} diff --git a/yaml/compiler/encode.go b/yaml/compiler/encode.go new file mode 100644 index 0000000..2e65cb3 --- /dev/null +++ b/yaml/compiler/encode.go @@ -0,0 +1,53 @@ +package compiler + +import ( + "encoding/base64" + "strconv" + "strings" + + json "github.com/ghodss/yaml" + "gopkg.in/yaml.v2" +) + +// helper funciton encodes an interface value as a string. +// this function assumes all types were unmarshaled by the +// yaml.v2 library. The yaml.v2 package only supports a +// subset of primative types. +func encode(v interface{}) string { + switch v := v.(type) { + case string: + return v + case bool: + return strconv.FormatBool(v) + case int: + return strconv.Itoa(v) + case float64: + return strconv.FormatFloat(v, 'g', -1, 64) + case []byte: + return base64.StdEncoding.EncodeToString(v) + case []interface{}: + return encodeSlice(v) + default: + return encodeMap(v) + } +} + +// helper function encodes a parameter in map format. +func encodeMap(v interface{}) string { + yml, _ := yaml.Marshal(v) + out, _ := json.YAMLToJSON(yml) + return string(out) +} + +// helper function encodes a parameter in slice format. +func encodeSlice(v interface{}) string { + out, _ := yaml.Marshal(v) + + in := []string{} + err := yaml.Unmarshal(out, &in) + if err == nil { + return strings.Join(in, ",") + } + out, _ = json.YAMLToJSON(out) + return string(out) +} diff --git a/yaml/compiler/encode_test.go b/yaml/compiler/encode_test.go new file mode 100644 index 0000000..29c3b65 --- /dev/null +++ b/yaml/compiler/encode_test.go @@ -0,0 +1,59 @@ +package compiler + +import "testing" + +func TestEncode(t *testing.T) { + testdatum := []struct { + data interface{} + text string + }{ + { + data: "foo", + text: "foo", + }, + { + data: true, + text: "true", + }, + { + data: 42, + text: "42", + }, + { + data: float64(42.424242), + text: "42.424242", + }, + { + data: []interface{}{"foo", "bar", "baz"}, + text: "foo,bar,baz", + }, + { + data: []interface{}{1, 1, 2, 3, 5, 8}, + text: "1,1,2,3,5,8", + }, + { + data: []byte("foo"), + text: "Zm9v", + }, + { + data: []interface{}{ + struct { + Name string `json:"name"` + }{ + Name: "john", + }, + }, + text: `[{"name":"john"}]`, + }, + { + data: map[interface{}]interface{}{"foo": "bar"}, + text: `{"foo":"bar"}`, + }, + } + + for _, testdata := range testdatum { + if got, want := encode(testdata.data), testdata.text; got != want { + t.Errorf("Want interface{} encoded to %q, got %q", want, got) + } + } +} diff --git a/yaml/compiler/image/image.go b/yaml/compiler/image/image.go new file mode 100644 index 0000000..79f13fa --- /dev/null +++ b/yaml/compiler/image/image.go @@ -0,0 +1,67 @@ +package image + +import "github.com/docker/distribution/reference" + +// Trim returns the short image name without tag. +func Trim(name string) string { + ref, err := reference.ParseAnyReference(name) + if err != nil { + return name + } + named, err := reference.ParseNamed(ref.String()) + if err != nil { + return name + } + named = reference.TrimNamed(named) + return reference.FamiliarName(named) +} + +// Expand returns the fully qualified image name. +func Expand(name string) string { + ref, err := reference.ParseAnyReference(name) + if err != nil { + return name + } + named, err := reference.ParseNamed(ref.String()) + if err != nil { + return name + } + named = reference.TagNameOnly(named) + return named.String() +} + +// Match returns true if the image name matches +// an image in the list. Note the image tag is not used +// in the matching logic. +func Match(from string, to ...string) bool { + from = Trim(from) + for _, match := range to { + if from == Trim(match) { + return true + } + } + return false +} + +// MatchTag returns true if the image name matches +// an image in the list, including the tag. +func MatchTag(a, b string) bool { + return Expand(a) == Expand(b) +} + +// MatchHostname returns true if the image hostname +// matches the specified hostname. +func MatchHostname(image, hostname string) bool { + ref, err := reference.ParseAnyReference(image) + if err != nil { + return false + } + named, err := reference.ParseNamed(ref.String()) + if err != nil { + return false + } + if hostname == "index.docker.io" { + hostname = "docker.io" + } + return reference.Domain(named) == hostname +} diff --git a/yaml/compiler/image/image_test.go b/yaml/compiler/image/image_test.go new file mode 100644 index 0000000..cddd396 --- /dev/null +++ b/yaml/compiler/image/image_test.go @@ -0,0 +1,295 @@ +package image + +import "testing" + +func Test_trimImage(t *testing.T) { + testdata := []struct { + from string + want string + }{ + { + from: "golang", + want: "golang", + }, + { + from: "golang:latest", + want: "golang", + }, + { + from: "golang:1.0.0", + want: "golang", + }, + { + from: "library/golang", + want: "golang", + }, + { + from: "library/golang:latest", + want: "golang", + }, + { + from: "library/golang:1.0.0", + want: "golang", + }, + { + from: "index.docker.io/library/golang:1.0.0", + want: "golang", + }, + { + from: "docker.io/library/golang:1.0.0", + want: "golang", + }, + { + from: "gcr.io/library/golang:1.0.0", + want: "gcr.io/library/golang", + }, + // error cases, return input unmodified + { + from: "foo/bar?baz:boo", + want: "foo/bar?baz:boo", + }, + } + for _, test := range testdata { + got, want := Trim(test.from), test.want + if got != want { + t.Errorf("Want image %q trimmed to %q, got %q", test.from, want, got) + } + } +} + +func Test_expandImage(t *testing.T) { + testdata := []struct { + from string + want string + }{ + { + from: "golang", + want: "docker.io/library/golang:latest", + }, + { + from: "golang:latest", + want: "docker.io/library/golang:latest", + }, + { + from: "golang:1.0.0", + want: "docker.io/library/golang:1.0.0", + }, + { + from: "library/golang", + want: "docker.io/library/golang:latest", + }, + { + from: "library/golang:latest", + want: "docker.io/library/golang:latest", + }, + { + from: "library/golang:1.0.0", + want: "docker.io/library/golang:1.0.0", + }, + { + from: "index.docker.io/library/golang:1.0.0", + want: "docker.io/library/golang:1.0.0", + }, + { + from: "gcr.io/golang", + want: "gcr.io/golang:latest", + }, + { + from: "gcr.io/golang:1.0.0", + want: "gcr.io/golang:1.0.0", + }, + // error cases, return input unmodified + { + from: "foo/bar?baz:boo", + want: "foo/bar?baz:boo", + }, + } + for _, test := range testdata { + got, want := Expand(test.from), test.want + if got != want { + t.Errorf("Want image %q expanded to %q, got %q", test.from, want, got) + } + } +} + +func Test_matchImage(t *testing.T) { + testdata := []struct { + from, to string + want bool + }{ + { + from: "golang", + to: "golang", + want: true, + }, + { + from: "golang:latest", + to: "golang", + want: true, + }, + { + from: "library/golang:latest", + to: "golang", + want: true, + }, + { + from: "index.docker.io/library/golang:1.0.0", + to: "golang", + want: true, + }, + { + from: "golang", + to: "golang:latest", + want: true, + }, + { + from: "library/golang:latest", + to: "library/golang", + want: true, + }, + { + from: "gcr.io/golang", + to: "gcr.io/golang", + want: true, + }, + { + from: "gcr.io/golang:1.0.0", + to: "gcr.io/golang", + want: true, + }, + { + from: "gcr.io/golang:latest", + to: "gcr.io/golang", + want: true, + }, + { + from: "gcr.io/golang", + to: "gcr.io/golang:latest", + want: true, + }, + { + from: "golang", + to: "library/golang", + want: true, + }, + { + from: "golang", + to: "gcr.io/project/golang", + want: false, + }, + { + from: "golang", + to: "gcr.io/library/golang", + want: false, + }, + { + from: "golang", + to: "gcr.io/golang", + want: false, + }, + } + for _, test := range testdata { + got, want := Match(test.from, test.to), test.want + if got != want { + t.Errorf("Want image %q matching %q is %v", test.from, test.to, want) + } + } +} + +func Test_matchHostname(t *testing.T) { + testdata := []struct { + image, hostname string + want bool + }{ + { + image: "golang", + hostname: "docker.io", + want: true, + }, + { + image: "golang:latest", + hostname: "docker.io", + want: true, + }, + { + image: "golang:latest", + hostname: "index.docker.io", + want: true, + }, + { + image: "library/golang:latest", + hostname: "docker.io", + want: true, + }, + { + image: "docker.io/library/golang:1.0.0", + hostname: "docker.io", + want: true, + }, + { + image: "gcr.io/golang", + hostname: "docker.io", + want: false, + }, + { + image: "gcr.io/golang:1.0.0", + hostname: "gcr.io", + want: true, + }, + { + image: "1.2.3.4:8000/golang:1.0.0", + hostname: "1.2.3.4:8000", + want: true, + }, + { + image: "*&^%", + hostname: "1.2.3.4:8000", + want: false, + }, + } + for _, test := range testdata { + got, want := MatchHostname(test.image, test.hostname), test.want + if got != want { + t.Errorf("Want image %q matching hostname %q is %v", test.image, test.hostname, want) + } + } +} + +func Test_matchTag(t *testing.T) { + testdata := []struct { + a, b string + want bool + }{ + { + a: "golang:1.0", + b: "golang:1.0", + want: true, + }, + { + a: "golang", + b: "golang:latest", + want: true, + }, + { + a: "docker.io/library/golang", + b: "golang:latest", + want: true, + }, + { + a: "golang", + b: "golang:1.0", + want: false, + }, + { + a: "golang:1.0", + b: "golang:2.0", + want: false, + }, + } + for _, test := range testdata { + got, want := MatchTag(test.a, test.b), test.want + if got != want { + t.Errorf("Want image %q matching image tag %q is %v", test.a, test.b, want) + } + } +} diff --git a/yaml/compiler/internal/rand/rand.go b/yaml/compiler/internal/rand/rand.go new file mode 100644 index 0000000..935d68a --- /dev/null +++ b/yaml/compiler/internal/rand/rand.go @@ -0,0 +1,36 @@ +package rand + +import ( + "crypto/rand" +) + +var chars = []byte("abcdefghijklmnopqrstuvwxyz0123456789") + +// random string length +const length = 32 + +// String returns a string value. +func String() string { + clen := len(chars) + maxrb := 255 - (256 % clen) + b := make([]byte, length) + r := make([]byte, length+(length/4)) // storage for random bytes. + i := 0 + for { + if _, err := rand.Read(r); err != nil { + panic("rand: error reading random bytes") + } + for _, rb := range r { + c := int(rb) + if c > maxrb { + // Skip this number to avoid modulo bias. + continue + } + b[i] = chars[c%clen] + i++ + if i == length { + return string(b) + } + } + } +} diff --git a/yaml/compiler/script_posix.go b/yaml/compiler/script_posix.go new file mode 100644 index 0000000..66baede --- /dev/null +++ b/yaml/compiler/script_posix.go @@ -0,0 +1,70 @@ +package compiler + +import ( + "bytes" + "fmt" + "strings" + + "github.com/drone/drone-runtime/engine" + "github.com/drone/drone-yaml/yaml" + "github.com/drone/drone-yaml/yaml/compiler/internal/rand" +) + +func setupScript(spec *engine.Spec, dst *engine.Step, src *yaml.Container) { + var buf bytes.Buffer + for _, command := range src.Commands { + escaped := fmt.Sprintf("%q", command) + escaped = strings.Replace(escaped, "$", `\$`, -1) + buf.WriteString(fmt.Sprintf( + traceScript, + escaped, + command, + )) + } + script := fmt.Sprintf( + buildScript, + buf.String(), + ) + spec.Files = append(spec.Files, &engine.File{ + Metadata: engine.Metadata{ + UID: rand.String(), + Namespace: spec.Metadata.Namespace, + Name: src.Name, + }, + Data: []byte(script), + }) + dst.Files = append(dst.Files, &engine.FileMount{ + Name: src.Name, + Path: "/usr/drone/bin/init", + Mode: 0777, + }) + + dst.Docker.Command = []string{"/bin/sh"} + dst.Docker.Args = []string{"/usr/drone/bin/init"} +} + +// buildScript is a helper script this is added to the build +// to prepare the environment and execute the build commands. +const buildScript = ` +if [ -n "$CI_NETRC_MACHINE" ]; then +cat < $HOME/.netrc +machine $CI_NETRC_MACHINE +login $CI_NETRC_USERNAME +password $CI_NETRC_PASSWORD +EOF +chmod 0600 $HOME/.netrc +fi +unset CI_NETRC_USERNAME +unset CI_NETRC_PASSWORD +unset DRONE_NETRC_USERNAME +unset DRONE_NETRC_PASSWORD +set -e +%s +` + +// traceScript is a helper script that is added to +// the build script to trace a command. +const traceScript = ` +echo + %s +%s +` diff --git a/yaml/compiler/script_posix_test.go b/yaml/compiler/script_posix_test.go new file mode 100644 index 0000000..a20d4fe --- /dev/null +++ b/yaml/compiler/script_posix_test.go @@ -0,0 +1 @@ +package compiler diff --git a/yaml/compiler/script_win.go b/yaml/compiler/script_win.go new file mode 100644 index 0000000..35bc1db --- /dev/null +++ b/yaml/compiler/script_win.go @@ -0,0 +1,58 @@ +package compiler + +import ( + "bytes" + "encoding/base64" + "fmt" + "strings" + + "github.com/drone/drone-runtime/engine" + "github.com/drone/drone-yaml/yaml" +) + +func setupScriptWin(spec *engine.Spec, dst *engine.Step, src *yaml.Container) { + var buf bytes.Buffer + for _, command := range src.Commands { + escaped := fmt.Sprintf("%q", command) + escaped = strings.Replace(escaped, "$", `\$`, -1) + buf.WriteString(fmt.Sprintf( + traceScriptWin, + escaped, + command, + )) + } + script := fmt.Sprintf( + buildScriptWin, + buf.String(), + ) + dst.Docker.Command = []string{"powershell", "-noprofile", "-noninteractive", "-command"} + dst.Docker.Args = []string{"[System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($Env:CI_SCRIPT)) | iex"} + dst.Envs["CI_SCRIPT"] = base64.StdEncoding.EncodeToString([]byte(script)) + dst.Envs["SHELL"] = "powershell.exe" +} + +// buildScriptWin is a helper script this is added to the build +// to prepare the environment and execute the build commands. +const buildScriptWin = ` +if ($Env:CI_NETRC_MACHINE) { +@" +machine $Env:CI_NETRC_MACHINE +login $Env:CI_NETRC_USERNAME +password $Env:CI_NETRC_PASSWORD +"@ > (Join-Path $Env:USERPROFILE '_netrc'); +} +[Environment]::SetEnvironmentVariable("CI_NETRC_USERNAME", $null); +[Environment]::SetEnvironmentVariable("CI_NETRC_PASSWORD", $null); +[Environment]::SetEnvironmentVariable("DRONE_NETRC_USERNAME", $null); +[Environment]::SetEnvironmentVariable("DRONE_NETRC_PASSWORD", $null); +[Environment]::SetEnvironmentVariable("CI_SCRIPT", $null); +$ErrorActionPreference = 'Stop'; +%s +` + +// traceScriptWin is a helper script that is added to +// the build script to trace a command. +const traceScriptWin = ` +Write-Output ('+ %s'); +& %s; if ($LASTEXITCODE -ne 0) {exit $LASTEXITCODE} +` diff --git a/yaml/compiler/script_win_test.go b/yaml/compiler/script_win_test.go new file mode 100644 index 0000000..a20d4fe --- /dev/null +++ b/yaml/compiler/script_win_test.go @@ -0,0 +1 @@ +package compiler diff --git a/yaml/compiler/skip.go b/yaml/compiler/skip.go new file mode 100644 index 0000000..7e512e5 --- /dev/null +++ b/yaml/compiler/skip.go @@ -0,0 +1,37 @@ +package compiler + +import "github.com/drone/drone-yaml/yaml" + +// SkipData provides build metadata use to determine if a +// pipeline step should be skipped. +type SkipData struct { + Branch string + Event string + Instance string + Ref string + Repo string + Target string +} + +// SkipFunc returns a function that can be used to skip +// individual pipeline steps based on build metadata. +func SkipFunc(data SkipData) func(*yaml.Container) bool { + return func(container *yaml.Container) bool { + switch { + case !container.When.Branch.Match(data.Branch): + return true + case !container.When.Event.Match(data.Event): + return true + case !container.When.Instance.Match(data.Instance): + return true + case !container.When.Ref.Match(data.Ref): + return true + case !container.When.Repo.Match(data.Repo): + return true + case !container.When.Target.Match(data.Target): + return true + default: + return false + } + } +} diff --git a/yaml/compiler/skip_test.go b/yaml/compiler/skip_test.go new file mode 100644 index 0000000..7147079 --- /dev/null +++ b/yaml/compiler/skip_test.go @@ -0,0 +1,105 @@ +package compiler + +import ( + "testing" + + "github.com/drone/drone-yaml/yaml" +) + +func TestSkipFunc(t *testing.T) { + tests := []struct { + data SkipData + when yaml.Conditions + want bool + }{ + // + // test branch conditions + // + { + data: SkipData{Branch: "master"}, + when: yaml.Conditions{Branch: yaml.Condition{Include: []string{"master"}}}, + want: false, + }, + { + data: SkipData{Branch: "master"}, + when: yaml.Conditions{Branch: yaml.Condition{Exclude: []string{"master"}}}, + want: true, + }, + // + // test event conditions + // + { + data: SkipData{Event: "push"}, + when: yaml.Conditions{Event: yaml.Condition{Include: []string{"push"}}}, + want: false, + }, + + { + data: SkipData{Event: "push"}, + when: yaml.Conditions{Event: yaml.Condition{Exclude: []string{"push"}}}, + want: true, + }, + // + // test instance conditions + // + { + data: SkipData{Instance: "drone.company.com"}, + when: yaml.Conditions{Instance: yaml.Condition{Include: []string{"drone.company.com"}}}, + want: false, + }, + + { + data: SkipData{Instance: "drone.company.com"}, + when: yaml.Conditions{Instance: yaml.Condition{Exclude: []string{"drone.company.com"}}}, + want: true, + }, + // + // test ref conditions + // + { + data: SkipData{Ref: "refs/heads/master"}, + when: yaml.Conditions{Ref: yaml.Condition{Include: []string{"refs/heads/*"}}}, + want: false, + }, + + { + data: SkipData{Ref: "refs/heads/master"}, + when: yaml.Conditions{Ref: yaml.Condition{Exclude: []string{"refs/heads/*"}}}, + want: true, + }, + // + // test repo conditions + // + { + data: SkipData{Repo: "octocat/hello-world"}, + when: yaml.Conditions{Repo: yaml.Condition{Include: []string{"octocat/hello-world"}}}, + want: false, + }, + + { + data: SkipData{Repo: "octocat/hello-world"}, + when: yaml.Conditions{Repo: yaml.Condition{Exclude: []string{"octocat/hello-world"}}}, + want: true, + }, + // + // test target conditions + // + { + data: SkipData{Target: "prod"}, + when: yaml.Conditions{Target: yaml.Condition{Include: []string{"prod"}}}, + want: false, + }, + { + data: SkipData{Target: "prod"}, + when: yaml.Conditions{Target: yaml.Condition{Exclude: []string{"prod"}}}, + want: true, + }, + } + for i, test := range tests { + container := &yaml.Container{When: test.when} + got := SkipFunc(test.data)(container) + if got != test.want { + t.Errorf("Want skip %v at index %d", test.want, i) + } + } +} diff --git a/yaml/compiler/step.go b/yaml/compiler/step.go new file mode 100644 index 0000000..2516e7a --- /dev/null +++ b/yaml/compiler/step.go @@ -0,0 +1,180 @@ +package compiler + +import ( + "strings" + + "github.com/drone/drone-runtime/engine" + "github.com/drone/drone-yaml/yaml" + "github.com/drone/drone-yaml/yaml/compiler/image" + "github.com/drone/drone-yaml/yaml/compiler/internal/rand" +) + +func createStep(spec *engine.Spec, src *yaml.Container) *engine.Step { + dst := &engine.Step{ + Metadata: engine.Metadata{ + UID: rand.String(), + Name: src.Name, + Namespace: spec.Metadata.Namespace, + Labels: map[string]string{ + "io.drone.step.name": src.Name, + }, + }, + Detach: src.Detach, + DependsOn: src.DependsOn, + Devices: nil, + Docker: &engine.DockerStep{ + Args: src.Command, + Command: src.Entrypoint, + DNS: src.DNS, + DNSSearch: src.DNSSearch, + ExtraHosts: src.ExtraHosts, + Image: image.Expand(src.Image), + Networks: nil, // set in compiler.go + Ports: toPorts(src), + Privileged: src.Privileged, + PullPolicy: toPullPolicy(src), + }, + Envs: map[string]string{}, + Files: nil, // set below + IgnoreErr: toIgnoreErr(src), + IgnoreStderr: false, + IgnoreStdout: false, + Resources: toResources(src), + RunPolicy: toRunPolicy(src), + Secrets: nil, // set below + Volumes: nil, // set below + WorkingDir: "", // set in compiler.go + } + + // if the user is running a service container with + // no custom commands, we should revert back to the + // user-defined working directory, which may be empty. + if dst.Detach && len(src.Commands) == 0 { + dst.WorkingDir = src.WorkingDir + } + + // appends the volumes to the container def. + for _, vol := range src.Volumes { + // the user should never be able to directly + // mount the docker socket. This should be + // restricted by the linter, but we place this + // check here just to be safe. + if vol.Name == "_docker_socket" { + continue + } + mount := &engine.VolumeMount{ + Name: vol.Name, + Path: vol.MountPath, + } + dst.Volumes = append(dst.Volumes, mount) + } + + // appends the environment variables to the + // container definition. + for key, value := range src.Environment { + if value.Secret != "" { + sec := &engine.SecretVar{ + Name: value.Secret, + Env: key, + } + dst.Secrets = append(dst.Secrets, sec) + } else { + dst.Envs[key] = value.Value + } + } + + // appends the settings variables to the + // container definition. + for key, value := range src.Settings { + // all settings are passed to the plugin env + // variables, prefixed with PLUGIN_ + key = "PLUGIN_" + strings.ToUpper(key) + + // if the setting parameter is sources from the + // secret we create a secret enviornment variable. + if value.Secret != "" { + sec := &engine.SecretVar{ + Name: value.Secret, + Env: key, + } + dst.Secrets = append(dst.Secrets, sec) + } else { + // else if the setting parameter is opaque + // we inject as a string-encoded environment + // variable. + dst.Envs[key] = encode(value.Value) + } + } + + // if the step specifies shell commands we generate a + // script. The script is copied to the container at + // runtime (or mounted as a config map) and then executed + // as the entrypoint. + if len(src.Commands) > 0 { + switch spec.Platform.OS { + case "windows": + setupScriptWin(spec, dst, src) + default: + setupScript(spec, dst, src) + } + } + + return dst +} + +func createBuildStep(spec *engine.Spec, src *yaml.Container) *engine.Step { + dst := &engine.Step{ + Metadata: engine.Metadata{ + UID: rand.String(), + Name: src.Name, + Namespace: spec.Metadata.Namespace, + Labels: map[string]string{ + "io.drone.step.name": src.Name, + }, + }, + Detach: src.Detach, + DependsOn: src.DependsOn, + Devices: nil, + Docker: &engine.DockerStep{ + Args: []string{"--build"}, + DNS: src.DNS, + Image: "drone/docker", + PullPolicy: engine.PullIfNotExists, + }, + Envs: map[string]string{}, + IgnoreErr: toIgnoreErr(src), + IgnoreStderr: false, + IgnoreStdout: false, + Resources: toResources(src), + RunPolicy: toRunPolicy(src), + } + + // if v := src.Build.Args; len(v) > 0 { + // dst.Envs["DOCKER_BUILD_ARGS"] = strings.Join(v, ",") + // } + if v := src.Build.CacheFrom; len(v) > 0 { + dst.Envs["DOCKER_BUILD_CACHE_FROM"] = strings.Join(v, ",") + } + // if len(src.Build.Labels) > 0 { + // dst.Envs["DOCKER_BUILD_LABELS"] = strings.Join(v, ",") + // } + if v := src.Build.Dockerfile; v != "" { + dst.Envs["DOCKER_BUILD_DOCKERFILE"] = v + + } + if v := src.Build.Context; v != "" { + dst.Envs["DOCKER_BUILD_CONTEXT"] = v + } + if v := src.Build.Image; v != "" { + alias := image.Trim(v) + ":" + dst.Metadata.UID + dst.Envs["DOCKER_BUILD_IMAGE"] = image.Expand(v) + dst.Envs["DOCKER_BUILD_IMAGE_ALIAS"] = image.Expand(alias) + } + + dst.Volumes = append(dst.Volumes, &engine.VolumeMount{ + Name: "_docker_socket", + Path: "/var/run/docker.sock", + }) + + return dst +} diff --git a/yaml/compiler/transform/auths.go b/yaml/compiler/transform/auths.go new file mode 100644 index 0000000..7153942 --- /dev/null +++ b/yaml/compiler/transform/auths.go @@ -0,0 +1,29 @@ +package transform + +import "github.com/drone/drone-runtime/engine" + +// WithAuths is a transform function that adds a set +// of global registry credentials to the container. +func WithAuths(auths []*engine.DockerAuth) func(*engine.Spec) { + return func(spec *engine.Spec) { + for _, auth := range auths { + spec.Docker.Auths = append(spec.Docker.Auths, auth) + } + } +} + +// AuthsFunc is a callback function used to request +// registry credentials to pull private images. +type AuthsFunc func() []*engine.DockerAuth + +// WithAuthsFunc is a transform function that provides +// the sepcification with registry authentication +// credentials via a callback function. +func WithAuthsFunc(f AuthsFunc) func(*engine.Spec) { + return func(spec *engine.Spec) { + auths := f() + if len(auths) != 0 { + spec.Docker.Auths = append(spec.Docker.Auths, auths...) + } + } +} diff --git a/yaml/compiler/transform/auths_test.go b/yaml/compiler/transform/auths_test.go new file mode 100644 index 0000000..9047d6b --- /dev/null +++ b/yaml/compiler/transform/auths_test.go @@ -0,0 +1,49 @@ +package transform + +import ( + "testing" + + "github.com/drone/drone-runtime/engine" + "github.com/google/go-cmp/cmp" +) + +func TestWithAuths(t *testing.T) { + spec := &engine.Spec{ + Steps: []*engine.Step{}, + Docker: &engine.DockerConfig{}, + } + auths := []*engine.DockerAuth{ + { + Address: "docker.io", + Username: "octocat", + Password: "correct-horse-battery-staple", + }, + } + WithAuths(auths)(spec) + if diff := cmp.Diff(auths, spec.Docker.Auths); diff != "" { + t.Errorf("Unexpected auths transform") + t.Log(diff) + } +} + +func TestWithAuthsFunc(t *testing.T) { + spec := &engine.Spec{ + Steps: []*engine.Step{}, + Docker: &engine.DockerConfig{}, + } + auths := []*engine.DockerAuth{ + { + Address: "docker.io", + Username: "octocat", + Password: "correct-horse-battery-staple", + }, + } + fn := func() []*engine.DockerAuth { + return auths + } + WithAuthsFunc(fn)(spec) + if diff := cmp.Diff(auths, spec.Docker.Auths); diff != "" { + t.Errorf("Unexpected auths transform") + t.Log(diff) + } +} diff --git a/yaml/compiler/transform/combine.go b/yaml/compiler/transform/combine.go new file mode 100644 index 0000000..e6257c2 --- /dev/null +++ b/yaml/compiler/transform/combine.go @@ -0,0 +1,13 @@ +package transform + +import "github.com/drone/drone-runtime/engine" + +// Combine is a transform function that combines +// one or many transform functions. +func Combine(fns ...func(*engine.Spec)) func(*engine.Spec) { + return func(spec *engine.Spec) { + for _, fn := range fns { + fn(spec) + } + } +} diff --git a/yaml/compiler/transform/combine_test.go b/yaml/compiler/transform/combine_test.go new file mode 100644 index 0000000..4ca4f4f --- /dev/null +++ b/yaml/compiler/transform/combine_test.go @@ -0,0 +1,33 @@ +package transform + +import ( + "testing" + + "github.com/drone/drone-runtime/engine" + "github.com/google/go-cmp/cmp" +) + +func TestCombine(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{ + UID: "1", + Name: "build", + }, + Envs: map[string]string{}, + } + spec := &engine.Spec{ + Steps: []*engine.Step{step}, + } + Combine( + WithEnviron(map[string]string{"GOOS": "linux"}), + WithEnviron(map[string]string{"GOARCH": "amd64"}), + )(spec) + want := map[string]string{ + "GOOS": "linux", + "GOARCH": "amd64", + } + if diff := cmp.Diff(want, step.Envs); diff != "" { + t.Errorf("Unexpected transform") + t.Log(diff) + } +} diff --git a/yaml/compiler/transform/env.go b/yaml/compiler/transform/env.go new file mode 100644 index 0000000..3dfe856 --- /dev/null +++ b/yaml/compiler/transform/env.go @@ -0,0 +1,15 @@ +package transform + +import "github.com/drone/drone-runtime/engine" + +// WithEnviron is a transform function that adds a set +// of environment variables to each container. +func WithEnviron(envs map[string]string) func(*engine.Spec) { + return func(spec *engine.Spec) { + for key, value := range envs { + for _, step := range spec.Steps { + step.Envs[key] = value + } + } + } +} diff --git a/yaml/compiler/transform/env_test.go b/yaml/compiler/transform/env_test.go new file mode 100644 index 0000000..3797137 --- /dev/null +++ b/yaml/compiler/transform/env_test.go @@ -0,0 +1,30 @@ +package transform + +import ( + "testing" + + "github.com/drone/drone-runtime/engine" + "github.com/google/go-cmp/cmp" +) + +func TestWithEnviron(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{ + UID: "1", + Name: "build", + }, + Envs: map[string]string{}, + } + spec := &engine.Spec{ + Steps: []*engine.Step{step}, + } + envs := map[string]string{ + "GOOS": "linux", + "GOARCH": "amd64", + } + WithEnviron(envs)(spec) + if diff := cmp.Diff(envs, step.Envs); diff != "" { + t.Errorf("Unexpected transform") + t.Log(diff) + } +} diff --git a/yaml/compiler/transform/filter.go b/yaml/compiler/transform/filter.go new file mode 100644 index 0000000..e18da81 --- /dev/null +++ b/yaml/compiler/transform/filter.go @@ -0,0 +1,72 @@ +package transform + +import "github.com/drone/drone-runtime/engine" + +// Include is a transform function that limits the +// pipeline execution to a whitelist of named steps. +func Include(names []string) func(*engine.Spec) { + set := map[string]struct{}{} + for _, name := range names { + set[name] = struct{}{} + } + return func(spec *engine.Spec) { + if len(names) == 0 { + return + } + for _, step := range spec.Steps { + if step.Metadata.Name == "clone" { + continue + } + _, ok := set[step.Metadata.Name] + if !ok { + // if the step is not included in the + // whitelist the run policy is set to never. + step.RunPolicy = engine.RunNever + } + } + } +} + +// Exclude is a transform function that limits the +// pipeline execution to a whitelist of named steps. +func Exclude(names []string) func(*engine.Spec) { + set := map[string]struct{}{} + for _, name := range names { + set[name] = struct{}{} + } + return func(spec *engine.Spec) { + if len(names) == 0 { + return + } + for _, step := range spec.Steps { + if step.Metadata.Name == "clone" { + continue + } + _, ok := set[step.Metadata.Name] + if ok { + // if the step is included in the blacklist + // the run policy is set to never. + step.RunPolicy = engine.RunNever + } + } + } +} + +// ResumeAt is a transform function that modifies the +// exuction to resume at a named step. +func ResumeAt(name string) func(*engine.Spec) { + return func(spec *engine.Spec) { + if name == "" { + return + } + for _, step := range spec.Steps { + if step.Metadata.Name == name { + return + } + if step.Metadata.Name == "clone" { + continue + } + step.RunPolicy = engine.RunNever + } + } +} diff --git a/yaml/compiler/transform/filter_test.go b/yaml/compiler/transform/filter_test.go new file mode 100644 index 0000000..c905e80 --- /dev/null +++ b/yaml/compiler/transform/filter_test.go @@ -0,0 +1,139 @@ +package transform + +import ( + "testing" + + "github.com/drone/drone-runtime/engine" +) + +func TestInclude(t *testing.T) { + step1 := &engine.Step{ + Metadata: engine.Metadata{Name: "clone"}, + RunPolicy: engine.RunOnSuccess, + } + step2 := &engine.Step{ + Metadata: engine.Metadata{Name: "build"}, + RunPolicy: engine.RunOnSuccess, + } + step3 := &engine.Step{ + Metadata: engine.Metadata{Name: "test"}, + RunPolicy: engine.RunOnSuccess, + } + spec := &engine.Spec{ + Metadata: engine.Metadata{}, + Steps: []*engine.Step{step1, step2, step3}, + } + Include([]string{"test"})(spec) + if got, want := step1.RunPolicy, engine.RunOnSuccess; got != want { + t.Errorf("Want run policy %s got %s", want, got) + } + if got, want := step2.RunPolicy, engine.RunNever; got != want { + t.Errorf("Want run policy %s got %s", want, got) + } + if got, want := step3.RunPolicy, engine.RunOnSuccess; got != want { + t.Errorf("Want run policy %s got %s", want, got) + } +} + +func TestInclude_Empty(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{}, + RunPolicy: engine.RunOnSuccess, + } + spec := &engine.Spec{ + Metadata: engine.Metadata{}, + Steps: []*engine.Step{step}, + } + Include(nil)(spec) + if got, want := step.RunPolicy, engine.RunOnSuccess; got != want { + t.Errorf("Want run policy %s got %s", want, got) + } +} + +func TestExclude(t *testing.T) { + step1 := &engine.Step{ + Metadata: engine.Metadata{Name: "clone"}, + RunPolicy: engine.RunOnSuccess, + } + step2 := &engine.Step{ + Metadata: engine.Metadata{Name: "build"}, + RunPolicy: engine.RunOnSuccess, + } + step3 := &engine.Step{ + Metadata: engine.Metadata{Name: "test"}, + RunPolicy: engine.RunOnSuccess, + } + spec := &engine.Spec{ + Metadata: engine.Metadata{}, + Steps: []*engine.Step{step1, step2, step3}, + } + Exclude([]string{"test"})(spec) + if got, want := step1.RunPolicy, engine.RunOnSuccess; got != want { + t.Errorf("Want run policy %s got %s", want, got) + } + if got, want := step2.RunPolicy, engine.RunOnSuccess; got != want { + t.Errorf("Want run policy %s got %s", want, got) + } + if got, want := step3.RunPolicy, engine.RunNever; got != want { + t.Errorf("Want run policy %s got %s", want, got) + } +} + +func TestExclude_Empty(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{}, + RunPolicy: engine.RunOnSuccess, + } + spec := &engine.Spec{ + Metadata: engine.Metadata{}, + Steps: []*engine.Step{step}, + } + Exclude(nil)(spec) + if got, want := step.RunPolicy, engine.RunOnSuccess; got != want { + t.Errorf("Want run policy %s got %s", want, got) + } +} + +func TestResumeAt(t *testing.T) { + step1 := &engine.Step{ + Metadata: engine.Metadata{Name: "clone"}, + RunPolicy: engine.RunOnSuccess, + } + step2 := &engine.Step{ + Metadata: engine.Metadata{Name: "build"}, + RunPolicy: engine.RunOnSuccess, + } + step3 := &engine.Step{ + Metadata: engine.Metadata{Name: "test"}, + RunPolicy: engine.RunOnSuccess, + } + spec := &engine.Spec{ + Metadata: engine.Metadata{}, + Steps: []*engine.Step{step1, step2, step3}, + } + ResumeAt("test")(spec) + if got, want := step1.RunPolicy, engine.RunOnSuccess; got != want { + t.Errorf("Want run policy %s got %s", want, got) + } + if got, want := step2.RunPolicy, engine.RunNever; got != want { + t.Errorf("Want run policy %s got %s", want, got) + } + if got, want := step3.RunPolicy, engine.RunOnSuccess; got != want { + t.Errorf("Want run policy %s got %s", want, got) + } +} + +func TestResume_Empty(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{}, + RunPolicy: engine.RunOnSuccess, + } + spec := &engine.Spec{ + Metadata: engine.Metadata{}, + Steps: []*engine.Step{step}, + } + ResumeAt("")(spec) + if got, want := step.RunPolicy, engine.RunOnSuccess; got != want { + t.Errorf("Want run policy %s got %s", want, got) + } +} diff --git a/yaml/compiler/transform/labels.go b/yaml/compiler/transform/labels.go new file mode 100644 index 0000000..45b80a5 --- /dev/null +++ b/yaml/compiler/transform/labels.go @@ -0,0 +1,23 @@ +package transform + +import "github.com/drone/drone-runtime/engine" + +// WithLables is a transform function that adds a set +// of labels to each resource. +func WithLables(labels map[string]string) func(*engine.Spec) { + return func(spec *engine.Spec) { + for k, v := range labels { + spec.Metadata.Labels[k] = v + } + for _, resource := range spec.Docker.Volumes { + for k, v := range labels { + resource.Metadata.Labels[k] = v + } + } + for _, resource := range spec.Steps { + for k, v := range labels { + resource.Metadata.Labels[k] = v + } + } + } +} diff --git a/yaml/compiler/transform/labels_test.go b/yaml/compiler/transform/labels_test.go new file mode 100644 index 0000000..852d9a2 --- /dev/null +++ b/yaml/compiler/transform/labels_test.go @@ -0,0 +1,48 @@ +package transform + +import ( + "testing" + + "github.com/drone/drone-runtime/engine" + "github.com/google/go-cmp/cmp" +) + +func TestWithLabels(t *testing.T) { + volume := &engine.Volume{ + Metadata: engine.Metadata{ + Labels: map[string]string{}, + }, + } + step := &engine.Step{ + Metadata: engine.Metadata{ + Labels: map[string]string{}, + }, + Envs: map[string]string{}, + } + spec := &engine.Spec{ + Metadata: engine.Metadata{ + Labels: map[string]string{}, + }, + Steps: []*engine.Step{step}, + Docker: &engine.DockerConfig{ + Volumes: []*engine.Volume{volume}, + }, + } + labels := map[string]string{ + "io.drone.build.number": "1", + "io.drone.build.event": "push", + } + WithLables(labels)(spec) + if diff := cmp.Diff(labels, spec.Metadata.Labels); diff != "" { + t.Errorf("Unexpected spec labels") + t.Log(diff) + } + if diff := cmp.Diff(labels, step.Metadata.Labels); diff != "" { + t.Errorf("Unexpected step labels") + t.Log(diff) + } + if diff := cmp.Diff(labels, volume.Metadata.Labels); diff != "" { + t.Errorf("Unexpected volume labels") + t.Log(diff) + } +} diff --git a/yaml/compiler/transform/limits.go b/yaml/compiler/transform/limits.go new file mode 100644 index 0000000..098c194 --- /dev/null +++ b/yaml/compiler/transform/limits.go @@ -0,0 +1,29 @@ +package transform + +import "github.com/drone/drone-runtime/engine" +// WithLimits is a transform function that applies +// resource limits to the container processes. +func WithLimits(memlimit int64, cpulimit int64) func(*engine.Spec) { + return func(spec *engine.Spec) { + // if no limits are defined exit immediately. + if memlimit == 0 && cpulimit == 0 { + return + } + // otherwise apply the resource limits to every + // step in the runtime spec. + for _, step := range spec.Steps { + if step.Resources == nil { + step.Resources = &engine.Resources{} + } + if step.Resources.Limits == nil { + step.Resources.Limits = &engine.ResourceObject{} + } + if memlimit != 0 { + step.Resources.Limits.Memory = memlimit + } + if cpulimit != 0 { + step.Resources.Limits.CPU = cpulimit * 1000 + } + } + } +} diff --git a/yaml/compiler/transform/limits_test.go b/yaml/compiler/transform/limits_test.go new file mode 100644 index 0000000..f9394f9 --- /dev/null +++ b/yaml/compiler/transform/limits_test.go @@ -0,0 +1,84 @@ +package transform + +import ( + "testing" + + "github.com/drone/drone-runtime/engine" +) + +func TestWithLimits(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{ + UID: "1", + Name: "build", + }, + Docker: &engine.DockerStep{}, + } + spec := &engine.Spec{ + Steps: []*engine.Step{step}, + } + WithLimits(1, 2)(spec) + if got, want := step.Resources.Limits.Memory, int64(1); got != want { + t.Errorf("Want memory limit %v, got %v", want, got) + } + if got, want := step.Resources.Limits.CPU, int64(2000); got != want { + t.Errorf("Want cpu limit %v, got %v", want, got) + } +} + +func TestWithMemory(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{ + UID: "1", + Name: "build", + }, + Docker: &engine.DockerStep{}, + } + spec := &engine.Spec{ + Steps: []*engine.Step{step}, + } + WithLimits(1, 0)(spec) + if got, want := step.Resources.Limits.Memory, int64(1); got != want { + t.Errorf("Want memory limit %v, got %v", want, got) + } + if got, want := step.Resources.Limits.CPU, int64(0); got != want { + t.Errorf("Want cpu limit %v, got %v", want, got) + } +} + +func TestWithCPU(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{ + UID: "1", + Name: "build", + }, + Docker: &engine.DockerStep{}, + } + spec := &engine.Spec{ + Steps: []*engine.Step{step}, + } + WithLimits(0, 3)(spec) + if got, want := step.Resources.Limits.Memory, int64(0); got != want { + t.Errorf("Want memory limit %v, got %v", want, got) + } + if got, want := step.Resources.Limits.CPU, int64(3000); got != want { + t.Errorf("Want cpu limit %v, got %v", want, got) + } +} + +func TestWithNoLimits(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{ + UID: "1", + Name: "build", + }, + Docker: &engine.DockerStep{}, + } + spec := &engine.Spec{ + Steps: []*engine.Step{step}, + } + WithLimits(0, 0)(spec) + if step.Resources != nil { + t.Errorf("Expect no limits applied") + } +} diff --git a/yaml/compiler/transform/netrc.go b/yaml/compiler/transform/netrc.go new file mode 100644 index 0000000..4e4b916 --- /dev/null +++ b/yaml/compiler/transform/netrc.go @@ -0,0 +1,73 @@ +package transform + +import ( + "fmt" + + "github.com/drone/drone-runtime/engine" + "github.com/drone/drone-yaml/yaml/compiler/internal/rand" +) + +const ( + netrcName = ".netrc" + netrcPath = "/var/run/drone/.netrc" + netrcMode = 0777 +) + +const disableNetrcMount = true + +// WithNetrc is a helper function that creates a netrc file +// and mounts the file to all container steps. +func WithNetrc(machine, username, password string) func(*engine.Spec) { + return func(spec *engine.Spec) { + if username == "" || password == "" { + return + } + + // TODO(bradrydzewski) temporarily disable mounting + // the netrc file due to issues with kubernetes + // compatibility. + if disableNetrcMount == false { + // Currently file mounts don't seem to work in Windows so environment + // variables are used instead + // FIXME: https://github.com/drone/drone-yaml/issues/20 + if spec.Platform.OS != "windows" { + netrc := generateNetrc(machine, username, password) + spec.Files = append(spec.Files, &engine.File{ + Metadata: engine.Metadata{ + UID: rand.String(), + Name: netrcName, + Namespace: spec.Metadata.Namespace, + }, + Data: []byte(netrc), + }) + for _, step := range spec.Steps { + step.Files = append(step.Files, &engine.FileMount{ + Name: netrcName, + Path: netrcPath, + Mode: netrcMode, + }) + } + } + } + + // TODO(bradrydzewski) these should only be injected + // if the file is not mounted, if OS == Windows. + for _, step := range spec.Steps { + if step.Envs == nil { + step.Envs = map[string]string{} + } + step.Envs["CI_NETRC_MACHINE"] = machine + step.Envs["CI_NETRC_USERNAME"] = username + step.Envs["CI_NETRC_PASSWORD"] = password + + step.Envs["DRONE_NETRC_MACHINE"] = machine + step.Envs["DRONE_NETRC_USERNAME"] = username + step.Envs["DRONE_NETRC_PASSWORD"] = password + } + } +} + +func generateNetrc(machine, username, password string) string { + return fmt.Sprintf("machine %s login %s password %s", + machine, username, password) +} diff --git a/yaml/compiler/transform/netrc_test.go b/yaml/compiler/transform/netrc_test.go new file mode 100644 index 0000000..f9d4b53 --- /dev/null +++ b/yaml/compiler/transform/netrc_test.go @@ -0,0 +1,78 @@ +package transform + +import ( + "testing" + + "github.com/drone/drone-runtime/engine" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +var ignoreMetadata = cmpopts.IgnoreFields( + engine.Metadata{}, "UID") + +func TestWithNetrc(t *testing.T) { + if true { + t.Skipf("mounting the netrc is temporarily disabled") + return + } + step := &engine.Step{ + Metadata: engine.Metadata{ + UID: "1", + Name: "build", + }, + } + spec := &engine.Spec{ + Metadata: engine.Metadata{ + UID: "acdj0yjqv7uh5hidveg0ggr42x8oj67b", + Namespace: "pivqfthg1c9hy83ylht1sxx4nygjc7tk", + }, + Steps: []*engine.Step{step}, + } + WithNetrc("@machine", "@username", "@password")(spec) + if len(step.Files) == 0 { + t.Errorf("File mount not added to step") + return + } + if len(spec.Files) == 0 { + t.Errorf("File not declared in spec") + return + } + file := &engine.File{ + Metadata: engine.Metadata{ + Name: ".netrc", + Namespace: "pivqfthg1c9hy83ylht1sxx4nygjc7tk", + }, + Data: []byte("machine @machine login @username password @password"), + } + if diff := cmp.Diff(file, spec.Files[0], ignoreMetadata); diff != "" { + t.Errorf("Unexpected file declaration") + t.Log(diff) + } + + fileMount := &engine.FileMount{Name: ".netrc", Path: "/root/.netrc", Mode: 0600} + if diff := cmp.Diff(fileMount, step.Files[0], ignoreMetadata); diff != "" { + t.Errorf("Unexpected file mount") + t.Log(diff) + } +} + +func TestWithEmptyNetrc(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{ + UID: "1", + Name: "build", + }, + } + spec := &engine.Spec{ + Steps: []*engine.Step{step}, + } + WithNetrc("@machine", "", "")(spec) + if len(spec.Files) != 0 { + t.Errorf("Unexpected file declaration") + } + if len(step.Files) != 0 { + t.Errorf("Unexpected file mount") + } +} diff --git a/yaml/compiler/transform/network.go b/yaml/compiler/transform/network.go new file mode 100644 index 0000000..7bbe7a7 --- /dev/null +++ b/yaml/compiler/transform/network.go @@ -0,0 +1,14 @@ +package transform + +import "github.com/drone/drone-runtime/engine" + +// WithNetworks is a transform function that attaches a +// list of user-defined Docker networks to each step. +func WithNetworks(networks []string) func(*engine.Spec) { + return func(spec *engine.Spec) { + for _, step := range spec.Steps { + step.Docker.Networks = append( + step.Docker.Networks, networks...) + } + } +} diff --git a/yaml/compiler/transform/network_test.go b/yaml/compiler/transform/network_test.go new file mode 100644 index 0000000..9cfe173 --- /dev/null +++ b/yaml/compiler/transform/network_test.go @@ -0,0 +1,29 @@ +package transform + +import ( + "testing" + + "github.com/drone/drone-runtime/engine" + "github.com/google/go-cmp/cmp" +) + +func TestWithNetwork(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{ + UID: "1", + Name: "build", + }, + Docker: &engine.DockerStep{ + Networks: nil, + }, + } + spec := &engine.Spec{ + Steps: []*engine.Step{step}, + } + nets := []string{"a", "b"} + WithNetworks(nets)(spec) + if diff := cmp.Diff(nets, step.Docker.Networks); diff != "" { + t.Errorf("Unexpected transform") + t.Log(diff) + } +} diff --git a/yaml/compiler/transform/proxy.go b/yaml/compiler/transform/proxy.go new file mode 100644 index 0000000..f997592 --- /dev/null +++ b/yaml/compiler/transform/proxy.go @@ -0,0 +1,39 @@ +package transform + +import ( + "os" + "strings" + + "github.com/drone/drone-runtime/engine" +) + +// WithProxy is a transform function that adds the +// http_proxy environment variables to every container. +func WithProxy() func(*engine.Spec) { + environ := map[string]string{} + if value := getenv("no_proxy"); value != "" { + environ["no_proxy"] = value + environ["NO_PROXY"] = value + } + if value := getenv("http_proxy"); value != "" { + environ["http_proxy"] = value + environ["HTTP_PROXY"] = value + } + if value := getenv("https_proxy"); value != "" { + environ["https_proxy"] = value + environ["HTTPS_PROXY"] = value + } + return WithEnviron(environ) +} + +func getenv(name string) (value string) { + name = strings.ToUpper(name) + if value := os.Getenv(name); value != "" { + return value + } + name = strings.ToLower(name) + if value := os.Getenv(name); value != "" { + return value + } + return +} diff --git a/yaml/compiler/transform/proxy_test.go b/yaml/compiler/transform/proxy_test.go new file mode 100644 index 0000000..5f381af --- /dev/null +++ b/yaml/compiler/transform/proxy_test.go @@ -0,0 +1,52 @@ +package transform + +import ( + "os" + "testing" + + "github.com/drone/drone-runtime/engine" +) + +func TestWithProxy(t *testing.T) { + var ( + noProxy = getenv("no_proxy") + httpProxy = getenv("https_proxy") + httpsProxy = getenv("https_proxy") + ) + defer func() { + os.Setenv("no_proxy", noProxy) + os.Setenv("NO_PROXY", noProxy) + os.Setenv("http_proxy", httpProxy) + os.Setenv("HTTP_PROXY", httpProxy) + os.Setenv("HTTPS_PROXY", httpsProxy) + os.Setenv("https_proxy", httpsProxy) + }() + + testdata := map[string]string{ + "NO_PROXY": "http://dummy.no.proxy", + "http_proxy": "http://dummy.http.proxy", + "https_proxy": "http://dummy.https.proxy", + } + + for k, v := range testdata { + os.Setenv(k, v) + } + + step := &engine.Step{ + Metadata: engine.Metadata{ + UID: "1", + Name: "build", + }, + Envs: map[string]string{}, + } + spec := &engine.Spec{ + Steps: []*engine.Step{step}, + } + WithProxy()(spec) + for k, v := range testdata { + step := spec.Steps[0] + if step.Envs[k] != v { + t.Errorf("Expect proxy varaible %s=%q, got %q", k, v, step.Envs[k]) + } + } +} diff --git a/yaml/compiler/transform/secret.go b/yaml/compiler/transform/secret.go new file mode 100644 index 0000000..9e21606 --- /dev/null +++ b/yaml/compiler/transform/secret.go @@ -0,0 +1,62 @@ +package transform + +import ( + "github.com/drone/drone-runtime/engine" + "github.com/drone/drone-yaml/yaml/compiler/internal/rand" +) + +// WithSecrets is a transform function that adds a set +// of global secrets to the container. +func WithSecrets(secrets map[string]string) func(*engine.Spec) { + return func(spec *engine.Spec) { + for key, value := range secrets { + spec.Secrets = append(spec.Secrets, + &engine.Secret{ + Metadata: engine.Metadata{ + UID: rand.String(), + Name: key, + Namespace: spec.Metadata.Namespace, + }, + Data: value, + }, + ) + } + } +} + +// SecretFunc is a callback function used to request +// named secret, required by a pipeline step. +type SecretFunc func(string) *engine.Secret + +// WithSecretFunc is a transform function that resolves +// all named secrets through a callback function, and +// adds the secrets to the specification. +func WithSecretFunc(f SecretFunc) func(*engine.Spec) { + return func(spec *engine.Spec) { + // first we get a unique list of all secrets + // used by the specification. + set := map[string]struct{}{} + for _, step := range spec.Steps { + // if we know the step is not going to run, + // we can ignore any secrets that it requires. + if step.RunPolicy == engine.RunNever { + continue + } + for _, v := range step.Secrets { + set[v.Name] = struct{}{} + } + } + + // next we use the callback function to + // get the value for each secret, and append + // to the specification. + for name := range set { + secret := f(name) + if secret != nil { + secret.Metadata.UID = rand.String() + secret.Metadata.Namespace = spec.Metadata.Namespace + spec.Secrets = append(spec.Secrets, secret) + } + } + } +} diff --git a/yaml/compiler/transform/secret_test.go b/yaml/compiler/transform/secret_test.go new file mode 100644 index 0000000..a9d67b3 --- /dev/null +++ b/yaml/compiler/transform/secret_test.go @@ -0,0 +1,107 @@ +package transform + +import ( + "testing" + + "github.com/drone/drone-runtime/engine" + "github.com/google/go-cmp/cmp" +) + +func TestWithSecret(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{ + UID: "1", + Name: "build", + }, + Envs: map[string]string{}, + } + spec := &engine.Spec{ + Metadata: engine.Metadata{ + UID: "acdj0yjqv7uh5hidveg0ggr42x8oj67b", + Namespace: "pivqfthg1c9hy83ylht1sxx4nygjc7tk", + }, + Steps: []*engine.Step{step}, + } + secrets := map[string]string{ + "password": "correct-horse-battery-staple", + } + WithSecrets(secrets)(spec) + + want := []*engine.Secret{ + { + Metadata: engine.Metadata{ + Name: "password", + Namespace: "pivqfthg1c9hy83ylht1sxx4nygjc7tk", + }, + Data: "correct-horse-battery-staple", + }, + } + if diff := cmp.Diff(want, spec.Secrets, ignoreMetadata); diff != "" { + t.Errorf("Unexpected secret transform") + t.Log(diff) + } +} + +func TestWithSecretFunc(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{ + UID: "1", + Name: "build", + }, + Envs: map[string]string{}, + Secrets: []*engine.SecretVar{ + { + Name: "password", + Env: "PASSWORD", + }, + }, + } + spec := &engine.Spec{ + Metadata: engine.Metadata{ + UID: "acdj0yjqv7uh5hidveg0ggr42x8oj67b", + Namespace: "pivqfthg1c9hy83ylht1sxx4nygjc7tk", + }, + Steps: []*engine.Step{ + step, + // this is a step that requests a secret + // but should be skipped. + { + RunPolicy: engine.RunNever, + Secrets: []*engine.SecretVar{ + { + Name: "github_token", + Env: "GITHUB_TOKEN", + }, + }, + }, + }, + } + + fn := func(name string) *engine.Secret { + if name == "github_token" { + t.Errorf("Requested secret for skipped step") + return nil + } + return &engine.Secret{ + Metadata: engine.Metadata{ + Name: "password", + }, + Data: "correct-horse-battery-staple", + } + } + WithSecretFunc(fn)(spec) + + want := []*engine.Secret{ + { + Metadata: engine.Metadata{ + Name: "password", + Namespace: "pivqfthg1c9hy83ylht1sxx4nygjc7tk", + }, + Data: "correct-horse-battery-staple", + }, + } + if diff := cmp.Diff(want, spec.Secrets, ignoreMetadata); diff != "" { + t.Errorf("Unexpected secret transform") + t.Log(diff) + } +} diff --git a/yaml/compiler/transform/volume.go b/yaml/compiler/transform/volume.go new file mode 100644 index 0000000..a8f5ad6 --- /dev/null +++ b/yaml/compiler/transform/volume.go @@ -0,0 +1,53 @@ +package transform + +import ( + "strings" + + "github.com/drone/drone-runtime/engine" + "github.com/drone/drone-yaml/yaml/compiler/internal/rand" +) + +// WithVolumes is a transform function that adds a set +// of global volumes to the container. +func WithVolumes(volumes map[string]string) func(*engine.Spec) { + return func(spec *engine.Spec) { + for key, value := range volumes { + volume := &engine.Volume{ + Metadata: engine.Metadata{ + UID: rand.String(), + Name: rand.String(), + Namespace: spec.Metadata.Name, + Labels: map[string]string{}, + }, + HostPath: &engine.VolumeHostPath{ + Path: key, + }, + } + spec.Docker.Volumes = append(spec.Docker.Volumes, volume) + for _, step := range spec.Steps { + mount := &engine.VolumeMount{ + Name: volume.Metadata.Name, + Path: value, + } + step.Volumes = append(step.Volumes, mount) + } + } + } +} + +// WithVolumeSlice is a transform function that adds a set +// of global volumes to the container that are defined in +// --volume=host:container format. +func WithVolumeSlice(volumes []string) func(*engine.Spec) { + to := map[string]string{} + for _, s := range volumes { + parts := strings.Split(s, ":") + if len(parts) != 2 { + continue + } + key := parts[0] + val := parts[1] + to[key] = val + } + return WithVolumes(to) +} diff --git a/yaml/compiler/transform/volume_test.go b/yaml/compiler/transform/volume_test.go new file mode 100644 index 0000000..b1cab16 --- /dev/null +++ b/yaml/compiler/transform/volume_test.go @@ -0,0 +1,69 @@ +package transform + +import ( + "testing" + + "github.com/drone/drone-runtime/engine" +) + +func TestWithVolumes(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{ + UID: "1", + Name: "build", + }, + Docker: &engine.DockerStep{ + Networks: nil, + }, + } + spec := &engine.Spec{ + Docker: &engine.DockerConfig{}, + Steps: []*engine.Step{step}, + } + vols := map[string]string{"/path/on/host": "/path/in/container"} + WithVolumes(vols)(spec) + + if len(step.Volumes) == 0 { + t.Error("Expected volume added to container") + } + if got, want := step.Volumes[0].Path, "/path/in/container"; got != want { + t.Errorf("Want mount path %s, got %s", want, got) + } + if len(spec.Docker.Volumes) == 0 { + t.Error("Expected volume added to spec") + } + if got, want := spec.Docker.Volumes[0].HostPath.Path, "/path/on/host"; got != want { + t.Errorf("Want host mount path %s, got %s", want, got) + } +} + +func TestWithVolumeSlice(t *testing.T) { + step := &engine.Step{ + Metadata: engine.Metadata{ + UID: "1", + Name: "build", + }, + Docker: &engine.DockerStep{ + Networks: nil, + }, + } + spec := &engine.Spec{ + Docker: &engine.DockerConfig{}, + Steps: []*engine.Step{step}, + } + vols := []string{"/path/on/host:/path/in/container"} + WithVolumeSlice(vols)(spec) + + if len(step.Volumes) == 0 { + t.Error("Expected volume added to container") + } + if got, want := step.Volumes[0].Path, "/path/in/container"; got != want { + t.Errorf("Want mount path %s, got %s", want, got) + } + if len(spec.Docker.Volumes) == 0 { + t.Error("Expected volume added to spec") + } + if got, want := spec.Docker.Volumes[0].HostPath.Path, "/path/on/host"; got != want { + t.Errorf("Want host mount path %s, got %s", want, got) + } +} diff --git a/yaml/compiler/workspace.go b/yaml/compiler/workspace.go new file mode 100644 index 0000000..a3b6c1e --- /dev/null +++ b/yaml/compiler/workspace.go @@ -0,0 +1,144 @@ +package compiler + +import ( + unixpath "path" + "strings" + + "github.com/drone/drone-runtime/engine" + "github.com/drone/drone-yaml/yaml" + "github.com/drone/drone-yaml/yaml/compiler/internal/rand" +) + +const ( + workspacePath = "/drone/src" + workspaceName = "workspace" + workspaceHostName = "host" +) + +func setupWorkingDir(src *yaml.Container, dst *engine.Step, path string) { + // if the working directory is already set + // do not alter. + if dst.WorkingDir != "" { + return + } + // if the user is running the container as a + // service (detached mode) with no commands, we + // should use the default working directory. + if dst.Detach && len(src.Commands) == 0 { + return + } + // else set the working directory. + dst.WorkingDir = path +} + +// helper function appends the workspace base and +// path to the step's list of environment variables. +func setupWorkspaceEnv(step *engine.Step, base, path, full string) { + step.Envs["DRONE_WORKSPACE_BASE"] = base + step.Envs["DRONE_WORKSPACE_PATH"] = path + step.Envs["DRONE_WORKSPACE"] = full + step.Envs["CI_WORKSPACE_BASE"] = base + step.Envs["CI_WORKSPACE_PATH"] = path + step.Envs["CI_WORKSPACE"] = full +} + +// helper function converts the path to a valid windows +// path, including the default C drive. +func toWindowsDrive(s string) string { + return "c:" + toWindowsPath(s) +} + +// helper function converts the path to a valid windows +// path, replacing backslashes with forward slashes. +func toWindowsPath(s string) string { + return strings.Replace(s, "/", "\\", -1) +} + +// +// +// + +func createWorkspace(from *yaml.Pipeline) (base, path, full string) { + base = from.Workspace.Base + path = from.Workspace.Path + if base == "" { + base = workspacePath + } + full = unixpath.Join(base, path) + + if from.Platform.OS == "windows" { + base = toWindowsDrive(base) + path = toWindowsPath(path) + full = toWindowsDrive(full) + } + return base, path, full +} + +// +// +// + +// CreateWorkspace creates the workspace volume as +// an empty directory mount. +func CreateWorkspace(spec *engine.Spec) { + spec.Docker.Volumes = append(spec.Docker.Volumes, + &engine.Volume{ + Metadata: engine.Metadata{ + UID: rand.String(), + Name: workspaceName, + Namespace: spec.Metadata.Namespace, + Labels: map[string]string{}, + }, + EmptyDir: &engine.VolumeEmptyDir{}, + }, + ) +} + +// CreateHostWorkspace returns a WorkspaceFunc that +// mounts a host machine volume as the pipeline +// workspace. +func CreateHostWorkspace(workdir string) func(*engine.Spec) { + return func(spec *engine.Spec) { + CreateWorkspace(spec) + spec.Docker.Volumes = append( + spec.Docker.Volumes, + &engine.Volume{ + Metadata: engine.Metadata{ + UID: rand.String(), + Name: workspaceHostName, + }, + HostPath: &engine.VolumeHostPath{ + Path: workdir, + }, + }, + ) + } +} + +// +// +// + +// MountWorkspace is a WorkspaceFunc that mounts the +// default workspace volume to the pipeline step. +func MountWorkspace(step *engine.Step, base, path, full string) { + step.Volumes = append(step.Volumes, &engine.VolumeMount{ + Name: workspaceName, + Path: base, + }) +} + +// MountHostWorkspace is a WorkspaceFunc that mounts +// the default workspace and host volume to the pipeline. +func MountHostWorkspace(step *engine.Step, base, path, full string) { + step.Volumes = append(step.Volumes, &engine.VolumeMount{ + Name: workspaceHostName, + Path: full, + }) + if path != "" { + step.Volumes = append(step.Volumes, &engine.VolumeMount{ + Name: workspaceName, + Path: base, + }) + } +} diff --git a/yaml/compiler/workspace_test.go b/yaml/compiler/workspace_test.go new file mode 100644 index 0000000..024489d --- /dev/null +++ b/yaml/compiler/workspace_test.go @@ -0,0 +1,144 @@ +package compiler + +import ( + "testing" + + "github.com/drone/drone-runtime/engine" + "github.com/drone/drone-yaml/yaml" +) + +func TestSetupWorkspace(t *testing.T) { + tests := []struct { + path string + src *yaml.Container + dst *engine.Step + want string + }{ + { + path: "/drone/src", + src: &yaml.Container{}, + dst: &engine.Step{}, + want: "/drone/src", + }, + // do not override the user-defined working dir. + { + path: "/drone/src", + src: &yaml.Container{}, + dst: &engine.Step{WorkingDir: "/foo"}, + want: "/foo", + }, + // do not override the default working directory + // for service containers with no commands. + { + path: "/drone/src", + src: &yaml.Container{}, + dst: &engine.Step{Detach: true}, + want: "", + }, + // overrides the default working directory + // for service containers with commands. + { + path: "/drone/src", + src: &yaml.Container{Commands: []string{"whoami"}}, + dst: &engine.Step{Detach: true}, + want: "/drone/src", + }, + } + for _, test := range tests { + setupWorkingDir(test.src, test.dst, test.path) + if got, want := test.dst.WorkingDir, test.want; got != want { + t.Errorf("Want working_dir %s, got %s", want, got) + } + } +} + +func TestToWindows(t *testing.T) { + got := toWindowsDrive("/go/src/github.com/octocat/hello-world") + want := "c:\\go\\src\\github.com\\octocat\\hello-world" + if got != want { + t.Errorf("Want windows drive %q, got %q", want, got) + } +} + +func TestCreateWorkspace(t *testing.T) { + tests := []struct { + from *yaml.Pipeline + base string + path string + full string + }{ + { + from: &yaml.Pipeline{ + Workspace: yaml.Workspace{ + Base: "", + Path: "", + }, + }, + base: "/drone/src", + path: "", + full: "/drone/src", + }, + { + from: &yaml.Pipeline{ + Workspace: yaml.Workspace{ + Base: "", + Path: "", + }, + Platform: yaml.Platform{ + OS: "windows", + }, + }, + base: "c:\\drone\\src", + path: "", + full: "c:\\drone\\src", + }, + { + from: &yaml.Pipeline{ + Workspace: yaml.Workspace{ + Base: "/drone", + Path: "src", + }, + }, + base: "/drone", + path: "src", + full: "/drone/src", + }, + { + from: &yaml.Pipeline{ + Workspace: yaml.Workspace{ + Base: "/drone", + Path: "src", + }, + Platform: yaml.Platform{ + OS: "windows", + }, + }, + base: "c:\\drone", + path: "src", + full: "c:\\drone\\src", + }, + { + from: &yaml.Pipeline{ + Workspace: yaml.Workspace{ + Base: "/foo", + Path: "bar", + }, + }, + base: "/foo", + path: "bar", + full: "/foo/bar", + }, + } + for _, test := range tests { + base, path, full := createWorkspace(test.from) + if got, want := test.base, base; got != want { + t.Errorf("Want workspace base %s, got %s", want, got) + } + if got, want := test.path, path; got != want { + t.Errorf("Want workspace path %s, got %s", want, got) + } + if got, want := test.full, full; got != want { + t.Errorf("Want workspace %s, got %s", want, got) + } + } +} diff --git a/yaml/cond.go b/yaml/cond.go new file mode 100644 index 0000000..4aefd17 --- /dev/null +++ b/yaml/cond.go @@ -0,0 +1,85 @@ +package yaml + +import filepath "github.com/bmatcuk/doublestar" + +// Conditions defines a group of conditions. +type Conditions struct { + Ref Condition `json:"ref,omitempty"` + Repo Condition `json:"repo,omitempty"` + Instance Condition `json:"instance,omitempty"` + Target Condition `json:"target,omitempty"` + Event Condition `json:"event,omitempty"` + Branch Condition `json:"branch,omitempty"` + Status Condition `json:"status,omitempty"` + Paths Condition `json:"paths,omitempty"` +} + +// Condition defines a runtime condition. +type Condition struct { + Include []string `yaml:"include,omitempty" json:"include,omitempty"` + Exclude []string `yaml:"exclude,omitempty" json:"exclude,omitempty"` +} + +// Match returns true if the string matches the include +// patterns and does not match any of the exclude patterns. +func (c *Condition) Match(v string) bool { + if c.Excludes(v) { + return false + } + if c.Includes(v) { + return true + } + if len(c.Include) == 0 { + return true + } + return false +} + +// Includes returns true if the string matches the include +// patterns. +func (c *Condition) Includes(v string) bool { + for _, pattern := range c.Include { + if ok, _ := filepath.Match(pattern, v); ok { + return true + } + } + return false +} + +// Excludes returns true if the string matches the exclude +// patterns. +func (c *Condition) Excludes(v string) bool { + for _, pattern := range c.Exclude { + if ok, _ := filepath.Match(pattern, v); ok { + return true + } + } + return false +} + +// UnmarshalYAML implements yml unmarshalling. +func (c *Condition) UnmarshalYAML(unmarshal func(interface{}) error) error { + var out1 string + var out2 []string + var out3 = struct { + Include []string + Exclude []string + }{} + + err := unmarshal(&out1) + if err == nil { + c.Include = []string{out1} + return nil + } + + unmarshal(&out2) + unmarshal(&out3) + + c.Exclude = out3.Exclude + c.Include = append( + out3.Include, + out2..., + ) + + return nil +} diff --git a/yaml/cond_test.go b/yaml/cond_test.go new file mode 100644 index 0000000..85fa56d --- /dev/null +++ b/yaml/cond_test.go @@ -0,0 +1,166 @@ +package yaml + +import ( + "testing" + + "gopkg.in/yaml.v2" +) + +func TestConstraintMatch(t *testing.T) { + testdata := []struct { + conf string + with string + want bool + }{ + // string value + { + conf: "master", + with: "develop", + want: false, + }, + { + conf: "master", + with: "master", + want: true, + }, + { + conf: "feature/*", + with: "feature/foo", + want: true, + }, + // slice value + { + conf: "[ master, feature/* ]", + with: "develop", + want: false, + }, + { + conf: "[ master, feature/* ]", + with: "master", + want: true, + }, + { + conf: "[ master, feature/* ]", + with: "feature/foo", + want: true, + }, + // includes block + { + conf: "include: [ master ]", + with: "develop", + want: false, + }, + { + conf: "include: [ master] ", + with: "master", + want: true, + }, + { + conf: "include: [ feature/* ]", + with: "master", + want: false, + }, + { + conf: "include: [ feature/* ]", + with: "feature/foo", + want: true, + }, + { + conf: "include: [ master, feature/* ]", + with: "develop", + want: false, + }, + { + conf: "include: [ master, feature/* ]", + with: "master", + want: true, + }, + { + conf: "include: [ master, feature/* ]", + with: "feature/foo", + want: true, + }, + // excludes block + { + conf: "exclude: [ master ]", + with: "develop", + want: true, + }, + { + conf: "exclude: [ master ]", + with: "master", + want: false, + }, + { + conf: "exclude: [ feature/* ]", + with: "master", + want: true, + }, + { + conf: "exclude: [ feature/* ]", + with: "feature/foo", + want: false, + }, + { + conf: "exclude: [ master, develop ]", + with: "master", + want: false, + }, + { + conf: "exclude: [ feature/*, bar ]", + with: "master", + want: true, + }, + { + conf: "exclude: [ feature/*, bar ]", + with: "feature/foo", + want: false, + }, + // include and exclude blocks + { + conf: "{ include: [ master, feature/* ], exclude: [ develop ] }", + with: "master", + want: true, + }, + { + conf: "{ include: [ master, feature/* ], exclude: [ feature/bar ] }", + with: "feature/bar", + want: false, + }, + { + conf: "{ include: [ master, feature/* ], exclude: [ master, develop ] }", + with: "master", + want: false, + }, + // empty blocks + { + conf: "", + with: "master", + want: true, + }, + // double star + { + conf: "foo/**", + with: "foo/bar/baz/qux", + want: true, + }, + { + conf: "foo/**/qux", + with: "foo/bar/baz/qux", + want: true, + }, + } + for _, test := range testdata { + c := parseCondition(test.conf) + got, want := c.Match(test.with), test.want + if got != want { + t.Errorf("Expect %q matches %q is %v", test.with, test.conf, want) + } + } +} + +func parseCondition(s string) *Condition { + c := &Condition{} + yaml.Unmarshal([]byte(s), c) + return c +} diff --git a/yaml/converter/bitbucket/config.go b/yaml/converter/bitbucket/config.go new file mode 100644 index 0000000..b66d534 --- /dev/null +++ b/yaml/converter/bitbucket/config.go @@ -0,0 +1,104 @@ +package bitbucket + +import ( + "path" + "strings" +) + +type ( + // Config defines the pipeline configuration. + Config struct { + // Image specifies the Docker image with + // which we run your builds. + Image string + + // Clone defines the depth of Git clones + // for all pipelines. + Clone struct { + Depth int + } + + // Pipeline defines the pipeline configuration + // which includes a list of all steps for default, + // tag, and branch-specific execution. + Pipelines struct { + Default Stage + Tags map[string]Stage + Branches map[string]Stage + } + + Definitions struct { + Services map[string]*Step + Caches map[string]string + } + } + + // Stage contains a list of steps executed + // for a specific branch or tag. + Stage struct { + Name string + Steps []*Step + } + + // Step defines a build execution unit. + Step struct { + // Name of the pipeline step. + Name string + + // Image specifies the Docker image with + // which we run your builds. + Image string + + // Script contains the list of bash commands + // that are executed in sequence. + Script []string + + // Variables provides environment variables + // passed to the container at runtime. + Variables map[string]string + + // Artifacts defines files that are to be + // snapshotted and shared with the subsequent + // step. This is not used, because Drone uses + // a shared volume to share artifacts. + Artifacts []string + } +) + +// Pipeline returns the pipeline stage that best matches the branch +// and ref. If there is no matching pipeline specific to the branch +// or tag, the default pipeline is returned. +func (c *Config) Pipeline(ref string) Stage { + // match pipeline by tag name + tag := strings.TrimPrefix(ref, "refs/tags/") + for pattern, pipeline := range c.Pipelines.Tags { + if ok, _ := path.Match(pattern, tag); ok { + return pipeline + } + } + // match pipeline by branch name + branch := strings.TrimPrefix(ref, "refs/heads/") + for pattern, pipeline := range c.Pipelines.Branches { + if ok, _ := path.Match(pattern, branch); ok { + return pipeline + } + } + // use default + return c.Pipelines.Default +} + +// UnmarshalYAML implements custom parsing for the stage section of the yaml +// to cleanup the structure a bit. +func (s *Stage) UnmarshalYAML(unmarshal func(interface{}) error) error { + in := []struct { + Step *Step + }{} + err := unmarshal(&in) + if err != nil { + return err + } + for _, step := range in { + s.Steps = append(s.Steps, step.Step) + } + return nil +} diff --git a/yaml/converter/bitbucket/config_test.go b/yaml/converter/bitbucket/config_test.go new file mode 100644 index 0000000..2649fca --- /dev/null +++ b/yaml/converter/bitbucket/config_test.go @@ -0,0 +1 @@ +package bitbucket diff --git a/yaml/converter/bitbucket/convert.go b/yaml/converter/bitbucket/convert.go new file mode 100644 index 0000000..d215017 --- /dev/null +++ b/yaml/converter/bitbucket/convert.go @@ -0,0 +1,82 @@ +package bitbucket + +import ( + "bytes" + "fmt" + + droneyaml "github.com/drone/drone-yaml/yaml" + "github.com/drone/drone-yaml/yaml/pretty" + + "gopkg.in/yaml.v2" +) + +// Convert converts the yaml configuration file from +// the legacy format to the 1.0+ format. +func Convert(b []byte, ref string) ([]byte, error) { + config := new(Config) + err := yaml.Unmarshal(b, config) + if err != nil { + return nil, err + } + + // TODO (bradrydzewski) to correctly choose + // the pipeline we need to pass the branch + // and ref. + stage := config.Pipeline(ref) + + pipeline := &droneyaml.Pipeline{} + pipeline.Name = "default" + pipeline.Kind = "pipeline" + + // + // clone + // + + pipeline.Clone.Depth = config.Clone.Depth + + // + // steps + // + + for i, from := range stage.Steps { + to := toContainer(from) + // defaults to the global image if the + // step does not define an image. + if to.Image == "" { + to.Image = config.Image + } + if to.Name == "" { + to.Name = fmt.Sprintf("step_%d", i) + } + pipeline.Steps = append(pipeline.Steps, to) + } + + // + // services + // + + for name, from := range config.Definitions.Services { + to := toContainer(from) + to.Name = name + pipeline.Services = append(pipeline.Services, to) + } + + // + // wrap the pipeline in the manifest + // + + manifest := &droneyaml.Manifest{} + manifest.Resources = append(manifest.Resources, pipeline) + + buf := new(bytes.Buffer) + pretty.Print(buf, manifest) + return buf.Bytes(), nil +} + +func toContainer(from *Step) *droneyaml.Container { + return &droneyaml.Container{ + Name: from.Name, + Image: from.Image, + Commands: from.Script, + } +} diff --git a/yaml/converter/bitbucket/convert_test.go b/yaml/converter/bitbucket/convert_test.go new file mode 100644 index 0000000..2e5e18d --- /dev/null +++ b/yaml/converter/bitbucket/convert_test.go @@ -0,0 +1,46 @@ +package bitbucket + +import ( + "bytes" + "io/ioutil" + "testing" +) + +func TestConvert(t *testing.T) { + tests := []struct { + before, after, ref string + }{ + { + before: "testdata/sample1.yaml", + after: "testdata/sample1.yaml.golden", + ref: "refs/heads/master", + }, + { + before: "testdata/sample2.yaml", + after: "testdata/sample2.yaml.golden", + ref: "refs/heads/feature/foo", + }, + } + + for _, test := range tests { + a, err := ioutil.ReadFile(test.before) + if err != nil { + t.Error(err) + return + } + b, err := ioutil.ReadFile(test.after) + if err != nil { + t.Error(err) + return + } + c, err := Convert([]byte(a), test.ref) + if err != nil { + t.Error(err) + return + } + if bytes.Equal(b, c) == false { + t.Errorf("Unexpected yaml conversion of %s", test.before) + t.Log(string(c)) + } + } +} diff --git a/yaml/converter/bitbucket/testdata/sample1.yaml b/yaml/converter/bitbucket/testdata/sample1.yaml new file mode 100644 index 0000000..daa7868 --- /dev/null +++ b/yaml/converter/bitbucket/testdata/sample1.yaml @@ -0,0 +1,32 @@ +pipelines: + default: + - step: + name: Build and test + image: node:8.5.0 + caches: + - node + script: + - npm install + - npm test + - npm build + artifacts: + - dist/** + - step: + name: Integration test + image: node:8.5.0 + caches: + - node + services: + - postgres + script: + - npm run integration-test + - step: + name: Deploy to beanstalk + image: python:3.5.1 + script: + - python deploy-to-beanstalk.py + +definitions: + services: + postgres: + image: postgres:9.6.4 diff --git a/yaml/converter/bitbucket/testdata/sample1.yaml.golden b/yaml/converter/bitbucket/testdata/sample1.yaml.golden new file mode 100644 index 0000000..45c7564 --- /dev/null +++ b/yaml/converter/bitbucket/testdata/sample1.yaml.golden @@ -0,0 +1,31 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: Build and test + image: node:8.5.0 + commands: + - npm install + - npm test + - npm build + +- name: Integration test + image: node:8.5.0 + commands: + - npm run integration-test + +- name: Deploy to beanstalk + image: python:3.5.1 + commands: + - python deploy-to-beanstalk.py + +services: +- name: postgres + image: postgres:9.6.4 + +... diff --git a/yaml/converter/bitbucket/testdata/sample2.yaml b/yaml/converter/bitbucket/testdata/sample2.yaml new file mode 100644 index 0000000..3efce48 --- /dev/null +++ b/yaml/converter/bitbucket/testdata/sample2.yaml @@ -0,0 +1,40 @@ +pipelines: + branches: + feature/*: + - step: + name: Test + image: node:latest + script: + - npm install + - npm test + default: + - step: + name: Build and test + image: node:8.5.0 + caches: + - node + script: + - npm install + - npm test + - npm build + artifacts: + - dist/** + - step: + name: Integration test + image: node:8.5.0 + caches: + - node + services: + - postgres + script: + - npm run integration-test + - step: + name: Deploy to beanstalk + image: python:3.5.1 + script: + - python deploy-to-beanstalk.py + +definitions: + services: + postgres: + image: postgres:9.6.4 diff --git a/yaml/converter/bitbucket/testdata/sample2.yaml.golden b/yaml/converter/bitbucket/testdata/sample2.yaml.golden new file mode 100644 index 0000000..b9fe2cd --- /dev/null +++ b/yaml/converter/bitbucket/testdata/sample2.yaml.golden @@ -0,0 +1,20 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: Test + image: node:latest + commands: + - npm install + - npm test + +services: +- name: postgres + image: postgres:9.6.4 + +... diff --git a/yaml/converter/circleci/config.go b/yaml/converter/circleci/config.go new file mode 100644 index 0000000..452cc60 --- /dev/null +++ b/yaml/converter/circleci/config.go @@ -0,0 +1,76 @@ +package circleci + +type ( + // Config defines the pipeline configuration. + Config struct { + // Version specifies the yaml configuration + // file version. + Version string + + // Jobs defines a list of pipeline jobs. + Jobs []*Job + + // Workflows are used to orchestrate jobs. + Workflows struct { + Version string + List map[string]*Workflow `yaml:",inline"` + } + } + + // Workflow ochestrates one or more jobs. + Workflow struct { + Jobs []string + } + + // Job defines a pipeline job. + Job struct { + // Name of the stage. + Name string + + // Docker configures a Docker executor. + Docker Docker + + // Environment variables passed to the executor. + Environment map[string]string + + // Steps configures the Job steps. + Steps map[string]Step + + // Branches limits execution by branch. + Branches []struct { + Only []string + Ignore []string + } + } + + // Step defines a build execution unit. + Step struct { + Run Run + AddSSHKeys map[string]interface{} `yaml:"add_ssh_keys"` + AttachWorkspace map[string]interface{} `yaml:"attach_workspace"` + Checkout map[string]interface{} `yaml:"checkout"` + Deploy map[string]interface{} `yaml:"deploy"` + PersistToWorkspace map[string]interface{} `yaml:"persist_to_workspace"` + RestoreCache map[string]interface{} `yaml:"restore_cache"` + SaveCache map[string]interface{} `yaml:"save_cache"` + SetupRemoteDocker map[string]interface{} `yaml:"setup_remote_docker"` + StoreArtifacts map[string]interface{} `yaml:"store_artifacts"` + StoreTestResults map[string]interface{} `yaml:"store_test_results"` + } +) + +// // UnmarshalYAML implements custom parsing for the stage section of the yaml +// // to cleanup the structure a bit. +// func (s *Stage) UnmarshalYAML(unmarshal func(interface{}) error) error { +// in := []struct { +// Step *Step +// }{} +// err := unmarshal(&in) +// if err != nil { +// return err +// } +// for _, step := range in { +// s.Steps = append(s.Steps, step.Step) +// } +// return nil +// } diff --git a/yaml/converter/circleci/docker.go b/yaml/converter/circleci/docker.go new file mode 100644 index 0000000..f27c0e4 --- /dev/null +++ b/yaml/converter/circleci/docker.go @@ -0,0 +1,28 @@ +package circleci + +// Docker configures a Docker executor. +type Docker struct { + // Image is the Docker image name. + Image string + + // Name is the Docker container hostname. + Name string + + // Entrypoint is the Docker container entrypoint. + Entrypoint []string + + // Command is the Docker container command. + Command []string + + // User is user that runs the Docker entrypoint. + User string + + // Environment variables passed to the container. + Environment map[string]string + + // Auth credentials to pull private images. + Auth map[string]string + + // Auth credentials to pull private ECR images. + AWSAuth map[string]string `yaml:"aws_auth"` +} diff --git a/yaml/converter/circleci/docker_test.go b/yaml/converter/circleci/docker_test.go new file mode 100644 index 0000000..f569327 --- /dev/null +++ b/yaml/converter/circleci/docker_test.go @@ -0,0 +1 @@ +package circleci diff --git a/yaml/converter/circleci/run.go b/yaml/converter/circleci/run.go new file mode 100644 index 0000000..2b481b5 --- /dev/null +++ b/yaml/converter/circleci/run.go @@ -0,0 +1,34 @@ +package circleci + +import "time" + +// Run defines a command +type Run struct { + // Name of the command + Name string + + // Command run in the shell. + Command string + + // Shell to use to execute the command. + Shell string + + // Workiring Directory in which the command + // is run. + WorkingDir string `yaml:"working_directory"` + + // Command is run in the background. + Background bool `yaml:"background"` + + // Amount of time the command can run with + // no output before being canceled. + NoOutputTimeout time.Duration `yaml:"no_output_timeout"` + + // Environment variables set when running + // the command in the shell. + Environment map[string]string + + // Defines when the command should be executed. + // Values are always, on_success, and on_fail. + When string +} diff --git a/yaml/converter/circleci/run_test.go b/yaml/converter/circleci/run_test.go new file mode 100644 index 0000000..9a81da8 --- /dev/null +++ b/yaml/converter/circleci/run_test.go @@ -0,0 +1,11 @@ +package circleci + +const testRun = ` +- run: + name: test + command: go test +` + +const testRunShort = ` +- run: go test +` diff --git a/yaml/converter/circleci/testdata/sample1.yml b/yaml/converter/circleci/testdata/sample1.yml new file mode 100644 index 0000000..567e666 --- /dev/null +++ b/yaml/converter/circleci/testdata/sample1.yml @@ -0,0 +1,22 @@ +version: 2 +jobs: + backend: + docker: + - image: golang:1.8 + steps: + - checkout + - run: go build + - run: go test + frontend: + docker: + - image: node:latest + steps: + - checkout + - run: npm install + - run: npm test +workflows: + version: 2 + default: + jobs: + - backend + - frontend \ No newline at end of file diff --git a/yaml/converter/circleci/testdata/sample1.yml.golden b/yaml/converter/circleci/testdata/sample1.yml.golden new file mode 100644 index 0000000..e69de29 diff --git a/yaml/converter/convert.go b/yaml/converter/convert.go new file mode 100644 index 0000000..0c1e283 --- /dev/null +++ b/yaml/converter/convert.go @@ -0,0 +1,51 @@ +package converter + +import ( + "github.com/drone/drone-yaml/yaml/converter/bitbucket" + "github.com/drone/drone-yaml/yaml/converter/gitlab" + "github.com/drone/drone-yaml/yaml/converter/legacy" +) + +// Metadata provides additional metadata used to +// convert the configuration file format. +type Metadata struct { + // Filename of the configuration file, helps + // determine the yaml configuration format. + Filename string + + // Ref of the commit use to choose the correct + // pipeline if the configuration format defines + // multiple pipelines (like Bitbucket) + Ref string +} + +// Convert converts the yaml configuration file from +// the legacy format to the 1.0+ format. +func Convert(d []byte, m Metadata) ([]byte, error) { + switch m.Filename { + case "bitbucket-pipelines.yml": + return bitbucket.Convert(d, m.Ref) + case "circle.yml", ".circleci/config.yml": + // TODO(bradrydzewski) + case ".gitlab-ci.yml": + return gitlab.Convert(d) + case ".travis.yml": + // TODO(bradrydzewski) + } + // if the filename does not match any external + // systems we check to see if the configuration + // file is a legacy (pre 1.0) .drone.yml format. + if legacy.Match(d) { + return legacy.Convert(d) + } + // else return the unmodified configuration + // back to the caller. + return d, nil +} + +// ConvertString converts the yaml configuration file from +// the legacy format to the 1.0+ format. +func ConvertString(s string, m Metadata) (string, error) { + b, err := Convert([]byte(s), m) + return string(b), err +} diff --git a/yaml/converter/convert_test.go b/yaml/converter/convert_test.go new file mode 100644 index 0000000..89f617e --- /dev/null +++ b/yaml/converter/convert_test.go @@ -0,0 +1 @@ +package converter diff --git a/yaml/converter/gitlab/config.go b/yaml/converter/gitlab/config.go new file mode 100644 index 0000000..8299c73 --- /dev/null +++ b/yaml/converter/gitlab/config.go @@ -0,0 +1,125 @@ +package gitlab + +import ( + "github.com/drone/drone-yaml/yaml/converter/internal" +) + +type ( + // Config defines the pipeline configuration. + Config struct { + // Image specifies the Docker image with + // which we run your builds. + Image Image + + // Stages is used to group steps into stages, + // where each stage is executed sequentially. + Stages []string + + // Services is used to define a set of services + // that should be started and linked to each + // step in the pipeline. + Services []*Image + + // Variables is used to customize execution, + // such as the clone strategy. + Variables map[string]string + + // Before contains the list of bash commands + // that are executed in sequence before the + // first job. + Before internal.StringSlice `yaml:"before_script"` + + // After contains the list of bash commands + // that are executed in sequence after the + // last job. + After internal.StringSlice `yaml:"after_script"` + + // Jobs is used to define individual units + // of execution that make up a stage. + Jobs map[string]*Job `yaml:",inline"` + } + + // Job defines a build execution unit. + Job struct { + // Name of the pipeline step. + Name string + + // Stage is the name of the stage. + Stage string + + // Image specifies the Docker image with + // which we run your builds. + Image Image + + // Script contains the list of bash commands + // that are executed in sequence. + Script internal.StringSlice + + // Before contains the list of bash commands + // that are executed in sequence before the + // primary script. + Before internal.StringSlice `yaml:"before_script"` + + // After contains the list of bash commands + // that are executed in sequence after the + // primary script. + After internal.StringSlice `yaml:"after_script"` + + // Services defines a set of services linked + // to the job. + Services []*Image + + // Only defines the names of branches and tags + // for which the job will run. + Only internal.StringSlice + + // Except defines the names of branches and tags + // for which the job will not run. + Except internal.StringSlice + + // Variables is used to customize execution, + // such as the clone strategy. + Variables map[string]string + + // Allow job to fail. Failed job doesn’t contribute + // to commit status + AllowFailure bool + + // Define when to run job. Can be on_success, on_failure, + // always or manual + When internal.StringSlice + } + + // Image defines a Docker image. + Image struct { + Name string + Entrypoint []string + Command []string + Alias string + } +) + +// UnmarshalYAML implements custom parsing for an Image. +func (i *Image) UnmarshalYAML(unmarshal func(interface{}) error) error { + var name string + err := unmarshal(&name) + if err == nil { + i.Name = name + return nil + } + data := struct { + Name string + Entrypoint internal.StringSlice + Command internal.StringSlice + Alias string + }{} + err = unmarshal(&data) + if err != nil { + return err + } + i.Name = data.Name + i.Entrypoint = data.Entrypoint + i.Command = data.Command + i.Alias = data.Alias + return nil +} diff --git a/yaml/converter/gitlab/convert.go b/yaml/converter/gitlab/convert.go new file mode 100644 index 0000000..3070f90 --- /dev/null +++ b/yaml/converter/gitlab/convert.go @@ -0,0 +1,95 @@ +package gitlab + +import ( + "bytes" + + droneyaml "github.com/drone/drone-yaml/yaml" + "github.com/drone/drone-yaml/yaml/pretty" + + "github.com/gosimple/slug" + "gopkg.in/yaml.v2" +) + +// Convert converts the yaml configuration file from +// the legacy format to the 1.0+ format. +func Convert(b []byte) ([]byte, error) { + config := new(Config) + err := yaml.Unmarshal(b, config) + if err != nil { + return nil, err + } + manifest := &droneyaml.Manifest{} + + // if no stages are defined, we create a single, + // default stage that will be used for all jobs. + if len(config.Stages) == 0 { + for name, job := range config.Jobs { + config.Stages = append(config.Stages, name) + job.Stage = name + } + } + + // create a new pipeline for each stage. + var prevstage string + for _, stage := range config.Stages { + pipeline := &droneyaml.Pipeline{} + pipeline.Name = stage + pipeline.Kind = droneyaml.KindPipeline + manifest.Resources = append(manifest.Resources, pipeline) + for name, job := range config.Jobs { + if job.Stage != stage { + continue + } + cmds := []string(config.Before) + cmds = append(cmds, []string(job.Before)...) + cmds = append(cmds, []string(job.Script)...) + cmds = append(cmds, []string(job.After)...) + cmds = append(cmds, []string(config.After)...) + + step := &droneyaml.Container{ + Name: name, + Image: job.Image.Name, + Command: job.Image.Command, + Entrypoint: job.Image.Entrypoint, + Commands: cmds, + } + + if job.AllowFailure { + step.Failure = "ignore" + } + + if step.Image == "" { + step.Image = config.Image.Name + } + // TODO: handle Services + // TODO: handle Only + // TODO: handle Except + // TODO: handle Variables + // TODO: handle When + + pipeline.Steps = append(pipeline.Steps, step) + } + + for _, step := range config.Services { + step := &droneyaml.Container{ + Name: step.Alias, + Image: step.Name, + Command: step.Command, + Entrypoint: step.Entrypoint, + } + if step.Name == "" { + step.Name = slug.Make(step.Image) + } + pipeline.Services = append(pipeline.Services, step) + } + + if prevstage != "" { + pipeline.DependsOn = []string{prevstage} + } + prevstage = stage + } + + buf := new(bytes.Buffer) + pretty.Print(buf, manifest) + return buf.Bytes(), nil +} diff --git a/yaml/converter/gitlab/convert_test.go b/yaml/converter/gitlab/convert_test.go new file mode 100644 index 0000000..0001f47 --- /dev/null +++ b/yaml/converter/gitlab/convert_test.go @@ -0,0 +1,57 @@ +package gitlab + +import ( + "bytes" + "io/ioutil" + "testing" + + "github.com/sergi/go-diff/diffmatchpatch" +) + +func TestConvert(t *testing.T) { + tests := []struct { + before, after, ref string + }{ + { + before: "testdata/example1.yml", + after: "testdata/example1.yml.golden", + }, + { + before: "testdata/example2.yml", + after: "testdata/example2.yml.golden", + }, + { + before: "testdata/example3.yml", + after: "testdata/example3.yml.golden", + }, + { + before: "testdata/example4.yml", + after: "testdata/example4.yml.golden", + }, + } + + for _, test := range tests { + a, err := ioutil.ReadFile(test.before) + if err != nil { + t.Error(err) + return + } + b, err := ioutil.ReadFile(test.after) + if err != nil { + t.Error(err) + return + } + c, err := Convert([]byte(a)) + if err != nil { + t.Error(err) + return + } + + if bytes.Equal(b, c) == false { + t.Errorf("Unexpected yaml conversion of %s", test.before) + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(string(b), string(c), false) + t.Log(dmp.DiffCleanupSemantic(diffs)) + } + } +} diff --git a/yaml/converter/gitlab/testdata/example1.yml b/yaml/converter/gitlab/testdata/example1.yml new file mode 100644 index 0000000..55b9516 --- /dev/null +++ b/yaml/converter/gitlab/testdata/example1.yml @@ -0,0 +1,11 @@ +image: ruby:2.2 + +services: + - postgres:9.3 + +before_script: + - bundle install + +test: + script: + - bundle exec rake spec diff --git a/yaml/converter/gitlab/testdata/example1.yml.golden b/yaml/converter/gitlab/testdata/example1.yml.golden new file mode 100644 index 0000000..4308b81 --- /dev/null +++ b/yaml/converter/gitlab/testdata/example1.yml.golden @@ -0,0 +1,20 @@ +--- +kind: pipeline +name: test + +platform: + os: linux + arch: amd64 + +steps: +- name: test + image: ruby:2.2 + commands: + - bundle install + - bundle exec rake spec + +services: +- name: postgres-9-3 + image: postgres:9.3 + +... diff --git a/yaml/converter/gitlab/testdata/example2.yml b/yaml/converter/gitlab/testdata/example2.yml new file mode 100644 index 0000000..9e54b8a --- /dev/null +++ b/yaml/converter/gitlab/testdata/example2.yml @@ -0,0 +1,16 @@ +before_script: + - bundle install + +test2.1: + image: ruby:2.1 + services: + - postgres:9.3 + script: + - bundle exec rake spec + +test2.2: + image: ruby:2.2 + services: + - postgres:9.4 + script: + - bundle exec rake spec \ No newline at end of file diff --git a/yaml/converter/gitlab/testdata/example2.yml.golden b/yaml/converter/gitlab/testdata/example2.yml.golden new file mode 100644 index 0000000..c5c178c --- /dev/null +++ b/yaml/converter/gitlab/testdata/example2.yml.golden @@ -0,0 +1,34 @@ +--- +kind: pipeline +name: test2.1 + +platform: + os: linux + arch: amd64 + +steps: +- name: test2.1 + image: ruby:2.1 + commands: + - bundle install + - bundle exec rake spec + +--- +kind: pipeline +name: test2.2 + +platform: + os: linux + arch: amd64 + +steps: +- name: test2.2 + image: ruby:2.2 + commands: + - bundle install + - bundle exec rake spec + +depends_on: +- test2.1 + +... diff --git a/yaml/converter/gitlab/testdata/example3.yml b/yaml/converter/gitlab/testdata/example3.yml new file mode 100644 index 0000000..ce1a4e0 --- /dev/null +++ b/yaml/converter/gitlab/testdata/example3.yml @@ -0,0 +1,16 @@ +image: + name: ruby:2.2 + entrypoint: ["/bin/bash"] + +services: +- name: my-postgres:9.4 + alias: db-postgres + entrypoint: ["/usr/local/bin/db-postgres"] + command: ["start"] + +before_script: +- bundle install + +test: + script: + - bundle exec rake spec \ No newline at end of file diff --git a/yaml/converter/gitlab/testdata/example3.yml.golden b/yaml/converter/gitlab/testdata/example3.yml.golden new file mode 100644 index 0000000..ab5f1c8 --- /dev/null +++ b/yaml/converter/gitlab/testdata/example3.yml.golden @@ -0,0 +1,24 @@ +--- +kind: pipeline +name: test + +platform: + os: linux + arch: amd64 + +steps: +- name: test + image: ruby:2.2 + commands: + - bundle install + - bundle exec rake spec + +services: +- name: db-postgres + image: my-postgres:9.4 + entrypoint: + - /usr/local/bin/db-postgres + command: + - start + +... diff --git a/yaml/converter/gitlab/testdata/example4.yml b/yaml/converter/gitlab/testdata/example4.yml new file mode 100644 index 0000000..f001558 --- /dev/null +++ b/yaml/converter/gitlab/testdata/example4.yml @@ -0,0 +1,22 @@ +stages: + - build + - test + - deploy + +image: ruby:2.2 + +job 1: + stage: build + script: make build dependencies + +job 2: + stage: build + script: make build artifacts + +job 3: + stage: test + script: make test + +job 4: + stage: deploy + script: make deploy diff --git a/yaml/converter/gitlab/testdata/example4.yml.golden b/yaml/converter/gitlab/testdata/example4.yml.golden new file mode 100644 index 0000000..1ed34d9 --- /dev/null +++ b/yaml/converter/gitlab/testdata/example4.yml.golden @@ -0,0 +1,54 @@ +--- +kind: pipeline +name: build + +platform: + os: linux + arch: amd64 + +steps: +- name: job 1 + image: ruby:2.2 + commands: + - make build dependencies + +- name: job 2 + image: ruby:2.2 + commands: + - make build artifacts + +--- +kind: pipeline +name: test + +platform: + os: linux + arch: amd64 + +steps: +- name: job 3 + image: ruby:2.2 + commands: + - make test + +depends_on: +- build + +--- +kind: pipeline +name: deploy + +platform: + os: linux + arch: amd64 + +steps: +- name: job 4 + image: ruby:2.2 + commands: + - make deploy + +depends_on: +- test + +... diff --git a/yaml/converter/internal/string_slice.go b/yaml/converter/internal/string_slice.go new file mode 100644 index 0000000..88eca58 --- /dev/null +++ b/yaml/converter/internal/string_slice.go @@ -0,0 +1,20 @@ +package internal + +// StringSlice represents a slice of strings or a string. +type StringSlice []string + +// UnmarshalYAML implements the Unmarshaller interface. +func (s *StringSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + var stringType string + if err := unmarshal(&stringType); err == nil { + *s = []string{stringType} + return nil + } + + var sliceType []string + if err := unmarshal(&sliceType); err != nil { + return err + } + *s = sliceType + return nil +} diff --git a/yaml/converter/internal/string_slice_test.go b/yaml/converter/internal/string_slice_test.go new file mode 100644 index 0000000..5540bba --- /dev/null +++ b/yaml/converter/internal/string_slice_test.go @@ -0,0 +1,45 @@ +package internal + +import ( + "reflect" + "testing" + + "gopkg.in/yaml.v2" +) + +func TestStringSlice(t *testing.T) { + var tests = []struct { + yaml string + want []string + }{ + { + yaml: "hello world", + want: []string{"hello world"}, + }, + { + yaml: "[ hello, world ]", + want: []string{"hello", "world"}, + }, + { + yaml: "42", + want: []string{"42"}, + }, + } + + for _, test := range tests { + var got StringSlice + + if err := yaml.Unmarshal([]byte(test.yaml), &got); err != nil { + t.Error(err) + } + + if !reflect.DeepEqual([]string(got), test.want) { + t.Errorf("Got slice %v want %v", got, test.want) + } + } + + var got StringSlice + if err := yaml.Unmarshal([]byte("{}"), &got); err == nil { + t.Errorf("Want error unmarshaling invalid string or slice value.") + } +} diff --git a/yaml/converter/legacy/convert.go b/yaml/converter/legacy/convert.go new file mode 100644 index 0000000..3ff1c4f --- /dev/null +++ b/yaml/converter/legacy/convert.go @@ -0,0 +1,9 @@ +package legacy + +import "github.com/drone/drone-yaml/yaml/converter/legacy/internal" + +// Convert converts the yaml configuration file from +// the legacy format to the 1.0+ format. +func Convert(d []byte) ([]byte, error) { + return yaml.Convert(d) +} diff --git a/yaml/converter/legacy/internal/config.go b/yaml/converter/legacy/internal/config.go new file mode 100644 index 0000000..ee82218 --- /dev/null +++ b/yaml/converter/legacy/internal/config.go @@ -0,0 +1,247 @@ +package yaml + +import ( + "bytes" + "fmt" + "strings" + + droneyaml "github.com/drone/drone-yaml/yaml" + "github.com/drone/drone-yaml/yaml/pretty" + + "gopkg.in/yaml.v2" +) + +// Config provides the high-level configuration. +type Config struct { + Workspace struct { + Base string + Path string + } + Clone Containers + Pipeline Containers + Services Containers + Branches Constraint + Secrets map[string]struct { + Driver string + DriverOpts map[string]string `yaml:"driver_opts"` + Path string + Vault string + } +} + +// Convert converts the yaml configuration file from +// the legacy format to the 1.0+ format. +func Convert(d []byte) ([]byte, error) { + from := new(Config) + err := yaml.Unmarshal(d, from) + if err != nil { + return nil, err + } + + pipeline := &droneyaml.Pipeline{} + pipeline.Name = "default" + pipeline.Kind = "pipeline" + + pipeline.Workspace.Base = from.Workspace.Base + pipeline.Workspace.Path = from.Workspace.Path + + if len(from.Clone.Containers) != 0 { + pipeline.Clone.Disable = true + for _, container := range from.Clone.Containers { + pipeline.Steps = append(pipeline.Steps, + toContainer(container), + ) + } + } + + for _, container := range from.Services.Containers { + pipeline.Services = append(pipeline.Services, + toContainer(container), + ) + } + + for _, container := range from.Pipeline.Containers { + pipeline.Steps = append(pipeline.Steps, + toContainer(container), + ) + } + pipeline.Volumes = toVolumes(from) + pipeline.Trigger.Branch.Include = from.Branches.Include + pipeline.Trigger.Branch.Exclude = from.Branches.Exclude + + manifest := &droneyaml.Manifest{} + manifest.Resources = append(manifest.Resources, pipeline) + + secrets := toSecrets(from) + if secrets != nil { + manifest.Resources = append(manifest.Resources, secrets) + } + + buf := new(bytes.Buffer) + pretty.Print(buf, manifest) + return buf.Bytes(), nil +} + +func toContainer(from *Container) *droneyaml.Container { + return &droneyaml.Container{ + Name: from.Name, + Image: from.Image, + Detach: from.Detached, + Command: from.Command, + Commands: from.Commands, + DNS: from.DNS, + DNSSearch: from.DNSSearch, + Entrypoint: from.Entrypoint, + Environment: toEnvironment(from), + ExtraHosts: from.ExtraHosts, + Pull: toPullPolicy(from.Pull), + Privileged: from.Privileged, + Settings: toSettings(from.Vargs), + Volumes: toVolumeMounts(from.Volumes), + When: toConditions(from.Constraints), + } +} + +// helper function converts the legacy constraint syntax +// to the new condition syntax. +func toConditions(from Constraints) droneyaml.Conditions { + return droneyaml.Conditions{ + Ref: droneyaml.Condition{ + Include: from.Ref.Include, + Exclude: from.Ref.Exclude, + }, + Repo: droneyaml.Condition{ + Include: from.Repo.Include, + Exclude: from.Repo.Exclude, + }, + Instance: droneyaml.Condition{ + Include: from.Instance.Include, + Exclude: from.Instance.Exclude, + }, + Target: droneyaml.Condition{ + Include: from.Environment.Include, + Exclude: from.Environment.Exclude, + }, + Event: droneyaml.Condition{ + Include: from.Event.Include, + Exclude: from.Event.Exclude, + }, + Branch: droneyaml.Condition{ + Include: from.Branch.Include, + Exclude: from.Branch.Exclude, + }, + Status: droneyaml.Condition{ + Include: from.Status.Include, + Exclude: from.Status.Exclude, + }, + } +} + +// helper function converts the legacy environment syntax +// to the new environment syntax. +func toEnvironment(from *Container) map[string]*droneyaml.Variable { + envs := map[string]*droneyaml.Variable{} + for key, val := range from.Environment.Map { + envs[key] = &droneyaml.Variable{ + Value: val, + } + } + for _, val := range from.Secrets.Secrets { + name := strings.ToUpper(val.Target) + envs[name] = &droneyaml.Variable{ + Secret: val.Source, + } + } + return envs +} + +// helper function converts the legacy image pull syntax +// to the new pull policy syntax. +func toPullPolicy(pull bool) string { + switch pull { + case true: + return "always" + default: + return "default" + } +} + +// helper function converts the legacy secret syntax to the +// new secret variable syntax. +func toSecrets(from *Config) *droneyaml.Secret { + secret := &droneyaml.Secret{} + secret.Kind = "secret" + secret.Type = "general" + secret.External = map[string]droneyaml.ExternalData{} + for key, val := range from.Secrets { + external := droneyaml.ExternalData{} + if val.Driver == "vault" { + if val.DriverOpts != nil { + external.Path = val.DriverOpts["path"] + external.Name = val.DriverOpts["key"] + } + } else if val.Path != "" { + external.Path = val.Path + } else { + external.Path = val.Vault + } + secret.External[key] = external + } + if len(secret.External) == 0 { + return nil + } + return secret +} + +// helper function converts the legacy vargs syntax to the +// new environment syntax. +func toSettings(from map[string]interface{}) map[string]*droneyaml.Parameter { + params := map[string]*droneyaml.Parameter{} + for key, val := range from { + params[key] = &droneyaml.Parameter{ + Value: val, + } + } + return params +} + +// helper function converts the legacy volume syntax +// to the new volume mount syntax. +func toVolumeMounts(from []*Volume) []*droneyaml.VolumeMount { + to := []*droneyaml.VolumeMount{} + for _, v := range from { + to = append(to, &droneyaml.VolumeMount{ + Name: fmt.Sprintf("%x", v.Source), + MountPath: v.Destination, + }) + } + return to +} + +// helper function converts the legacy volume syntax +// to the new volume mount syntax. +func toVolumes(from *Config) []*droneyaml.Volume { + set := map[string]struct{}{} + to := []*droneyaml.Volume{} + + containers := []*Container{} + containers = append(containers, from.Pipeline.Containers...) + containers = append(containers, from.Services.Containers...) + + for _, container := range containers { + for _, v := range container.Volumes { + name := fmt.Sprintf("%x", v.Source) + if _, ok := set[name]; ok { + continue + } + set[name] = struct{}{} + to = append(to, &droneyaml.Volume{ + Name: name, + HostPath: &droneyaml.VolumeHostPath{ + Path: v.Source, + }, + }) + } + } + return to +} diff --git a/yaml/converter/legacy/internal/config_test.go b/yaml/converter/legacy/internal/config_test.go new file mode 100644 index 0000000..66f74fb --- /dev/null +++ b/yaml/converter/legacy/internal/config_test.go @@ -0,0 +1,52 @@ +package yaml + +import ( + "bytes" + "io/ioutil" + "testing" +) + +func TestConvert(t *testing.T) { + tests := []struct { + before, after string + }{ + { + before: "testdata/simple.yml", + after: "testdata/simple.yml.golden", + }, + { + before: "testdata/vault_1.yml", + after: "testdata/vault_1.yml.golden", + }, + { + before: "testdata/vault_2.yml", + after: "testdata/vault_2.yml.golden", + }, + { + before: "testdata/vault_3.yml", + after: "testdata/vault_3.yml.golden", + }, + } + + for _, test := range tests { + a, err := ioutil.ReadFile(test.before) + if err != nil { + t.Error(err) + return + } + b, err := ioutil.ReadFile(test.after) + if err != nil { + t.Error(err) + return + } + c, err := Convert(a) + if err != nil { + t.Error(err) + return + } + if bytes.Equal(b, c) == false { + t.Errorf("Unexpected yaml conversion of %s", test.before) + t.Log(string(c)) + } + } +} diff --git a/yaml/converter/legacy/internal/constraint.go b/yaml/converter/legacy/internal/constraint.go new file mode 100644 index 0000000..e11f0a7 --- /dev/null +++ b/yaml/converter/legacy/internal/constraint.go @@ -0,0 +1,69 @@ +package yaml + +type ( + // Constraints defines a set of runtime constraints. + Constraints struct { + Ref Constraint + Repo Constraint + Instance Constraint + Environment Constraint + Event Constraint + Branch Constraint + Status Constraint + } + + // Constraint defines a runtime constraint. + Constraint struct { + Include []string + Exclude []string + } + + // ConstraintMap defines a runtime constraint map. + ConstraintMap struct { + Include map[string]string + Exclude map[string]string + } +) + +// UnmarshalYAML unmarshals the constraint. +func (c *Constraint) UnmarshalYAML(unmarshal func(interface{}) error) error { + var out1 = struct { + Include StringSlice + Exclude StringSlice + }{} + + var out2 StringSlice + + unmarshal(&out1) + unmarshal(&out2) + + c.Exclude = out1.Exclude + c.Include = append( + out1.Include, + out2..., + ) + return nil +} + +// UnmarshalYAML unmarshals the constraint map. +func (c *ConstraintMap) UnmarshalYAML(unmarshal func(interface{}) error) error { + out1 := struct { + Include map[string]string + Exclude map[string]string + }{ + Include: map[string]string{}, + Exclude: map[string]string{}, + } + + out2 := map[string]string{} + + unmarshal(&out1) + unmarshal(&out2) + + c.Include = out1.Include + c.Exclude = out1.Exclude + for k, v := range out2 { + c.Include[k] = v + } + return nil +} diff --git a/yaml/converter/legacy/internal/container.go b/yaml/converter/legacy/internal/container.go new file mode 100644 index 0000000..167667d --- /dev/null +++ b/yaml/converter/legacy/internal/container.go @@ -0,0 +1,58 @@ +package yaml + +import ( + "fmt" + + "gopkg.in/yaml.v2" +) + +type ( + // Containers represents an ordered list of containers. + Containers struct { + Containers []*Container + } + + // Container represents a Docker container. + Container struct { + Command StringSlice `yaml:"command,omitempty"` + Commands StringSlice `yaml:"commands,omitempty"` + Detached bool `yaml:"detach,omitempty"` + Devices []string `yaml:"devices,omitempty"` + ErrIgnore bool `yaml:"allow_failure,omitempty"` + Tmpfs []string `yaml:"tmpfs,omitempty"` + DNS StringSlice `yaml:"dns,omitempty"` + DNSSearch StringSlice `yaml:"dns_search,omitempty"` + Entrypoint StringSlice `yaml:"entrypoint,omitempty"` + Environment SliceMap `yaml:"environment,omitempty"` + ExtraHosts []string `yaml:"extra_hosts,omitempty"` + Image string `yaml:"image,omitempty"` + Name string `yaml:"name,omitempty"` + Privileged bool `yaml:"privileged,omitempty"` + Pull bool `yaml:"pull,omitempty"` + Shell string `yaml:"shell,omitempty"` + Volumes []*Volume `yaml:"volumes,omitempty"` + Secrets Secrets `yaml:"secrets,omitempty"` + Constraints Constraints `yaml:"when,omitempty"` + Vargs map[string]interface{} `yaml:",inline"` + } +) + +// UnmarshalYAML implements the Unmarshaller interface. +func (c *Containers) UnmarshalYAML(unmarshal func(interface{}) error) error { + slice := yaml.MapSlice{} + if err := unmarshal(&slice); err != nil { + return err + } + + for _, s := range slice { + container := Container{} + out, _ := yaml.Marshal(s.Value) + + if err := yaml.Unmarshal(out, &container); err != nil { + return err + } + container.Name = fmt.Sprintf("%v", s.Key) + c.Containers = append(c.Containers, &container) + } + return nil +} diff --git a/yaml/converter/legacy/internal/container_test.go b/yaml/converter/legacy/internal/container_test.go new file mode 100644 index 0000000..72f5087 --- /dev/null +++ b/yaml/converter/legacy/internal/container_test.go @@ -0,0 +1 @@ +package yaml diff --git a/yaml/converter/legacy/internal/secret.go b/yaml/converter/legacy/internal/secret.go new file mode 100644 index 0000000..cac8acc --- /dev/null +++ b/yaml/converter/legacy/internal/secret.go @@ -0,0 +1,30 @@ +package yaml + +type ( + // Secrets represents a list of container secrets. + Secrets struct { + Secrets []*Secret + } + + // Secret represents a container secret. + Secret struct { + Source string + Target string + } +) + +// UnmarshalYAML implements the Unmarshaller interface. +func (s *Secrets) UnmarshalYAML(unmarshal func(interface{}) error) error { + var strslice []string + err := unmarshal(&strslice) + if err == nil { + for _, str := range strslice { + s.Secrets = append(s.Secrets, &Secret{ + Source: str, + Target: str, + }) + } + return nil + } + return unmarshal(&s.Secrets) +} diff --git a/yaml/converter/legacy/internal/secret_test.go b/yaml/converter/legacy/internal/secret_test.go new file mode 100644 index 0000000..6e4f844 --- /dev/null +++ b/yaml/converter/legacy/internal/secret_test.go @@ -0,0 +1,62 @@ +package yaml + +import ( + "reflect" + "testing" + + "gopkg.in/yaml.v2" +) + +func TestUnmarshalSecrets(t *testing.T) { + testdata := []struct { + from string + want []*Secret + }{ + { + from: "[ mysql_username, mysql_password]", + want: []*Secret{ + { + Source: "mysql_username", + Target: "mysql_username", + }, + { + Source: "mysql_password", + Target: "mysql_password", + }, + }, + }, + { + from: "[ { source: mysql_prod_username, target: mysql_username } ]", + want: []*Secret{ + { + Source: "mysql_prod_username", + Target: "mysql_username", + }, + }, + }, + { + from: "[ { source: mysql_prod_username, target: mysql_username }, { source: redis_username, target: redis_username } ]", + want: []*Secret{ + { + Source: "mysql_prod_username", + Target: "mysql_username", + }, + { + Source: "redis_username", + Target: "redis_username", + }, + }, + }, + } + + for _, test := range testdata { + in := []byte(test.from) + got := Secrets{} + err := yaml.Unmarshal(in, &got) + if err != nil { + t.Error(err) + } else if !reflect.DeepEqual(test.want, got.Secrets) { + t.Errorf("got secret %v want %v", got.Secrets, test.want) + } + } +} diff --git a/yaml/converter/legacy/internal/slice_map.go b/yaml/converter/legacy/internal/slice_map.go new file mode 100644 index 0000000..2d7737a --- /dev/null +++ b/yaml/converter/legacy/internal/slice_map.go @@ -0,0 +1,32 @@ +package yaml + +import "strings" + +// SliceMap represents a slice or map of key pairs. +type SliceMap struct { + Map map[string]string +} + +// UnmarshalYAML implements custom Yaml unmarshaling. +func (s *SliceMap) UnmarshalYAML(unmarshal func(interface{}) error) error { + s.Map = map[string]string{} + err := unmarshal(&s.Map) + if err == nil { + return nil + } + + var slice []string + err = unmarshal(&slice) + if err != nil { + return err + } + for _, v := range slice { + parts := strings.SplitN(v, "=", 2) + if len(parts) == 2 { + key := parts[0] + val := parts[1] + s.Map[key] = val + } + } + return nil +} diff --git a/yaml/converter/legacy/internal/slice_map_test.go b/yaml/converter/legacy/internal/slice_map_test.go new file mode 100644 index 0000000..b4f4af3 --- /dev/null +++ b/yaml/converter/legacy/internal/slice_map_test.go @@ -0,0 +1,41 @@ +package yaml + +import ( + "reflect" + "testing" + + "gopkg.in/yaml.v2" +) + +func TestMapSlice(t *testing.T) { + var tests = []struct { + yaml string + want map[string]string + }{ + { + yaml: "[ foo=bar, baz=qux ]", + want: map[string]string{"foo": "bar", "baz": "qux"}, + }, + { + yaml: "{ foo: bar, baz: qux }", + want: map[string]string{"foo": "bar", "baz": "qux"}, + }, + } + + for _, test := range tests { + var got SliceMap + + if err := yaml.Unmarshal([]byte(test.yaml), &got); err != nil { + t.Error(err) + } + + if !reflect.DeepEqual(got.Map, test.want) { + t.Errorf("Got map %v want %v", got, test.want) + } + } + + var got SliceMap + if err := yaml.Unmarshal([]byte("1"), &got); err == nil { + t.Errorf("Want error unmarshaling invalid map value.") + } +} diff --git a/yaml/converter/legacy/internal/string_slice.go b/yaml/converter/legacy/internal/string_slice.go new file mode 100644 index 0000000..f4d1b6e --- /dev/null +++ b/yaml/converter/legacy/internal/string_slice.go @@ -0,0 +1,20 @@ +package yaml + +// StringSlice represents a slice of strings or a string. +type StringSlice []string + +// UnmarshalYAML implements the Unmarshaller interface. +func (s *StringSlice) UnmarshalYAML(unmarshal func(interface{}) error) error { + var stringType string + if err := unmarshal(&stringType); err == nil { + *s = []string{stringType} + return nil + } + + var sliceType []string + if err := unmarshal(&sliceType); err != nil { + return err + } + *s = sliceType + return nil +} diff --git a/yaml/converter/legacy/internal/string_slice_test.go b/yaml/converter/legacy/internal/string_slice_test.go new file mode 100644 index 0000000..e5e712e --- /dev/null +++ b/yaml/converter/legacy/internal/string_slice_test.go @@ -0,0 +1,45 @@ +package yaml + +import ( + "reflect" + "testing" + + "gopkg.in/yaml.v2" +) + +func TestStringSlice(t *testing.T) { + var tests = []struct { + yaml string + want []string + }{ + { + yaml: "hello world", + want: []string{"hello world"}, + }, + { + yaml: "[ hello, world ]", + want: []string{"hello", "world"}, + }, + { + yaml: "42", + want: []string{"42"}, + }, + } + + for _, test := range tests { + var got StringSlice + + if err := yaml.Unmarshal([]byte(test.yaml), &got); err != nil { + t.Error(err) + } + + if !reflect.DeepEqual([]string(got), test.want) { + t.Errorf("Got slice %v want %v", got, test.want) + } + } + + var got StringSlice + if err := yaml.Unmarshal([]byte("{}"), &got); err == nil { + t.Errorf("Want error unmarshaling invalid string or slice value.") + } +} diff --git a/yaml/converter/legacy/internal/testdata/simple.yml b/yaml/converter/legacy/internal/testdata/simple.yml new file mode 100644 index 0000000..d76856d --- /dev/null +++ b/yaml/converter/legacy/internal/testdata/simple.yml @@ -0,0 +1,48 @@ +workspace: + base: /go + path: src/github.com/octocat/hello-world + +pipeline: + build: + image: golang + commands: + - go get + - go build + volumes: + - /tmp/go:/go/bin + environment: + - GOOS=linux + - GOARCH=amd64 + + test: + image: golang:latest + volumes: + - /tmp/go:/go/bin + commands: + - go test -v + + docker: + image: plugins/docker + secrets: + - docker_username + - docker_password + repo: octocat/hello-world + when: + branch: master + + slack: + image: plugins/slack + secrets: + - source: token + target: slack_token + channel: general + +services: + database: + image: mysql + environment: + MYSQL_USERNAME: foo + MYSQL_PASSWORD: bar + +branches: +- master \ No newline at end of file diff --git a/yaml/converter/legacy/internal/testdata/simple.yml.golden b/yaml/converter/legacy/internal/testdata/simple.yml.golden new file mode 100644 index 0000000..dbe1583 --- /dev/null +++ b/yaml/converter/legacy/internal/testdata/simple.yml.golden @@ -0,0 +1,76 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +workspace: + base: /go + path: src/github.com/octocat/hello-world + +steps: +- name: build + pull: default + image: golang + commands: + - go get + - go build + environment: + GOARCH: amd64 + GOOS: linux + volumes: + - name: 2f746d702f676f + path: /go/bin + +- name: test + pull: default + image: golang:latest + commands: + - go test -v + volumes: + - name: 2f746d702f676f + path: /go/bin + +- name: docker + pull: default + image: plugins/docker + settings: + repo: octocat/hello-world + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + when: + branch: + - master + +- name: slack + pull: default + image: plugins/slack + settings: + channel: general + environment: + SLACK_TOKEN: + from_secret: token + +services: +- name: database + pull: default + image: mysql + environment: + MYSQL_PASSWORD: bar + MYSQL_USERNAME: foo + +volumes: +- name: 2f746d702f676f + host: + path: /tmp/go + +trigger: + branch: + - master + +... diff --git a/yaml/converter/legacy/internal/testdata/vault_1.yml b/yaml/converter/legacy/internal/testdata/vault_1.yml new file mode 100644 index 0000000..be6db8c --- /dev/null +++ b/yaml/converter/legacy/internal/testdata/vault_1.yml @@ -0,0 +1,16 @@ +pipeline: + docker: + image: plugins/docker + secrets: [ docker_username, docker_password ] + repo: octocat/hello-world + +secrets: + docker_username: + driver: vault + driver_opts: + path: secret/docker/username + docker_password: + driver: vault + driver_opts: + path: secret/docker + key: password diff --git a/yaml/converter/legacy/internal/testdata/vault_1.yml.golden b/yaml/converter/legacy/internal/testdata/vault_1.yml.golden new file mode 100644 index 0000000..22a7925 --- /dev/null +++ b/yaml/converter/legacy/internal/testdata/vault_1.yml.golden @@ -0,0 +1,31 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: docker + pull: default + image: plugins/docker + settings: + repo: octocat/hello-world + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + +--- +kind: secret +type: general +external_data: + docker_password: + path: secret/docker + name: password + docker_username: + path: secret/docker/username + +... diff --git a/yaml/converter/legacy/internal/testdata/vault_2.yml b/yaml/converter/legacy/internal/testdata/vault_2.yml new file mode 100644 index 0000000..9f607f2 --- /dev/null +++ b/yaml/converter/legacy/internal/testdata/vault_2.yml @@ -0,0 +1,11 @@ +pipeline: + docker: + image: plugins/docker + secrets: [ docker_username, docker_password ] + repo: octocat/hello-world + +secrets: + docker_username: + path: secret/docker/username + docker_password: + path: secret/docker/password diff --git a/yaml/converter/legacy/internal/testdata/vault_2.yml.golden b/yaml/converter/legacy/internal/testdata/vault_2.yml.golden new file mode 100644 index 0000000..e0dc567 --- /dev/null +++ b/yaml/converter/legacy/internal/testdata/vault_2.yml.golden @@ -0,0 +1,30 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: docker + pull: default + image: plugins/docker + settings: + repo: octocat/hello-world + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + +--- +kind: secret +type: general +external_data: + docker_password: + path: secret/docker/password + docker_username: + path: secret/docker/username + +... diff --git a/yaml/converter/legacy/internal/testdata/vault_3.yml b/yaml/converter/legacy/internal/testdata/vault_3.yml new file mode 100644 index 0000000..0276f34 --- /dev/null +++ b/yaml/converter/legacy/internal/testdata/vault_3.yml @@ -0,0 +1,11 @@ +pipeline: + docker: + image: plugins/docker + secrets: [ docker_username, docker_password ] + repo: octocat/hello-world + +secrets: + docker_username: + vault: secret/docker/username + docker_password: + vault: secret/docker/password diff --git a/yaml/converter/legacy/internal/testdata/vault_3.yml.golden b/yaml/converter/legacy/internal/testdata/vault_3.yml.golden new file mode 100644 index 0000000..e0dc567 --- /dev/null +++ b/yaml/converter/legacy/internal/testdata/vault_3.yml.golden @@ -0,0 +1,30 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: docker + pull: default + image: plugins/docker + settings: + repo: octocat/hello-world + environment: + DOCKER_PASSWORD: + from_secret: docker_password + DOCKER_USERNAME: + from_secret: docker_username + +--- +kind: secret +type: general +external_data: + docker_password: + path: secret/docker/password + docker_username: + path: secret/docker/username + +... diff --git a/yaml/converter/legacy/internal/volume.go b/yaml/converter/legacy/internal/volume.go new file mode 100644 index 0000000..024bfe3 --- /dev/null +++ b/yaml/converter/legacy/internal/volume.go @@ -0,0 +1,29 @@ +package yaml + +import "strings" + +// Volume represent a container volume. +type Volume struct { + Source string + Destination string + ReadOnly bool +} + +// UnmarshalYAML implements the Unmarshaller interface. +func (v *Volume) UnmarshalYAML(unmarshal func(interface{}) error) error { + var stringType string + if err := unmarshal(&stringType); err != nil { + return err + } + parts := strings.SplitN(stringType, ":", 3) + switch { + case len(parts) == 2: + v.Source = parts[0] + v.Destination = parts[1] + case len(parts) == 3: + v.Source = parts[0] + v.Destination = parts[1] + v.ReadOnly = parts[2] == "ro" + } + return nil +} diff --git a/yaml/converter/legacy/internal/volume_test.go b/yaml/converter/legacy/internal/volume_test.go new file mode 100644 index 0000000..7f316ed --- /dev/null +++ b/yaml/converter/legacy/internal/volume_test.go @@ -0,0 +1,43 @@ +package yaml + +import ( + "reflect" + "testing" + + "gopkg.in/yaml.v2" +) + +func TestVolume(t *testing.T) { + var tests = []struct { + yaml string + want Volume + }{ + { + yaml: "/opt/data:/var/lib/mysql", + want: Volume{Source: "/opt/data", Destination: "/var/lib/mysql"}, + }, + { + yaml: "/opt/data:/var/lib/mysql:ro", + want: Volume{Source: "/opt/data", Destination: "/var/lib/mysql", ReadOnly: true}, + }, + { + yaml: "/opt/data:/var/lib/mysql", + want: Volume{Source: "/opt/data", Destination: "/var/lib/mysql", ReadOnly: false}, + }, + } + + for _, test := range tests { + got := Volume{} + if err := yaml.Unmarshal([]byte(test.yaml), &got); err != nil { + t.Errorf("got error unmarshaling volume %q", test.yaml) + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("got volume %v want %v", got, test.want) + } + } + + var got Volume + if err := yaml.Unmarshal([]byte("{}"), &got); err == nil { + t.Errorf("Want error unmarshaling invalid volume string.") + } +} diff --git a/yaml/converter/legacy/match.go b/yaml/converter/legacy/match.go new file mode 100644 index 0000000..8b579e8 --- /dev/null +++ b/yaml/converter/legacy/match.go @@ -0,0 +1,14 @@ +package legacy + +import ( + "regexp" +) + +var re = regexp.MustCompile(`(?m)^pipeline:(\s+)?$`) + +// Match returns true if the yaml configuration file +// is legacy and requires converstion. +func Match(b []byte) bool { + matches := re.FindAll(b, -1) + return len(matches) != 0 +} diff --git a/yaml/converter/legacy/match_test.go b/yaml/converter/legacy/match_test.go new file mode 100644 index 0000000..8231e1d --- /dev/null +++ b/yaml/converter/legacy/match_test.go @@ -0,0 +1,33 @@ +package legacy + +import "testing" + +func TestMatch(t *testing.T) { + tests := []struct { + config string + result bool + }{ + { + config: "pipeline:\n build:\n image: golang:1.11", + result: true, + }, + { + config: "\n\npipeline:\n", + result: true, + }, + { + config: "\n\npipeline: \n", + result: true, + }, + { + config: "---\nkind: pipeline\n", + result: false, + }, + } + for i, test := range tests { + b := []byte(test.config) + if got, want := Match(b), test.result; got != want { + t.Errorf("Want IsLegacyBytes %v at index %d,", want, i) + } + } +} diff --git a/yaml/converter/legacy/testdata/legacy.yml b/yaml/converter/legacy/testdata/legacy.yml new file mode 100644 index 0000000..a857893 --- /dev/null +++ b/yaml/converter/legacy/testdata/legacy.yml @@ -0,0 +1,7 @@ + +pipeline: + build: + image: golang:1.11 + commands: + - echo foo + - echo bar diff --git a/yaml/cron.go b/yaml/cron.go new file mode 100644 index 0000000..03a0de5 --- /dev/null +++ b/yaml/cron.go @@ -0,0 +1,44 @@ +package yaml + +import "errors" + +type ( + // Cron is a resource that defines a cron job, used + // to execute pipelines at scheduled intervals. + Cron struct { + Version string `json:"version,omitempty"` + Kind string `json:"kind,omitempty"` + Type string `json:"type,omitempty"` + Name string `json:"name,omitempty"` + + Spec CronSpec `json:"spec,omitempty"` + } + + // CronSpec defines the cron job. + CronSpec struct { + Schedule string `json:"schedule,omitempty"` + Branch string `json:"branch,omitempty"` + Deploy CronDeployment `json:"deployment,omitempty" yaml:"deployment"` + } + + // CronDeployment defines a cron job deployment. + CronDeployment struct { + Target string `json:"target,omitempty"` + } +) + +// GetVersion returns the resource version. +func (c *Cron) GetVersion() string { return c.Version } + +// GetKind returns the resource kind. +func (c *Cron) GetKind() string { return c.Kind } + +// Validate returns an error if the cron is invalid. +func (c Cron) Validate() error { + switch { + case c.Spec.Branch == "": + return errors.New("yaml: invalid cron branch") + default: + return nil + } +} diff --git a/yaml/cron_test.go b/yaml/cron_test.go new file mode 100644 index 0000000..e516ac7 --- /dev/null +++ b/yaml/cron_test.go @@ -0,0 +1,28 @@ +package yaml + +import "testing" + +func TestCronUnmarshal(t *testing.T) { + diff, err := diff("testdata/cron.yml") + if err != nil { + t.Error(err) + } + if diff != "" { + t.Error("Failed to parse cron") + t.Log(diff) + } +} + +func TestCronValidate(t *testing.T) { + cron := new(Cron) + cron.Spec.Branch = "master" + if err := cron.Validate(); err != nil { + t.Error(err) + return + } + + cron.Spec.Branch = "" + if err := cron.Validate(); err == nil { + t.Errorf("Expect invalid cron error") + } +} diff --git a/yaml/env.go b/yaml/env.go new file mode 100644 index 0000000..653306f --- /dev/null +++ b/yaml/env.go @@ -0,0 +1,30 @@ +package yaml + +type ( + // Variable represents an environment variable that + // can be defined as a string literal or as a reference + // to a secret. + Variable struct { + Value string `json:"value,omitempty"` + Secret string `json:"from_secret,omitempty" yaml:"from_secret"` + } + + // variable is a tempoary type used to unmarshal + // variables with references to secrets. + variable struct { + Value string + Secret string `yaml:"from_secret"` + } +) + +// UnmarshalYAML implements yaml unmarshalling. +func (v *Variable) UnmarshalYAML(unmarshal func(interface{}) error) error { + d := new(variable) + err := unmarshal(&d.Value) + if err != nil { + err = unmarshal(d) + } + v.Value = d.Value + v.Secret = d.Secret + return err +} diff --git a/yaml/env_test.go b/yaml/env_test.go new file mode 100644 index 0000000..910d610 --- /dev/null +++ b/yaml/env_test.go @@ -0,0 +1,39 @@ +package yaml + +import ( + "testing" + + "gopkg.in/yaml.v2" +) + +func TestEnv(t *testing.T) { + tests := []struct { + yaml string + value string + from string + }{ + { + yaml: "bar", + value: "bar", + }, + { + yaml: "from_secret: username", + from: "username", + }, + } + for _, test := range tests { + in := []byte(test.yaml) + out := new(Variable) + err := yaml.Unmarshal(in, out) + if err != nil { + t.Error(err) + return + } + if got, want := out.Value, test.value; got != want { + t.Errorf("Want variable value %q, got %q", want, got) + } + if got, want := out.Secret, test.from; got != want { + t.Errorf("Want variable from_secret %q, got %q", want, got) + } + } +} diff --git a/yaml/linter/config.go b/yaml/linter/config.go new file mode 100644 index 0000000..f388f34 --- /dev/null +++ b/yaml/linter/config.go @@ -0,0 +1,65 @@ +package linter + +import ( + "errors" + + "github.com/drone/drone-yaml/yaml" +) + +// ErrDuplicatePipelineName is returned when two Pipeline +// resources have the same name. +var ErrDuplicatePipelineName = errors.New("linter: duplicate pipeline names") + +// ErrMissingPipelineDependency is returned when a Pipeline +// defines dependencies that are invlid or unknown. +var ErrMissingPipelineDependency = errors.New("linter: invalid or unknown pipeline dependency") + +// ErrCyclicalPipelineDependency is returned when a Pipeline +// defines a cyclical dependency, which would result in an +// infinite execution loop. +var ErrCyclicalPipelineDependency = errors.New("linter: cyclical pipeline dependency detected") + +// ErrPipelineSelfDependency is returned when a Pipeline +// defines a dependency on itself. +var ErrPipelineSelfDependency = errors.New("linter: pipeline cannot have a dependency on itself") + +// Manifest performs lint operations for a manifest. +func Manifest(manifest *yaml.Manifest, trusted bool) error { + return checkPipelines(manifest, trusted) +} + +func checkPipelines(manifest *yaml.Manifest, trusted bool) error { + names := map[string]struct{}{} + for _, resource := range manifest.Resources { + switch v := resource.(type) { + case *yaml.Pipeline: + _, ok := names[v.Name] + if ok { + return ErrDuplicatePipelineName + } + names[v.Name] = struct{}{} + err := checkPipelineDeps(v, names) + if err != nil { + return err + } + err = checkPipeline(v, trusted) + if err != nil { + return err + } + } + } + return nil +} + +func checkPipelineDeps(pipeline *yaml.Pipeline, deps map[string]struct{}) error { + for _, dep := range pipeline.DependsOn { + _, ok := deps[dep] + if !ok { + return ErrMissingPipelineDependency + } + if pipeline.Name == dep { + return ErrPipelineSelfDependency + } + } + return nil +} diff --git a/yaml/linter/config_test.go b/yaml/linter/config_test.go new file mode 100644 index 0000000..6bc4d0b --- /dev/null +++ b/yaml/linter/config_test.go @@ -0,0 +1,79 @@ +package linter + +import ( + "testing" + + "github.com/drone/drone-yaml/yaml" +) + +func TestManifest(t *testing.T) { + tests := []struct { + path string + trusted bool + invalid bool + message string + }{ + { + path: "testdata/simple.yml", + trusted: false, + invalid: false, + }, + { + path: "testdata/invalid_os.yml", + trusted: false, + invalid: true, + message: "linter: unsupported os: openbsd", + }, + { + path: "testdata/invalid_arch.yml", + trusted: false, + invalid: true, + message: "linter: unsupported architecture: s390x", + }, + { + path: "testdata/duplicate_name.yml", + trusted: false, + invalid: true, + message: "linter: duplicate pipeline names", + }, + { + path: "testdata/missing_dep.yml", + trusted: false, + invalid: true, + message: "linter: invalid or unknown pipeline dependency", + }, + } + for _, test := range tests { + t.Run(test.path, func(t *testing.T) { + manifest, err := yaml.ParseFile(test.path) + if err != nil { + t.Logf("yaml: %s", test.path) + t.Error(err) + return + } + + err = Manifest(manifest, test.trusted) + if err == nil && test.invalid == true { + t.Logf("yaml: %s", test.path) + t.Errorf("Expect lint error") + return + } + + if err != nil && test.invalid == false { + t.Logf("yaml: %s", test.path) + t.Errorf("Expect lint error is nil, got %s", err) + return + } + + if err == nil { + return + } + + if got, want := err.Error(), test.message; got != want { + t.Logf("yaml: %s", test.path) + t.Errorf("Want message %q, got %q", want, got) + return + } + }) + } +} diff --git a/yaml/linter/linter.go b/yaml/linter/linter.go new file mode 100644 index 0000000..a096c87 --- /dev/null +++ b/yaml/linter/linter.go @@ -0,0 +1,215 @@ +package linter + +import ( + "errors" + "fmt" + + "github.com/drone/drone-yaml/yaml" +) + +var os = map[string]struct{}{ + "linux": struct{}{}, + "windows": struct{}{}, +} + +var arch = map[string]struct{}{ + "arm": struct{}{}, + "arm64": struct{}{}, + "amd64": struct{}{}, +} + +// ErrDuplicateStepName is returned when two Pipeline steps +// have the same name. +var ErrDuplicateStepName = errors.New("linter: duplicate step names") + +// ErrMissingDependency is returned when a Pipeline step +// defines dependencies that are invlid or unknown. +var ErrMissingDependency = errors.New("linter: invalid or unknown step dependency") + +// ErrCyclicalDependency is returned when a Pipeline step +// defines a cyclical dependency, which would result in an +// infinite execution loop. +var ErrCyclicalDependency = errors.New("linter: cyclical step dependency detected") + +// Lint performs lint operations for a resource. +func Lint(resource yaml.Resource, trusted bool) error { + switch v := resource.(type) { + case *yaml.Cron: + return v.Validate() + case *yaml.Pipeline: + return checkPipeline(v, trusted) + case *yaml.Secret: + return v.Validate() + case *yaml.Registry: + return v.Validate() + case *yaml.Signature: + return v.Validate() + default: + return nil + } +} + +func checkPipeline(pipeline *yaml.Pipeline, trusted bool) error { + err := checkVolumes(pipeline, trusted) + if err != nil { + return err + } + err = checkPlatform(pipeline.Platform) + if err != nil { + return err + } + names := map[string]struct{}{} + if pipeline.Clone.Disable == false { + names["clone"] = struct{}{} + } + for _, container := range pipeline.Steps { + _, ok := names[container.Name] + if ok { + return ErrDuplicateStepName + } + names[container.Name] = struct{}{} + + err := checkContainer(container, trusted) + if err != nil { + return err + } + + err = checkDeps(container, names) + if err != nil { + return err + } + } + for _, container := range pipeline.Services { + _, ok := names[container.Name] + if ok { + return ErrDuplicateStepName + } + names[container.Name] = struct{}{} + + err := checkContainer(container, trusted) + if err != nil { + return err + } + } + return nil +} + +func checkPlatform(platform yaml.Platform) error { + if v := platform.OS; v != "" { + _, ok := os[v] + if !ok { + return fmt.Errorf("linter: unsupported os: %s", v) + } + } + if v := platform.Arch; v != "" { + _, ok := arch[v] + if !ok { + return fmt.Errorf("linter: unsupported architecture: %s", v) + } + } + return nil +} + +func checkContainer(container *yaml.Container, trusted bool) error { + err := checkPorts(container.Ports, trusted) + if err != nil { + return err + } + if container.Build == nil && container.Image == "" { + return errors.New("linter: invalid or missing image") + } + if container.Build != nil && container.Build.Image == "" { + return errors.New("linter: invalid or missing build image") + } + if container.Name == "" { + return errors.New("linter: invalid or missing name") + } + if trusted == false && container.Privileged { + return errors.New("linter: untrusted repositories cannot enable privileged mode") + } + if trusted == false && len(container.Devices) > 0 { + return errors.New("linter: untrusted repositories cannot mount devices") + } + if trusted == false && len(container.DNS) > 0 { + return errors.New("linter: untrusted repositories cannot configure dns") + } + if trusted == false && len(container.DNSSearch) > 0 { + return errors.New("linter: untrusted repositories cannot configure dns_search") + } + if trusted == false && len(container.ExtraHosts) > 0 { + return errors.New("linter: untrusted repositories cannot configure extra_hosts") + } + for _, mount := range container.Volumes { + switch mount.Name { + case "workspace", "_workspace", "_docker_socket": + return fmt.Errorf("linter: invalid volume name: %s", mount.Name) + } + } + return nil +} + +func checkPorts(ports []*yaml.Port, trusted bool) error { + for _, port := range ports { + err := checkPort(port, trusted) + if err != nil { + return err + } + } + return nil +} + +func checkPort(port *yaml.Port, trusted bool) error { + if trusted == false && port.Host != 0 { + return errors.New("linter: untrusted repositories cannot map to a host port") + } + return nil +} + +func checkVolumes(pipeline *yaml.Pipeline, trusted bool) error { + for _, volume := range pipeline.Volumes { + if volume.EmptyDir != nil { + err := checkEmptyDirVolume(volume.EmptyDir, trusted) + if err != nil { + return err + } + } + if volume.HostPath != nil { + err := checkHostPathVolume(volume.HostPath, trusted) + if err != nil { + return err + } + } + switch volume.Name { + case "workspace", "_workspace", "_docker_socket": + return fmt.Errorf("linter: invalid volume name: %s", volume.Name) + } + } + return nil +} + +func checkHostPathVolume(volume *yaml.VolumeHostPath, trusted bool) error { + if trusted == false { + return errors.New("linter: untrusted repositories cannot mount host volumes") + } + return nil +} + +func checkEmptyDirVolume(volume *yaml.VolumeEmptyDir, trusted bool) error { + if trusted == false && volume.Medium == "memory" { + return errors.New("linter: untrusted repositories cannot mount in-memory volumes") + } + return nil +} + +func checkDeps(container *yaml.Container, deps map[string]struct{}) error { + for _, dep := range container.DependsOn { + _, ok := deps[dep] + if !ok { + return ErrMissingDependency + } + if container.Name == dep { + return ErrCyclicalDependency + } + } + return nil +} diff --git a/yaml/linter/linter_test.go b/yaml/linter/linter_test.go new file mode 100644 index 0000000..6533710 --- /dev/null +++ b/yaml/linter/linter_test.go @@ -0,0 +1,246 @@ +package linter + +import ( + "path" + "testing" + + "github.com/drone/drone-yaml/yaml" +) + +func TestLint(t *testing.T) { + tests := []struct { + path string + trusted bool + invalid bool + message string + }{ + { + path: "testdata/simple.yml", + trusted: false, + invalid: false, + }, + { + path: "testdata/invalid_os.yml", + trusted: false, + invalid: true, + message: "linter: unsupported os: openbsd", + }, + { + path: "testdata/invalid_arch.yml", + trusted: false, + invalid: true, + message: "linter: unsupported architecture: s390x", + }, + { + path: "testdata/missing_build_image.yml", + invalid: true, + message: "linter: invalid or missing build image", + }, + { + path: "testdata/missing_image.yml", + invalid: true, + message: "linter: invalid or missing image", + }, + { + path: "testdata/missing_name.yml", + invalid: true, + message: "linter: invalid or missing name", + }, + // user should not use reserved volume names. + { + path: "testdata/volume_invalid_name.yml", + trusted: false, + invalid: true, + message: "linter: invalid volume name: _workspace", + }, + { + path: "testdata/pipeline_volume_invalid_name.yml", + trusted: false, + invalid: true, + message: "linter: invalid volume name: _docker_socket", + }, + // user should not be able to mount host path + // volumes unless the repository is trusted. + { + path: "testdata/volume_host_path.yml", + trusted: false, + invalid: true, + message: "linter: untrusted repositories cannot mount host volumes", + }, + { + path: "testdata/volume_host_path.yml", + trusted: true, + invalid: false, + }, + // user should be able to mount emptyDir volumes + // where no medium is specified. + { + path: "testdata/volume_empty_dir.yml", + trusted: false, + invalid: false, + }, + // user should not be able to mount in-memory + // emptyDir volumes unless the repository is + // trusted. + { + path: "testdata/volume_empty_dir_memory.yml", + trusted: false, + invalid: true, + message: "linter: untrusted repositories cannot mount in-memory volumes", + }, + { + path: "testdata/volume_empty_dir_memory.yml", + trusted: true, + invalid: false, + }, + // user should not be able to bind to host ports + // or IP addresses unless the repository is trusted. + { + path: "testdata/service_port_host.yml", + trusted: false, + invalid: true, + message: "linter: untrusted repositories cannot map to a host port", + }, + { + path: "testdata/service_port_host.yml", + trusted: true, + invalid: false, + }, + { + path: "testdata/pipeline_port_host.yml", + trusted: false, + invalid: true, + message: "linter: untrusted repositories cannot map to a host port", + }, + { + path: "testdata/pipeline_port_host.yml", + trusted: true, + invalid: false, + }, + // user should not be able to mount devices unless + // the repository is trusted. + { + path: "testdata/service_device.yml", + trusted: false, + invalid: true, + message: "linter: untrusted repositories cannot mount devices", + }, + { + path: "testdata/service_device.yml", + trusted: true, + invalid: false, + }, + { + path: "testdata/pipeline_device.yml", + trusted: false, + invalid: true, + message: "linter: untrusted repositories cannot mount devices", + }, + { + path: "testdata/pipeline_device.yml", + trusted: true, + invalid: false, + }, + // user should not be able to set the securityContext + // unless the repository is trusted. + { + path: "testdata/pipeline_privileged.yml", + trusted: false, + invalid: true, + message: "linter: untrusted repositories cannot enable privileged mode", + }, + { + path: "testdata/pipeline_privileged.yml", + trusted: true, + invalid: false, + }, + // user should not be able to set dns, dns_search or + // extra_hosts unless the repository is trusted. + { + path: "testdata/pipeline_dns.yml", + trusted: false, + invalid: true, + message: "linter: untrusted repositories cannot configure dns", + }, + { + path: "testdata/pipeline_dns.yml", + trusted: true, + invalid: false, + }, + { + path: "testdata/pipeline_dns_search.yml", + trusted: false, + invalid: true, + message: "linter: untrusted repositories cannot configure dns_search", + }, + { + path: "testdata/pipeline_dns_search.yml", + trusted: true, + invalid: false, + }, + { + path: "testdata/pipeline_extra_hosts.yml", + trusted: false, + invalid: true, + message: "linter: untrusted repositories cannot configure extra_hosts", + }, + { + path: "testdata/pipeline_extra_hosts.yml", + trusted: true, + invalid: false, + }, + // user should not be able to use duplicate names + // for steps or services. + { + path: "testdata/duplicate_step.yml", + invalid: true, + message: "linter: duplicate step names", + }, + { + path: "testdata/duplicate_step_service.yml", + invalid: true, + message: "linter: duplicate step names", + }, + } + for _, test := range tests { + name := path.Base(test.path) + if test.trusted { + name = name + "/trusted" + } + t.Run(name, func(t *testing.T) { + resources, err := yaml.ParseFile(test.path) + if err != nil { + t.Logf("yaml: %s", test.path) + t.Logf("trusted: %v", test.trusted) + t.Error(err) + return + } + + err = Lint(resources.Resources[0], test.trusted) + if err == nil && test.invalid == true { + t.Logf("yaml: %s", test.path) + t.Logf("trusted: %v", test.trusted) + t.Errorf("Expect lint error") + return + } + + if err != nil && test.invalid == false { + t.Logf("yaml: %s", test.path) + t.Logf("trusted: %v", test.trusted) + t.Errorf("Expect lint error is nil, got %s", err) + return + } + + if err == nil { + return + } + + if got, want := err.Error(), test.message; got != want { + t.Logf("yaml: %s", test.path) + t.Logf("trusted: %v", test.trusted) + t.Errorf("Want message %q, got %q", want, got) + return + } + }) + } +} diff --git a/yaml/linter/testdata/duplicate_name.yml b/yaml/linter/testdata/duplicate_name.yml new file mode 100644 index 0000000..7311e03 --- /dev/null +++ b/yaml/linter/testdata/duplicate_name.yml @@ -0,0 +1,23 @@ +--- +kind: pipeline +name: default + +steps: +- name: build + image: golang + commands: + - go build + - go test + +--- +kind: pipeline +name: default + +steps: +- name: build + image: golang + commands: + - go build + - go test + +... \ No newline at end of file diff --git a/yaml/linter/testdata/duplicate_step.yml b/yaml/linter/testdata/duplicate_step.yml new file mode 100644 index 0000000..3595548 --- /dev/null +++ b/yaml/linter/testdata/duplicate_step.yml @@ -0,0 +1,16 @@ +--- +kind: pipeline +name: default + +steps: +- name: build + image: golang + commands: + - go build + - go test + +- name: build + image: golang + commands: + - go build + - go test \ No newline at end of file diff --git a/yaml/linter/testdata/duplicate_step_service.yml b/yaml/linter/testdata/duplicate_step_service.yml new file mode 100644 index 0000000..1d0c803 --- /dev/null +++ b/yaml/linter/testdata/duplicate_step_service.yml @@ -0,0 +1,14 @@ +--- +kind: pipeline +name: default + +steps: +- name: test + image: golang + commands: + - go build + - go test + +services: +- name: test + image: redis diff --git a/yaml/linter/testdata/invalid_arch.yml b/yaml/linter/testdata/invalid_arch.yml new file mode 100644 index 0000000..aa1466a --- /dev/null +++ b/yaml/linter/testdata/invalid_arch.yml @@ -0,0 +1,14 @@ +--- +kind: pipeline +name: linux + +platform: + os: linux + arch: s390x + +steps: +- name: build + image: golang + commands: + - go build + - go test diff --git a/yaml/linter/testdata/invalid_os.yml b/yaml/linter/testdata/invalid_os.yml new file mode 100644 index 0000000..c91df74 --- /dev/null +++ b/yaml/linter/testdata/invalid_os.yml @@ -0,0 +1,13 @@ +--- +kind: pipeline +name: linux + +platform: + os: openbsd + +steps: +- name: build + image: golang + commands: + - go build + - go test diff --git a/yaml/linter/testdata/missing_build_image.yml b/yaml/linter/testdata/missing_build_image.yml new file mode 100644 index 0000000..0b5e45a --- /dev/null +++ b/yaml/linter/testdata/missing_build_image.yml @@ -0,0 +1,8 @@ +--- +kind: pipeline +name: linux + +steps: +- name: test + build: {} + diff --git a/yaml/linter/testdata/missing_dep.yml b/yaml/linter/testdata/missing_dep.yml new file mode 100644 index 0000000..4f32e99 --- /dev/null +++ b/yaml/linter/testdata/missing_dep.yml @@ -0,0 +1,34 @@ +--- +kind: pipeline +name: amd64 + +steps: +- name: test + image: golang + commands: + - go build + - go test + +services: +- name: database + image: redis + ports: + - 6379 + +--- +kind: pipeline +name: arm + +platform: + arch: arm + +steps: +- name: test + image: golang + commands: + - go build + - go test + +depends_on: +- foo +... \ No newline at end of file diff --git a/yaml/linter/testdata/missing_image.yml b/yaml/linter/testdata/missing_image.yml new file mode 100644 index 0000000..ffd753c --- /dev/null +++ b/yaml/linter/testdata/missing_image.yml @@ -0,0 +1,9 @@ +--- +kind: pipeline +name: linux + +steps: +- name: test + commands: + - go build + - go test diff --git a/yaml/linter/testdata/missing_name.yml b/yaml/linter/testdata/missing_name.yml new file mode 100644 index 0000000..9ec793d --- /dev/null +++ b/yaml/linter/testdata/missing_name.yml @@ -0,0 +1,9 @@ +--- +kind: pipeline +name: linux + +steps: +- image: golang + commands: + - go build + - go test diff --git a/yaml/linter/testdata/pipeline_device.yml b/yaml/linter/testdata/pipeline_device.yml new file mode 100644 index 0000000..d44ce17 --- /dev/null +++ b/yaml/linter/testdata/pipeline_device.yml @@ -0,0 +1,19 @@ +--- +kind: pipeline +name: linux + +steps: +- name: test + image: golang + commands: + - go build + - go test + devices: + - name: data + path: /dev/xvda + +services: +- name: database + image: redis + ports: + - 6379 diff --git a/yaml/linter/testdata/pipeline_dns.yml b/yaml/linter/testdata/pipeline_dns.yml new file mode 100644 index 0000000..4c2a7c7 --- /dev/null +++ b/yaml/linter/testdata/pipeline_dns.yml @@ -0,0 +1,12 @@ +--- +kind: pipeline +name: linux + +steps: +- name: test + image: golang + commands: + - go build + - go test + dns: + - 8.8.8.8 diff --git a/yaml/linter/testdata/pipeline_dns_search.yml b/yaml/linter/testdata/pipeline_dns_search.yml new file mode 100644 index 0000000..790ec07 --- /dev/null +++ b/yaml/linter/testdata/pipeline_dns_search.yml @@ -0,0 +1,13 @@ +--- +kind: pipeline +name: linux + +steps: +- name: test + image: golang + commands: + - go build + - go test + dns_search: + - dc1.example.com + - dc2.example.com diff --git a/yaml/linter/testdata/pipeline_extra_hosts.yml b/yaml/linter/testdata/pipeline_extra_hosts.yml new file mode 100644 index 0000000..749f805 --- /dev/null +++ b/yaml/linter/testdata/pipeline_extra_hosts.yml @@ -0,0 +1,13 @@ +--- +kind: pipeline +name: linux + +steps: +- name: test + image: golang + commands: + - go build + - go test + extra_hosts: + - "somehost:162.242.195.82" + - "otherhost:50.31.209.229" diff --git a/yaml/linter/testdata/pipeline_port_host.yml b/yaml/linter/testdata/pipeline_port_host.yml new file mode 100644 index 0000000..3a29213 --- /dev/null +++ b/yaml/linter/testdata/pipeline_port_host.yml @@ -0,0 +1,17 @@ +--- +kind: pipeline +name: linux + +steps: +- name: database + image: redis + detach: true + ports: + - port: 6379 + host: 6379 + +- name: test + image: golang + commands: + - go build + - go test diff --git a/yaml/linter/testdata/pipeline_privileged.yml b/yaml/linter/testdata/pipeline_privileged.yml new file mode 100644 index 0000000..be762de --- /dev/null +++ b/yaml/linter/testdata/pipeline_privileged.yml @@ -0,0 +1,17 @@ +--- +kind: pipeline +name: linux + +steps: +- name: test + image: golang + commands: + - go build + - go test + privileged: true + +services: +- name: database + image: redis + ports: + - 6379 diff --git a/yaml/linter/testdata/pipeline_volume_invalid_name.yml b/yaml/linter/testdata/pipeline_volume_invalid_name.yml new file mode 100644 index 0000000..42b3bb6 --- /dev/null +++ b/yaml/linter/testdata/pipeline_volume_invalid_name.yml @@ -0,0 +1,13 @@ +--- +kind: pipeline +name: linux + +steps: +- name: test + image: docker + volumes: + - name: _docker_socket + path: /var/run/docker.sock + commands: + - docker system prune + diff --git a/yaml/linter/testdata/service_device.yml b/yaml/linter/testdata/service_device.yml new file mode 100644 index 0000000..dd01460 --- /dev/null +++ b/yaml/linter/testdata/service_device.yml @@ -0,0 +1,19 @@ +--- +kind: pipeline +name: linux + +steps: +- name: test + image: golang + commands: + - go build + - go test + +services: +- name: database + image: redis + ports: + - 6379 + devices: + - name: data + path: /dev/xvda diff --git a/yaml/linter/testdata/service_port_host.yml b/yaml/linter/testdata/service_port_host.yml new file mode 100644 index 0000000..89c8d2d --- /dev/null +++ b/yaml/linter/testdata/service_port_host.yml @@ -0,0 +1,17 @@ +--- +kind: pipeline +name: linux + +steps: +- name: test + image: golang + commands: + - go build + - go test + +services: +- name: database + image: redis + ports: + - port: 6379 + host: 6379 diff --git a/yaml/linter/testdata/simple.yml b/yaml/linter/testdata/simple.yml new file mode 100644 index 0000000..da386da --- /dev/null +++ b/yaml/linter/testdata/simple.yml @@ -0,0 +1,38 @@ +--- +kind: pipeline +name: amd64 + +steps: +- name: build + image: golang + commands: + - go build + +- name: test + image: golang + commands: + - go test + +services: +- name: database + image: redis + ports: + - 6379 + +--- +kind: pipeline +name: arm + +platform: + arch: arm + +steps: +- name: test + image: golang + commands: + - go build + - go test + +depends_on: +- amd64 +... \ No newline at end of file diff --git a/yaml/linter/testdata/volume_empty_dir.yml b/yaml/linter/testdata/volume_empty_dir.yml new file mode 100644 index 0000000..95d75b6 --- /dev/null +++ b/yaml/linter/testdata/volume_empty_dir.yml @@ -0,0 +1,20 @@ +--- +kind: pipeline +name: linux + +steps: +- name: test + image: golang + commands: + - go build + - go test + +services: +- name: database + image: redis + ports: + - 6379 + +volumes: +- name: vol + temp: {} diff --git a/yaml/linter/testdata/volume_empty_dir_memory.yml b/yaml/linter/testdata/volume_empty_dir_memory.yml new file mode 100644 index 0000000..3deb8ef --- /dev/null +++ b/yaml/linter/testdata/volume_empty_dir_memory.yml @@ -0,0 +1,21 @@ +--- +kind: pipeline +name: linux + +steps: +- name: test + image: golang + commands: + - go build + - go test + +services: +- name: database + image: redis + ports: + - 6379 + +volumes: +- name: vol + temp: + medium: memory diff --git a/yaml/linter/testdata/volume_host_path.yml b/yaml/linter/testdata/volume_host_path.yml new file mode 100644 index 0000000..6dd7962 --- /dev/null +++ b/yaml/linter/testdata/volume_host_path.yml @@ -0,0 +1,21 @@ +--- +kind: pipeline +name: linux + +steps: +- name: test + image: golang + commands: + - go build + - go test + +services: +- name: database + image: redis + ports: + - 6379 + +volumes: +- name: vol + host: + path: /any/path/it/will/be/replaced diff --git a/yaml/linter/testdata/volume_invalid_name.yml b/yaml/linter/testdata/volume_invalid_name.yml new file mode 100644 index 0000000..5628a71 --- /dev/null +++ b/yaml/linter/testdata/volume_invalid_name.yml @@ -0,0 +1,20 @@ +--- +kind: pipeline +name: linux + +steps: +- name: test + image: golang + commands: + - go build + - go test + +services: +- name: database + image: redis + ports: + - 6379 + +volumes: +- name: _workspace + temp: {} diff --git a/yaml/manifest.go b/yaml/manifest.go new file mode 100644 index 0000000..366ef42 --- /dev/null +++ b/yaml/manifest.go @@ -0,0 +1,91 @@ +package yaml + +import ( + "encoding/json" + "errors" +) + +// Resource enums. +const ( + KindCron = "cron" + KindPipeline = "pipeline" + KindRegistry = "registry" + KindSecret = "secret" + KindSignature = "signature" +) + +type ( + // Manifest is a collection of Drone resources. + Manifest struct { + Resources []Resource + } + + // Resource represents a Drone resource. + Resource interface { + // GetVersion returns the resource version. + GetVersion() string + + // GetKind returns the resource kind. + GetKind() string + } + + // RawResource is a raw encoded resource with the + // resource kind and type extracted. + RawResource struct { + Version string + Kind string + Type string + Data []byte `yaml:"-"` + } + + resource struct { + Version string + Kind string `json:"kind"` + Type string `json:"type"` + } +) + +// UnmarshalJSON implement the json.Unmarshaler. +func (m *Manifest) UnmarshalJSON(b []byte) error { + messages := []json.RawMessage{} + err := json.Unmarshal(b, &messages) + if err != nil { + return err + } + for _, message := range messages { + res := new(resource) + err := json.Unmarshal(message, res) + if err != nil { + return err + } + var obj Resource + switch res.Kind { + case "cron": + obj = new(Cron) + case "secret": + obj = new(Secret) + case "signature": + obj = new(Signature) + case "registry": + obj = new(Registry) + default: + obj = new(Pipeline) + } + err = json.Unmarshal(message, obj) + if err != nil { + return err + } + m.Resources = append(m.Resources, obj) + } + return nil +} + +// MarshalJSON implement the json.Marshaler. +func (m *Manifest) MarshalJSON() ([]byte, error) { + return json.Marshal(m.Resources) +} + +// MarshalYAML implement the yaml.Marshaler. +func (m *Manifest) MarshalYAML() (interface{}, error) { + return nil, errors.New("yaml: marshal not implemented") +} diff --git a/yaml/manifest_test.go b/yaml/manifest_test.go new file mode 100644 index 0000000..24dd549 --- /dev/null +++ b/yaml/manifest_test.go @@ -0,0 +1,40 @@ +package yaml + +import ( + "encoding/json" + "io/ioutil" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestManifestUnmarshal(t *testing.T) { + diff, err := diff("testdata/manifest.yml") + if err != nil { + t.Error(err) + } + if diff != "" { + t.Error("Failed to parse manifest with multiple entries") + t.Log(diff) + } +} + +func diff(file string) (string, error) { + a, err := ParseFile(file) + if err != nil { + return "", err + } + // enc := json.NewEncoder(os.Stdout) + // enc.SetIndent("", " ") + // enc.Encode(a) + d, err := ioutil.ReadFile(file + ".golden") + if err != nil { + return "", err + } + b := new(Manifest) + err = json.Unmarshal(d, b) + if err != nil { + return "", err + } + return cmp.Diff(a, b), nil +} diff --git a/yaml/param.go b/yaml/param.go new file mode 100644 index 0000000..3ca9de1 --- /dev/null +++ b/yaml/param.go @@ -0,0 +1,31 @@ +package yaml + +type ( + // Parameter represents an configuration parameter that + // can be defined as a literal or as a reference + // to a secret. + Parameter struct { + Value interface{} `json:"value,omitempty"` + Secret string `json:"from_secret,omitempty" yaml:"from_secret"` + } + + // parameter is a tempoary type used to unmarshal + // parameters with references to secrets. + parameter struct { + Secret string `yaml:"from_secret"` + } +) + +// UnmarshalYAML implements yaml unmarshalling. +func (p *Parameter) UnmarshalYAML(unmarshal func(interface{}) error) error { + d := new(parameter) + err := unmarshal(d) + if err == nil && d.Secret != ""{ + p.Secret = d.Secret + return nil + } + var i interface{} + err = unmarshal(&i) + p.Value = i + return err +} diff --git a/yaml/param_test.go b/yaml/param_test.go new file mode 100644 index 0000000..1982ff6 --- /dev/null +++ b/yaml/param_test.go @@ -0,0 +1,39 @@ +package yaml + +import ( + "testing" + + "gopkg.in/yaml.v2" +) + +func TestParam(t *testing.T) { + tests := []struct { + yaml string + value interface{} + from string + }{ + { + yaml: "bar", + value: "bar", + }, + { + yaml: "from_secret: username", + from: "username", + }, + } + for _, test := range tests { + in := []byte(test.yaml) + out := new(Parameter) + err := yaml.Unmarshal(in, out) + if err != nil { + t.Error(err) + return + } + if got, want := out.Value, test.value; got != want { + t.Errorf("Want value %q, got %q", want, got) + } + if got, want := out.Secret, test.from; got != want { + t.Errorf("Want from_secret %q, got %q", want, got) + } + } +} diff --git a/yaml/parse.go b/yaml/parse.go new file mode 100644 index 0000000..48cd0cb --- /dev/null +++ b/yaml/parse.go @@ -0,0 +1,155 @@ +package yaml + +import ( + "bufio" + "bytes" + "io" + "os" + "strings" + + "gopkg.in/yaml.v2" +) + +// Parse parses the configuration from io.Reader r. +func Parse(r io.Reader) (*Manifest, error) { + resources, err := ParseRaw(r) + if err != nil { + return nil, err + } + manifest := new(Manifest) + for _, raw := range resources { + if raw == nil { + continue + } + resource, err := parseRaw(raw) + if err != nil { + return nil, err + } + manifest.Resources = append( + manifest.Resources, + resource, + ) + } + return manifest, nil +} + +// ParseBytes parses the configuration from bytes b. +func ParseBytes(b []byte) (*Manifest, error) { + return Parse( + bytes.NewBuffer(b), + ) +} + +// ParseString parses the configuration from string s. +func ParseString(s string) (*Manifest, error) { + return ParseBytes( + []byte(s), + ) +} + +// ParseFile parses the configuration from path p. +func ParseFile(p string) (*Manifest, error) { + f, err := os.Open(p) + if err != nil { + return nil, err + } + defer f.Close() + return Parse(f) +} + +func parseRaw(r *RawResource) (Resource, error) { + var obj Resource + switch r.Kind { + case "cron": + obj = new(Cron) + case "secret": + obj = new(Secret) + case "signature": + obj = new(Signature) + case "registry": + obj = new(Registry) + default: + obj = new(Pipeline) + } + err := yaml.Unmarshal(r.Data, obj) + return obj, err +} + +// ParseRaw parses the multi-document yaml from the +// io.Reader and returns a slice of raw resources. +func ParseRaw(r io.Reader) ([]*RawResource, error) { + const newline = '\n' + var resources []*RawResource + var resource *RawResource + + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + if isSeparator(line) { + resource = nil + } + if resource == nil { + resource = &RawResource{} + resources = append(resources, resource) + } + if isSeparator(line) { + continue + } + if isTerminator(line) { + break + } + if scanner.Err() == io.EOF { + break + } + resource.Data = append( + resource.Data, + line..., + ) + resource.Data = append( + resource.Data, + newline, + ) + } + for _, resource := range resources { + err := yaml.Unmarshal(resource.Data, resource) + if err != nil { + return nil, err + } + } + return resources, nil +} + +// ParseRawString parses the multi-document yaml from s +// and returns a slice of raw resources. +func ParseRawString(s string) ([]*RawResource, error) { + return ParseRaw( + strings.NewReader(s), + ) +} + +// ParseRawBytes parses the multi-document yaml from b +// and returns a slice of raw resources. +func ParseRawBytes(b []byte) ([]*RawResource, error) { + return ParseRaw( + bytes.NewReader(b), + ) +} + +// ParseRawFile parses the multi-document yaml from path p +// and returns a slice of raw resources. +func ParseRawFile(p string) ([]*RawResource, error) { + f, err := os.Open(p) + if err != nil { + return nil, err + } + defer f.Close() + return ParseRaw(f) +} + +func isSeparator(s string) bool { + return strings.HasPrefix(s, "---") +} + +func isTerminator(s string) bool { + return strings.HasPrefix(s, "...") +} diff --git a/yaml/parse_test.go b/yaml/parse_test.go new file mode 100644 index 0000000..f7b86c1 --- /dev/null +++ b/yaml/parse_test.go @@ -0,0 +1,95 @@ +package yaml + +import ( + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestParseRaw(t *testing.T) { + tests := []struct { + data string + want []*RawResource + }{ + // + // empty document returns nil resources. + // + { + data: "", + want: nil, + }, + // + // single document files. + // + { + data: "kind: pipeline\nfoo: bar", + want: []*RawResource{ + {Kind: "pipeline", Data: []byte("kind: pipeline\nfoo: bar\n")}, + }, + }, + { + data: "kind: pipeline\nfoo: bar\n", + want: []*RawResource{ + {Kind: "pipeline", Data: []byte("kind: pipeline\nfoo: bar\n")}, + }, + }, + { + data: "---\nkind: pipeline\nfoo: bar\n...", + want: []*RawResource{ + {Kind: "pipeline", Data: []byte("kind: pipeline\nfoo: bar\n")}, + }, + }, + { + data: "---\nkind: pipeline\nfoo: bar\n...\n", + want: []*RawResource{ + {Kind: "pipeline", Data: []byte("kind: pipeline\nfoo: bar\n")}, + }, + }, + // + // multi-document files. + // + { + data: "kind: a\nb: c\n---\nkind: d\ne: f", + want: []*RawResource{ + {Kind: "a", Data: []byte("kind: a\nb: c\n")}, + {Kind: "d", Data: []byte("kind: d\ne: f\n")}, + }, + }, + { + data: "---\nkind: a\nb: c\n---\nkind: d\ne: f\n", + want: []*RawResource{ + {Kind: "a", Data: []byte("kind: a\nb: c\n")}, + {Kind: "d", Data: []byte("kind: d\ne: f\n")}, + }, + }, + { + data: "---\nkind: a\nb: c\n---\nkind: d\ne: f\n...", + want: []*RawResource{ + {Kind: "a", Data: []byte("kind: a\nb: c\n")}, + {Kind: "d", Data: []byte("kind: d\ne: f\n")}, + }, + }, + { + data: "---\nkind: a\nb: c\n---\nkind: d\ne: f\n...\n", + want: []*RawResource{ + {Kind: "a", Data: []byte("kind: a\nb: c\n")}, + {Kind: "d", Data: []byte("kind: d\ne: f\n")}, + }, + }, + } + + for i, test := range tests { + name := fmt.Sprint(i) + t.Run(name, func(t *testing.T) { + got, err := ParseRawString(test.data) + if err != nil { + t.Error(err) + } + if diff := cmp.Diff(got, test.want); diff != "" { + t.Fail() + t.Log(diff) + } + }) + } +} diff --git a/yaml/pipeline.go b/yaml/pipeline.go new file mode 100644 index 0000000..45ca13e --- /dev/null +++ b/yaml/pipeline.go @@ -0,0 +1,137 @@ +package yaml + +// Pipeline is a resource that defines a continuous +// delivery pipeline. +type Pipeline struct { + Version string `json:"version,omitempty"` + Kind string `json:"kind,omitempty"` + Type string `json:"type,omitempty"` + Name string `json:"name,omitempty"` + + Clone Clone `json:"clone,omitempty"` + Concurrency Concurrency `json:"concurrency,omitempty"` + DependsOn []string `json:"depends_on,omitempty" yaml:"depends_on" ` + Node map[string]string `json:"node,omitempty" yaml:"node"` + Platform Platform `json:"platform,omitempty"` + Services []*Container `json:"services,omitempty"` + Steps []*Container `json:"steps,omitempty"` + Trigger Conditions `json:"trigger,omitempty"` + Volumes []*Volume `json:"volumes,omitempty"` + Workspace Workspace `json:"workspace,omitempty"` +} + +// GetVersion returns the resource version. +func (p *Pipeline) GetVersion() string { return p.Version } + +// GetKind returns the resource kind. +func (p *Pipeline) GetKind() string { return p.Kind } + +type ( + // Clone configures the git clone. + Clone struct { + Disable bool `json:"disable,omitempty"` + Depth int `json:"depth,omitempty"` + SkipVerify bool `json:"skip_verify,omitempty" yaml:"skip_verify"` + } + + // Concurrency limits pipeline concurrency. + Concurrency struct { + Limit int `json:"limit,omitempty"` + } + + // Container defines a Docker container configuration. + Container struct { + Build *Build `json:"build,omitempty"` + Command []string `json:"command,omitempty"` + Commands []string `json:"commands,omitempty"` + Detach bool `json:"detach,omitempty"` + DependsOn []string `json:"depends_on,omitempty" yaml:"depends_on"` + Devices []*VolumeDevice `json:"devices,omitempty"` + DNS []string `json:"dns,omitempty"` + DNSSearch []string `json:"dns_search,omitempty" yaml:"dns_search"` + Entrypoint []string `json:"entrypoint,omitempty"` + Environment map[string]*Variable `json:"environment,omitempty"` + ExtraHosts []string `json:"extra_hosts,omitempty" yaml:"extra_hosts"` + Failure string `json:"failure,omitempty"` + Image string `json:"image,omitempty"` + Name string `json:"name,omitempty"` + Ports []*Port `json:"ports,omitempty"` + Privileged bool `json:"privileged,omitempty"` + Pull string `json:"pull,omitempty"` + Push *Push `json:"push,omitempty"` + Resources *Resources `json:"resources,omitempty"` + Settings map[string]*Parameter `json:"settings,omitempty"` + Shell string `json:"shell,omitempty"` + Volumes []*VolumeMount `json:"volumes,omitempty"` + When Conditions `json:"when,omitempty"` + WorkingDir string `json:"working_dir,omitempty" yaml:"working_dir"` + } + + // Resources describes the compute resource + // requirements. + Resources struct { + // Limits describes the maximum amount of compute + // resources allowed. + Limits *ResourceObject `json:"limits,omitempty"` + + // Requests describes the minimum amount of + // compute resources required. + Requests *ResourceObject `json:"requests,omitempty"` + } + + // ResourceObject describes compute resource + // requirements. + ResourceObject struct { + CPU MilliSize `json:"cpu"` + Memory BytesSize `json:"memory"` + } + + // Platform defines the target platform. + Platform struct { + OS string `json:"os,omitempty"` + Arch string `json:"arch,omitempty"` + Variant string `json:"variant,omitempty"` + Version string `json:"version,omitempty"` + } + + // Volume that can be mounted by containers. + Volume struct { + Name string `json:"name,omitempty"` + EmptyDir *VolumeEmptyDir `json:"temp,omitempty" yaml:"temp"` + HostPath *VolumeHostPath `json:"host,omitempty" yaml:"host"` + } + + // VolumeDevice describes a mapping of a raw block + // device within a container. + VolumeDevice struct { + Name string `json:"name,omitempty"` + DevicePath string `json:"path,omitempty" yaml:"path"` + } + + // VolumeMount describes a mounting of a Volume + // within a container. + VolumeMount struct { + Name string `json:"name,omitempty"` + MountPath string `json:"path,omitempty" yaml:"path"` + } + + // VolumeEmptyDir mounts a temporary directory from the + // host node's filesystem into the container. This can + // be used as a shared scratch space. + VolumeEmptyDir struct { + Medium string `json:"medium,omitempty"` + SizeLimit BytesSize `json:"size_limit,omitempty" yaml:"size_limit"` + } + + // VolumeHostPath mounts a file or directory from the + // host node's filesystem into your container. + VolumeHostPath struct { + Path string `json:"path,omitempty"` + } + + // Workspace represents the pipeline workspace configuraiton. + Workspace struct { + Base string `json:"base,omitempty"` + Path string `json:"path,omitempty"` + } +) diff --git a/yaml/pipeline_test.go b/yaml/pipeline_test.go new file mode 100644 index 0000000..4840887 --- /dev/null +++ b/yaml/pipeline_test.go @@ -0,0 +1,14 @@ +package yaml + +import "testing" + +func TestPipelineUnmarshal(t *testing.T) { + diff, err := diff("testdata/pipeline.yml") + if err != nil { + t.Error(err) + } + if diff != "" { + t.Error("Failed to parse pipeline") + t.Log(diff) + } +} diff --git a/yaml/port.go b/yaml/port.go new file mode 100644 index 0000000..79d9085 --- /dev/null +++ b/yaml/port.go @@ -0,0 +1,30 @@ +package yaml + +type ( + + // Port represents a network port in a single container. + Port struct { + Port int `json:"port,omitempty"` + Host int `json:"host,omitempty"` + Protocol string `json:"protocol,omitempty"` + } + + port struct { + Port int + Host int + Protocol string + } +) + +// UnmarshalYAML implements yaml unmarshalling. +func (p *Port) UnmarshalYAML(unmarshal func(interface{}) error) error { + out := new(port) + err := unmarshal(&out.Port) + if err != nil { + err = unmarshal(&out) + } + p.Port = out.Port + p.Host = out.Host + p.Protocol = out.Protocol + return err +} diff --git a/yaml/port_test.go b/yaml/port_test.go new file mode 100644 index 0000000..410f5eb --- /dev/null +++ b/yaml/port_test.go @@ -0,0 +1,45 @@ +package yaml + +import ( + "testing" + + "gopkg.in/yaml.v2" +) + +func TestPort(t *testing.T) { + tests := []struct { + yaml string + port int + host int + protocol string + }{ + { + yaml: "80", + port: 80, + }, + { + yaml: "{ port: 80, host: 8080, protocol: TCP }", + port: 80, + host: 8080, + protocol: "TCP", + }, + } + for _, test := range tests { + in := []byte(test.yaml) + out := new(Port) + err := yaml.Unmarshal(in, out) + if err != nil { + t.Error(err) + return + } + if got, want := out.Port, test.port; got != want { + t.Errorf("Want Port %d, got %d", want, got) + } + if got, want := out.Host, test.host; got != want { + t.Errorf("Want Host %d, got %d", want, got) + } + if got, want := out.Protocol, test.protocol; got != want { + t.Errorf("Want Host %s, got %s", want, got) + } + } +} diff --git a/yaml/pretty/container.go b/yaml/pretty/container.go new file mode 100644 index 0000000..6480f55 --- /dev/null +++ b/yaml/pretty/container.go @@ -0,0 +1,219 @@ +package pretty + +import ( + "sort" + + "github.com/drone/drone-yaml/yaml" +) + +// helper function pretty prints the container mapping. +func printContainer(w writer, v *yaml.Container) { + w.WriteTagValue("name", v.Name) + w.WriteTagValue("pull", v.Pull) + w.WriteTagValue("image", v.Image) + + if v.Build != nil { + printBuild(w, v.Build) + } + if v.Push != nil { + w.WriteTagValue("push", v.Push.Image) + } + + w.WriteTagValue("detach", v.Detach) + w.WriteTagValue("shell", v.Shell) + w.WriteTagValue("entrypoint", v.Entrypoint) + w.WriteTagValue("command", v.Command) + w.WriteTagValue("commands", v.Commands) + w.WriteTagValue("dns", v.DNS) + w.WriteTagValue("dns_search", v.DNSSearch) + w.WriteTagValue("extra_hosts", v.ExtraHosts) + + if len(v.Settings) > 0 { + printSettings(w, v.Settings) + } + + if len(v.Environment) > 0 { + printEnviron(w, v.Environment) + } + + w.WriteTagValue("failure", v.Failure) + w.WriteTagValue("privileged", v.Privileged) + w.WriteTagValue("working_dir", v.WorkingDir) + + if len(v.Devices) > 0 { + printDeviceMounts(w, v.Devices) + } + if len(v.Ports) > 0 { + printPorts(w, v.Ports) + } + if v.Resources != nil { + printResources(w, v.Resources) + } + if len(v.Volumes) > 0 { + printVolumeMounts(w, v.Volumes) + } + if !isConditionsEmpty(v.When) { + printConditions(w, "when", v.When) + } + if len(v.DependsOn) > 0 { + printDependsOn(w, v.DependsOn) + } + w.WriteByte('\n') +} + +// helper function pretty prints the build node. +func printBuild(w writer, v *yaml.Build) { + if shortBuild(v) { + w.WriteTagValue("build", v.Image) + } else { + w.WriteTag("build") + w.IndentIncrease() + w.WriteTagValue("image", v.Image) + w.WriteTagValue("args", v.Args) + w.WriteTagValue("cache_from", v.CacheFrom) + w.WriteTagValue("context", v.Context) + w.WriteTagValue("dockerfile", v.Dockerfile) + w.WriteTagValue("labels", v.Labels) + w.IndentDecrease() + } +} + +// helper function pretty prints the depends_on sequence. +func printDependsOn(w writer, v []string) { + w.WriteTagValue("depends_on", v) +} + +// helper function pretty prints the device sequence. +func printDeviceMounts(w writer, v []*yaml.VolumeDevice) { + w.WriteTag("devices") + for _, v := range v { + s := new(indexWriter) + s.writer = w + s.IndentIncrease() + s.WriteTagValue("name", v.Name) + s.WriteTagValue("path", v.DevicePath) + s.IndentDecrease() + } +} + +// helper function pretty prints the environment mapping. +func printEnviron(w writer, v map[string]*yaml.Variable) { + var keys []string + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + + w.WriteTag("environment") + w.IndentIncrease() + for _, k := range keys { + v := v[k] + if v.Secret == "" { + w.WriteTagValue(k, v.Value) + } else { + w.WriteTag(k) + w.IndentIncrease() + w.WriteTagValue("from_secret", v.Secret) + w.IndentDecrease() + } + } + w.IndentDecrease() +} + +// helper function pretty prints the port sequence. +func printPorts(w writer, v []*yaml.Port) { + w.WriteTag("ports") + for _, v := range v { + if shortPort(v) { + w.WriteByte('\n') + w.Indent() + w.WriteByte('-') + w.WriteByte(' ') + writeInt(w, v.Port) + continue + } + + s := new(indexWriter) + s.writer = w + s.IndentIncrease() + s.WriteTagValue("port", v.Port) + s.WriteTagValue("host", v.Host) + s.WriteTagValue("protocol", v.Protocol) + s.IndentDecrease() + } +} + +// helper function pretty prints the resoure mapping. +func printResources(w writer, v *yaml.Resources) { + w.WriteTag("resources") + w.IndentIncrease() + + if v.Limits != nil { + w.WriteTag("limits") + w.IndentIncrease() + w.WriteTagValue("cpu", v.Limits.CPU) + w.WriteTagValue("memory", v.Limits.Memory) + w.IndentDecrease() + } + if v.Requests != nil { + w.WriteTag("requests") + w.IndentIncrease() + w.WriteTagValue("cpu", v.Requests.CPU) + w.WriteTagValue("memory", v.Requests.Memory) + w.IndentDecrease() + } + w.IndentDecrease() +} + +// helper function pretty prints the resoure mapping. +func printSettings(w writer, v map[string]*yaml.Parameter) { + var keys []string + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + + w.WriteTag("settings") + w.IndentIncrease() + for _, k := range keys { + v := v[k] + if v.Secret == "" { + w.WriteTagValue(k, v.Value) + } else { + w.WriteTag(k) + w.IndentIncrease() + w.WriteTagValue("from_secret", v.Secret) + w.IndentDecrease() + } + } + w.IndentDecrease() +} + +// helper function pretty prints the volume sequence. +func printVolumeMounts(w writer, v []*yaml.VolumeMount) { + w.WriteTag("volumes") + for _, v := range v { + s := new(indexWriter) + s.writer = w + s.IndentIncrease() + s.WriteTagValue("name", v.Name) + s.WriteTagValue("path", v.MountPath) + s.IndentDecrease() + } +} + +// helper function returns true if the Build block should +// be printed in short form. +func shortBuild(b *yaml.Build) bool { + return len(b.Args) == 0 && + len(b.CacheFrom) == 0 && + len(b.Context) == 0 && + len(b.Dockerfile) == 0 && + len(b.Labels) == 0 +} + +// helper function returns true if the Port block should +// be printed in short form. +func shortPort(p *yaml.Port) bool { + return p.Host == 0 && len(p.Protocol) == 0 +} diff --git a/yaml/pretty/container_test.go b/yaml/pretty/container_test.go new file mode 100644 index 0000000..fac5cb9 --- /dev/null +++ b/yaml/pretty/container_test.go @@ -0,0 +1 @@ +package pretty diff --git a/yaml/pretty/cron.go b/yaml/pretty/cron.go new file mode 100644 index 0000000..739cc70 --- /dev/null +++ b/yaml/pretty/cron.go @@ -0,0 +1,41 @@ +package pretty + +import "github.com/drone/drone-yaml/yaml" + +// helper function pretty prints the cron resource. +func printCron(w writer, v *yaml.Cron) { + w.WriteString("---") + w.WriteTagValue("version", v.Version) + w.WriteTagValue("kind", v.Kind) + w.WriteTagValue("name", v.Name) + printSpec(w, v) + w.WriteByte('\n') + w.WriteByte('\n') +} + +// helper function pretty prints the spec block. +func printSpec(w writer, v *yaml.Cron) { + w.WriteTag("spec") + + w.IndentIncrease() + w.WriteTagValue("schedule", v.Spec.Schedule) + w.WriteTagValue("branch", v.Spec.Branch) + if hasDeployment(v) { + printDeploy(w, v) + } + w.IndentDecrease() +} + +// helper function pretty prints the deploy block. +func printDeploy(w writer, v *yaml.Cron) { + w.WriteTag("deployment") + w.IndentIncrease() + w.WriteTagValue("target", v.Spec.Deploy.Target) + w.IndentDecrease() +} + +// helper function returns true if the deployment +// object is empty. +func hasDeployment(v *yaml.Cron) bool { + return v.Spec.Deploy.Target != "" +} diff --git a/yaml/pretty/cron_test.go b/yaml/pretty/cron_test.go new file mode 100644 index 0000000..c783b33 --- /dev/null +++ b/yaml/pretty/cron_test.go @@ -0,0 +1,12 @@ +package pretty + +import "testing" + +func TestCron(t *testing.T) { + ok, err := diff("testdata/cron.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} diff --git a/yaml/pretty/pipeline.go b/yaml/pretty/pipeline.go new file mode 100644 index 0000000..a982c1d --- /dev/null +++ b/yaml/pretty/pipeline.go @@ -0,0 +1,265 @@ +package pretty + +import ( + "github.com/drone/drone-yaml/yaml" +) + +// helper function to pretty print the pipeline resource. +func printPipeline(w writer, v *yaml.Pipeline) { + w.WriteString("---") + w.WriteTagValue("version", v.Version) + w.WriteTagValue("kind", v.Kind) + w.WriteTagValue("type", v.Type) + w.WriteTagValue("name", v.Name) + w.WriteByte('\n') + + if !isPlatformEmpty(v.Platform) { + printPlatform(w, v.Platform) + } else { + printPlatformDefault(w) + } + if !isCloneEmpty(v.Clone) { + printClone(w, v.Clone) + } + if !isConcurrencyEmpty(v.Concurrency) { + printConcurrency(w, v.Concurrency) + } + if !isWorkspaceEmpty(v.Workspace) { + printWorkspace(w, v.Workspace) + } + + if len(v.Steps) > 0 { + w.WriteTag("steps") + for _, step := range v.Steps { + if step == nil { + continue + } + seq := new(indexWriter) + seq.writer = w + seq.IndentIncrease() + printContainer(seq, step) + seq.IndentDecrease() + } + } + + if len(v.Services) > 0 { + w.WriteTag("services") + for _, step := range v.Services { + if step == nil { + continue + } + seq := new(indexWriter) + seq.writer = w + seq.IndentIncrease() + printContainer(seq, step) + seq.IndentDecrease() + } + } + + if len(v.Volumes) != 0 { + printVolumes(w, v.Volumes) + w.WriteByte('\n') + } + + if len(v.Node) > 0 { + printNode(w, v.Node) + w.WriteByte('\n') + } + + if !isConditionsEmpty(v.Trigger) { + printConditions(w, "trigger", v.Trigger) + w.WriteByte('\n') + } + + if len(v.DependsOn) > 0 { + printDependsOn(w, v.DependsOn) + w.WriteByte('\n') + } + + w.WriteByte('\n') +} + +// helper function pretty prints the clone block. +func printClone(w writer, v yaml.Clone) { + w.WriteTag("clone") + w.IndentIncrease() + w.WriteTagValue("depth", v.Depth) + w.WriteTagValue("disable", v.Disable) + w.WriteTagValue("skip_verify", v.SkipVerify) + w.WriteByte('\n') + w.IndentDecrease() +} + +// helper function pretty prints the clone block. +func printConcurrency(w writer, v yaml.Concurrency) { + w.WriteTag("concurrency") + w.IndentIncrease() + w.WriteTagValue("limit", v.Limit) + w.WriteByte('\n') + w.IndentDecrease() +} + +// helper function pretty prints the conditions mapping. +func printConditions(w writer, name string, v yaml.Conditions) { + w.WriteTag(name) + w.IndentIncrease() + if !isConditionEmpty(v.Branch) { + printCondition(w, "branch", v.Branch) + } + if !isConditionEmpty(v.Event) { + printCondition(w, "event", v.Event) + } + if !isConditionEmpty(v.Instance) { + printCondition(w, "instance", v.Instance) + } + if !isConditionEmpty(v.Paths) { + printCondition(w, "paths", v.Paths) + } + if !isConditionEmpty(v.Ref) { + printCondition(w, "ref", v.Ref) + } + if !isConditionEmpty(v.Repo) { + printCondition(w, "repo", v.Repo) + } + if !isConditionEmpty(v.Status) { + printCondition(w, "status", v.Status) + } + if !isConditionEmpty(v.Target) { + printCondition(w, "target", v.Target) + } + w.IndentDecrease() +} + +// helper function pretty prints a condition mapping. +func printCondition(w writer, k string, v yaml.Condition) { + w.WriteTag(k) + if len(v.Include) != 0 && len(v.Exclude) == 0 { + w.WriteByte('\n') + w.Indent() + writeValue(w, v.Include) + } + if len(v.Include) != 0 && len(v.Exclude) != 0 { + w.IndentIncrease() + w.WriteTagValue("include", v.Include) + w.IndentDecrease() + } + if len(v.Exclude) != 0 { + w.IndentIncrease() + w.WriteTagValue("exclude", v.Exclude) + w.IndentDecrease() + } +} + +// helper function pretty prints the node mapping. +func printNode(w writer, v map[string]string) { + w.WriteTagValue("node", v) +} + +// helper function pretty prints the target platform. +func printPlatform(w writer, v yaml.Platform) { + w.WriteTag("platform") + w.IndentIncrease() + w.WriteTagValue("os", v.OS) + w.WriteTagValue("arch", v.Arch) + w.WriteTagValue("variant", v.Variant) + w.WriteTagValue("version", v.Version) + w.WriteByte('\n') + w.IndentDecrease() +} + +// helper function prints default platform values. +// Including target platform is considered a best-practive. +func printPlatformDefault(w writer) { + w.WriteTag("platform") + w.IndentIncrease() + w.WriteTagValue("os", "linux") + w.WriteTagValue("arch", "amd64") + w.WriteByte('\n') + w.IndentDecrease() +} + +// helper function pretty prints the volume sequence. +func printVolumes(w writer, v []*yaml.Volume) { + w.WriteTag("volumes") + for _, v := range v { + s := new(indexWriter) + s.writer = w + s.IndentIncrease() + + s.WriteTagValue("name", v.Name) + if v := v.EmptyDir; v != nil { + s.WriteTag("temp") + s.IndentIncrease() + s.WriteTagValue("medium", v.Medium) + s.WriteTagValue("size_limit", v.SizeLimit) + s.IndentDecrease() + } + + if v := v.HostPath; v != nil { + s.WriteTag("host") + s.IndentIncrease() + s.WriteTagValue("path", v.Path) + s.IndentDecrease() + } + + s.IndentDecrease() + } +} + +// helper function pretty prints the workspace block. +func printWorkspace(w writer, v yaml.Workspace) { + w.WriteTag("workspace") + w.IndentIncrease() + w.WriteTagValue("base", v.Base) + w.WriteTagValue("path", v.Path) + w.WriteByte('\n') + w.IndentDecrease() +} + +// helper function returns true if the workspace +// object is empty. +func isWorkspaceEmpty(v yaml.Workspace) bool { + return v.Path == "" && v.Base == "" +} + +// helper function returns true if the platform +// object is empty. +func isPlatformEmpty(v yaml.Platform) bool { + return v.OS == "" && + v.Arch == "" && + v.Variant == "" && + v.Version == "" +} + +// helper function returns true if the clone +// object is empty. +func isCloneEmpty(v yaml.Clone) bool { + return v.Depth == 0 && + v.Disable == false && + v.SkipVerify == false +} + +// helper function returns true if the concurrency +// object is empty. +func isConcurrencyEmpty(v yaml.Concurrency) bool { + return v.Limit == 0 +} + +// helper function returns true if the conditions +// object is empty. +func isConditionsEmpty(v yaml.Conditions) bool { + return isConditionEmpty(v.Branch) && + isConditionEmpty(v.Event) && + isConditionEmpty(v.Instance) && + isConditionEmpty(v.Paths) && + isConditionEmpty(v.Ref) && + isConditionEmpty(v.Repo) && + isConditionEmpty(v.Status) && + isConditionEmpty(v.Target) +} + +// helper function returns true if the condition +// object is empty. +func isConditionEmpty(v yaml.Condition) bool { + return len(v.Exclude) == 0 && len(v.Include) == 0 +} diff --git a/yaml/pretty/pipeline_test.go b/yaml/pretty/pipeline_test.go new file mode 100644 index 0000000..c55411b --- /dev/null +++ b/yaml/pretty/pipeline_test.go @@ -0,0 +1,147 @@ +package pretty + +import "testing" + +func TestPipeline_Build_Short(t *testing.T) { + ok, err := diff("testdata/pipeline_build_short.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Build_Long(t *testing.T) { + ok, err := diff("testdata/pipeline_build_long.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Concurrency(t *testing.T) { + ok, err := diff("testdata/pipeline_concurrency.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Clone_Depth(t *testing.T) { + ok, err := diff("testdata/pipeline_clone_depth.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Clone_Disable(t *testing.T) { + ok, err := diff("testdata/pipeline_clone_disable.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Clone_SkipVerify(t *testing.T) { + ok, err := diff("testdata/pipeline_clone_skip_verify.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Depends(t *testing.T) { + ok, err := diff("testdata/pipeline_depends.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Node(t *testing.T) { + ok, err := diff("testdata/pipeline_node.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Push(t *testing.T) { + ok, err := diff("testdata/pipeline_push.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Ports(t *testing.T) { + ok, err := diff("testdata/pipeline_ports.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Resources(t *testing.T) { + ok, err := diff("testdata/pipeline_resources.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Services(t *testing.T) { + ok, err := diff("testdata/services.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Settings(t *testing.T) { + ok, err := diff("testdata/pipeline_settings.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Trigger(t *testing.T) { + ok, err := diff("testdata/pipeline_trigger.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Volumes(t *testing.T) { + ok, err := diff("testdata/pipeline_volumes.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestPipeline_Workspace(t *testing.T) { + ok, err := diff("testdata/pipeline_workspace.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} diff --git a/yaml/pretty/pretty.go b/yaml/pretty/pretty.go new file mode 100644 index 0000000..be3a671 --- /dev/null +++ b/yaml/pretty/pretty.go @@ -0,0 +1,29 @@ +package pretty + +import ( + "io" + + "github.com/drone/drone-yaml/yaml" +) + +// Print pretty prints the manifest. +func Print(w io.Writer, v *yaml.Manifest) { + state := new(baseWriter) + for _, r := range v.Resources { + switch t := r.(type) { + case *yaml.Cron: + printCron(state, t) + case *yaml.Secret: + printSecret(state, t) + case *yaml.Registry: + printRegistry(state, t) + case *yaml.Signature: + printSignature(state, t) + case *yaml.Pipeline: + printPipeline(state, t) + } + } + state.WriteString("...") + state.WriteByte('\n') + w.Write(state.Bytes()) +} diff --git a/yaml/pretty/pretty_test.go b/yaml/pretty/pretty_test.go new file mode 100644 index 0000000..4fd1d0a --- /dev/null +++ b/yaml/pretty/pretty_test.go @@ -0,0 +1,43 @@ +package pretty + +import ( + "bytes" + "io/ioutil" + "testing" + + "github.com/drone/drone-yaml/yaml" +) + +// +// +// + +func TestPrintManifest(t *testing.T) { + ok, err := diff("testdata/manifest.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func diff(file string) (bool, error) { + manifest, err := yaml.ParseFile(file) + if err != nil { + return false, err + } + golden, err := ioutil.ReadFile(file + ".golden") + if err != nil { + return false, err + } + + buf := new(bytes.Buffer) + Print(buf, manifest) + + equal := bytes.Equal(buf.Bytes(), golden) + if !equal { + println(">>>") + println(buf.String()) + } + return equal, nil +} diff --git a/yaml/pretty/registry.go b/yaml/pretty/registry.go new file mode 100644 index 0000000..fcc967a --- /dev/null +++ b/yaml/pretty/registry.go @@ -0,0 +1,20 @@ +package pretty + +import ( + "github.com/drone/drone-yaml/yaml" +) + +// helper function pretty prints the registry resource. +func printRegistry(w writer, v *yaml.Registry) { + w.WriteString("---") + w.WriteTagValue("version", v.Version) + w.WriteTagValue("kind", v.Kind) + w.WriteTagValue("type", v.Type) + if v.Type == "encrypted" { + printData(w, v.Data) + } else { + w.WriteTagValue("data", v.Data) + } + w.WriteByte('\n') + w.WriteByte('\n') +} diff --git a/yaml/pretty/registry_test.go b/yaml/pretty/registry_test.go new file mode 100644 index 0000000..599822b --- /dev/null +++ b/yaml/pretty/registry_test.go @@ -0,0 +1,12 @@ +package pretty + +import "testing" + +func TestRegistry(t *testing.T) { + ok, err := diff("testdata/registry.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} diff --git a/yaml/pretty/secret.go b/yaml/pretty/secret.go new file mode 100644 index 0000000..651b63c --- /dev/null +++ b/yaml/pretty/secret.go @@ -0,0 +1,88 @@ +package pretty + +import ( + "sort" + "strings" + + "github.com/drone/drone-yaml/yaml" +) + +// TODO consider "!!binary |" for secret value + +// helper function to pretty prints the signature resource. +func printSecret(w writer, v *yaml.Secret) { + w.WriteString("---") + w.WriteTagValue("version", v.Version) + w.WriteTagValue("kind", v.Kind) + w.WriteTagValue("type", toSecretType(v.Type)) + + if len(v.Data) > 0 { + printData(w, v.Data) + } + if len(v.External) > 0 { + printExternalData(w, v.External) + } + w.WriteByte('\n') + w.WriteByte('\n') +} + +// helper function returns the secret type text. +func toSecretType(s string) string { + s = strings.ToLower(s) + switch s { + case "docker", "ecr", "general": + return s + default: + return "general" + } +} + +// helper function prints the external data. +func printExternalData(w writer, d map[string]yaml.ExternalData) { + var keys []string + for k := range d { + keys = append(keys, k) + } + sort.Strings(keys) + + w.WriteTag("external_data") + w.IndentIncrease() + for _, k := range keys { + v := d[k] + w.WriteTag(k) + w.IndentIncrease() + w.WriteTagValue("path", v.Path) + w.WriteTagValue("name", v.Name) + w.IndentDecrease() + } + w.IndentDecrease() +} + +func printData(w writer, d map[string]string) { + var keys []string + for k := range d { + keys = append(keys, k) + } + sort.Strings(keys) + + w.WriteTag("data") + w.IndentIncrease() + for _, k := range keys { + v := d[k] + w.WriteTag(k) + w.WriteByte(' ') + w.WriteByte('>') + w.IndentIncrease() + v = spaceReplacer.Replace(v) + for _, s := range chunk(v, 60) { + w.WriteByte('\n') + w.Indent() + w.WriteString(s) + } + w.IndentDecrease() + } + w.IndentDecrease() +} + +// replace spaces and newlines. +var spaceReplacer = strings.NewReplacer(" ", "", "\n", "") diff --git a/yaml/pretty/secret_test.go b/yaml/pretty/secret_test.go new file mode 100644 index 0000000..b98246d --- /dev/null +++ b/yaml/pretty/secret_test.go @@ -0,0 +1,21 @@ +package pretty + +import "testing" + +func TestSecret(t *testing.T) { + ok, err := diff("testdata/secret.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} + +func TestExternalSecret(t *testing.T) { + ok, err := diff("testdata/secret_extern.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} diff --git a/yaml/pretty/signature.go b/yaml/pretty/signature.go new file mode 100644 index 0000000..e688899 --- /dev/null +++ b/yaml/pretty/signature.go @@ -0,0 +1,15 @@ +package pretty + +import ( + "github.com/drone/drone-yaml/yaml" +) + +// helper function pretty prints the signature resource. +func printSignature(w writer, v *yaml.Signature) { + w.WriteString("---") + w.WriteTagValue("version", v.Version) + w.WriteTagValue("kind", v.Kind) + w.WriteTagValue("hmac", v.Hmac) + w.WriteByte('\n') + w.WriteByte('\n') +} diff --git a/yaml/pretty/signature_test.go b/yaml/pretty/signature_test.go new file mode 100644 index 0000000..aaa3a61 --- /dev/null +++ b/yaml/pretty/signature_test.go @@ -0,0 +1,12 @@ +package pretty + +import "testing" + +func TestSignature(t *testing.T) { + ok, err := diff("testdata/signature.yml") + if err != nil { + t.Error(err) + } else if !ok { + t.Errorf("Unepxected formatting") + } +} diff --git a/yaml/pretty/testdata/cron.yml b/yaml/pretty/testdata/cron.yml new file mode 100644 index 0000000..9bf7360 --- /dev/null +++ b/yaml/pretty/testdata/cron.yml @@ -0,0 +1,12 @@ +version: 1 +name: nightly +kind: cron + +spec: + branch: master + + schedule: "1 * * * *" + + deployment: + + target: production diff --git a/yaml/pretty/testdata/cron.yml.golden b/yaml/pretty/testdata/cron.yml.golden new file mode 100644 index 0000000..aa5cdae --- /dev/null +++ b/yaml/pretty/testdata/cron.yml.golden @@ -0,0 +1,11 @@ +--- +version: 1 +kind: cron +name: nightly +spec: + schedule: "1 * * * *" + branch: master + deployment: + target: production + +... diff --git a/yaml/pretty/testdata/manifest.yml b/yaml/pretty/testdata/manifest.yml new file mode 100644 index 0000000..76abe47 --- /dev/null +++ b/yaml/pretty/testdata/manifest.yml @@ -0,0 +1,64 @@ +--- +kind: pipeline +name: default + +clone: + disable: true + depth: 50 + +platform: + os: linux + arch: arm64 + variant: 7 + version: 1803 + +steps: +- name: build + image: golang + commands: + - go get + - go build + +- name: test + image: golang + commands: + - go test -v + +trigger: + status: + - success + branch: + - master + - develop + +depends_on: +- foo +- bar + +--- +kind: registry +data: + index.docker.io: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + +--- +kind: secret +type: general +data: + username: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + password: YjgwNDc4ZDY4NmQzNzQzYjNkYmUwYmE3YjMwOTM2OWUK + +--- +kind: cron +name: nightly + +spec: + schedule: "1 * * * *" + branch: master + deployment: + target: production + +--- +kind: signature +hmac: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + +... diff --git a/yaml/pretty/testdata/manifest.yml.golden b/yaml/pretty/testdata/manifest.yml.golden new file mode 100644 index 0000000..fafe757 --- /dev/null +++ b/yaml/pretty/testdata/manifest.yml.golden @@ -0,0 +1,65 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: arm64 + variant: 7 + version: 1803 + +clone: + depth: 50 + disable: true + +steps: +- name: build + image: golang + commands: + - go get + - go build + +- name: test + image: golang + commands: + - go test -v + +trigger: + branch: + - master + - develop + status: + - success + +depends_on: +- foo +- bar + +--- +kind: registry +data: + index.docker.io: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + +--- +kind: secret +type: general +data: + password: > + YjgwNDc4ZDY4NmQzNzQzYjNkYmUwYmE3YjMwOTM2OWUK + username: > + N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + +--- +kind: cron +name: nightly +spec: + schedule: "1 * * * *" + branch: master + deployment: + target: production + +--- +kind: signature +hmac: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + +... diff --git a/yaml/pretty/testdata/pipeline.yml b/yaml/pretty/testdata/pipeline.yml new file mode 100644 index 0000000..d0e0093 --- /dev/null +++ b/yaml/pretty/testdata/pipeline.yml @@ -0,0 +1,151 @@ +--- +kind: pipeline +name: default + +clone: + disable: true + depth: 50 + +platform: + os: windows + arch: arm64 + variant: 7 + version: 1803 + +workspace: + base: /go + path: src/github.com/octocat/hello-world + +steps: +- name: test_build + build: + image: drone/drone + context: . + args: + foo: bar + baz: boo + labels: + qux: qoo + cache_from: + - alpine + - golang + +- name: test_push + push: + image: drone/drone + +- name: test_commands + image: drone/drone + pull: always + shell: bash + commands: + - go get + - go test + failure: ignore + +- name: test_volumes + image: docker + commands: + - docker build + - docker test + environment: + DOCKER_HOST: /var/run/docker.sock + privileged: true + volumes: + - name: docker + path: /var/run/docker.sock + +- name: test_dns + image: alpine + commands: + - ping google.com + dns: + - 8.8.8.8 + dns_search: + - dc1.example.com + - dc2.example.com + extra_hosts: + - "somehost:162.242.195.82" + - "otherhost:50.31.209.229" + +# - name: test_privileged +# image: alpine +# commands: +# - ls /proc +# privileged: true + +# - name: test_devices +# image: alpine +# devices: +# - name: xvda +# path: /dev/xvda + +- name: test_env_secrets + image: alpine + environment: + GOOS: linux + GOARCH: amd64 + SSH_KEY: + from_secret: username + commands: + - go get + - go build + +- name: test_when + image: alpine + depends_on: + - foo + - bar + when: + branch: + - master + - develop + status: + - success + ref: + include: + - refs/tags/* + exclude: + - refs/tags/feature-* + +services: +- name: test_entrypoint + image: reids:latest + entrypoint: + - /bin/redis-server + ports: + - 6379 + +# - name: test_command +# image: reids:latest +# command: +# - --port +# - 6380 +# ports: +# - 6380 + +# - name: test_working_dir +# image: redis:latest +# working_dir: /data +# ports: +# - port: 6379 +# host: 6380 +# protocol: TCP + +# volumes: +# - name: docker +# host: +# path: /var/run/docker.sock +# - name: temp +# temp: {} + +# trigger: +# branch: +# - master +# - develop +# status: +# - success + +# depends_on: +# - foo +# - bar diff --git a/yaml/pretty/testdata/pipeline.yml.golden b/yaml/pretty/testdata/pipeline.yml.golden new file mode 100644 index 0000000..e69de29 diff --git a/yaml/pretty/testdata/pipeline_build_long.yml b/yaml/pretty/testdata/pipeline_build_long.yml new file mode 100644 index 0000000..79c6526 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_build_long.yml @@ -0,0 +1,21 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: test_build + build: + image: octocat/hello-world + context: . + args: + foo: bar + baz: boo + labels: + qux: qoo + cache_from: + - alpine + - golang diff --git a/yaml/pretty/testdata/pipeline_build_long.yml.golden b/yaml/pretty/testdata/pipeline_build_long.yml.golden new file mode 100644 index 0000000..3525d03 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_build_long.yml.golden @@ -0,0 +1,23 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: test_build + build: + image: octocat/hello-world + args: + baz: boo + foo: bar + cache_from: + - alpine + - golang + context: . + labels: + qux: qoo + +... diff --git a/yaml/pretty/testdata/pipeline_build_short.yml b/yaml/pretty/testdata/pipeline_build_short.yml new file mode 100644 index 0000000..2fe8193 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_build_short.yml @@ -0,0 +1,7 @@ +--- +kind: pipeline +name: default + +steps: +- name: test_build + build: octocat/hello-world \ No newline at end of file diff --git a/yaml/pretty/testdata/pipeline_build_short.yml.golden b/yaml/pretty/testdata/pipeline_build_short.yml.golden new file mode 100644 index 0000000..a6e8484 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_build_short.yml.golden @@ -0,0 +1,13 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: test_build + build: octocat/hello-world + +... diff --git a/yaml/pretty/testdata/pipeline_clone_depth.yml b/yaml/pretty/testdata/pipeline_clone_depth.yml new file mode 100644 index 0000000..245dced --- /dev/null +++ b/yaml/pretty/testdata/pipeline_clone_depth.yml @@ -0,0 +1,13 @@ +--- +kind: pipeline +name: default + +clone: + depth: 50 + +steps: +- name: test + image: golang + commands: + - go build + - go test -v diff --git a/yaml/pretty/testdata/pipeline_clone_depth.yml.golden b/yaml/pretty/testdata/pipeline_clone_depth.yml.golden new file mode 100644 index 0000000..7b7f2ac --- /dev/null +++ b/yaml/pretty/testdata/pipeline_clone_depth.yml.golden @@ -0,0 +1,19 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +clone: + depth: 50 + +steps: +- name: test + image: golang + commands: + - go build + - go test -v + +... diff --git a/yaml/pretty/testdata/pipeline_clone_disable.yml b/yaml/pretty/testdata/pipeline_clone_disable.yml new file mode 100644 index 0000000..d0946e0 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_clone_disable.yml @@ -0,0 +1,12 @@ +--- +kind: pipeline +name: default + +clone: + disable: true + +steps: +- name: webhook + image: alpine + commands: + - curl -x POST http://abc.com:80/~smith/home.html diff --git a/yaml/pretty/testdata/pipeline_clone_disable.yml.golden b/yaml/pretty/testdata/pipeline_clone_disable.yml.golden new file mode 100644 index 0000000..67596b2 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_clone_disable.yml.golden @@ -0,0 +1,18 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +clone: + disable: true + +steps: +- name: webhook + image: alpine + commands: + - curl -x POST http://abc.com:80/~smith/home.html + +... diff --git a/yaml/pretty/testdata/pipeline_clone_skip_verify.yml b/yaml/pretty/testdata/pipeline_clone_skip_verify.yml new file mode 100644 index 0000000..6a3e7ba --- /dev/null +++ b/yaml/pretty/testdata/pipeline_clone_skip_verify.yml @@ -0,0 +1,12 @@ +--- +kind: pipeline +name: default + +clone: + skip_verify: true + +steps: +- name: webhook + image: alpine + commands: + - curl -x POST http://abc.com:80/~smith/home.html diff --git a/yaml/pretty/testdata/pipeline_clone_skip_verify.yml.golden b/yaml/pretty/testdata/pipeline_clone_skip_verify.yml.golden new file mode 100644 index 0000000..708f9e4 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_clone_skip_verify.yml.golden @@ -0,0 +1,18 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +clone: + skip_verify: true + +steps: +- name: webhook + image: alpine + commands: + - curl -x POST http://abc.com:80/~smith/home.html + +... diff --git a/yaml/pretty/testdata/pipeline_concurrency.yml b/yaml/pretty/testdata/pipeline_concurrency.yml new file mode 100644 index 0000000..2316dba --- /dev/null +++ b/yaml/pretty/testdata/pipeline_concurrency.yml @@ -0,0 +1,12 @@ +--- +kind: pipeline +name: default + +concurrency: + limit: 1 + +steps: +- name: webhook + image: alpine + commands: + - curl -x POST http://abc.com:80/~smith/home.html diff --git a/yaml/pretty/testdata/pipeline_concurrency.yml.golden b/yaml/pretty/testdata/pipeline_concurrency.yml.golden new file mode 100644 index 0000000..d891937 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_concurrency.yml.golden @@ -0,0 +1,18 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +concurrency: + limit: 1 + +steps: +- name: webhook + image: alpine + commands: + - curl -x POST http://abc.com:80/~smith/home.html + +... diff --git a/yaml/pretty/testdata/pipeline_depends.yml b/yaml/pretty/testdata/pipeline_depends.yml new file mode 100644 index 0000000..d739aa8 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_depends.yml @@ -0,0 +1,20 @@ +kind: pipeline +name: default + +steps: + - name: build + image: golang + commands: + - go build + + - name: test + image: golang + commands: + - go build + - go test -v + depends_on: + - build + +depends_on: + - foo + - bar diff --git a/yaml/pretty/testdata/pipeline_depends.yml.golden b/yaml/pretty/testdata/pipeline_depends.yml.golden new file mode 100644 index 0000000..22504fe --- /dev/null +++ b/yaml/pretty/testdata/pipeline_depends.yml.golden @@ -0,0 +1,27 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: build + image: golang + commands: + - go build + +- name: test + image: golang + commands: + - go build + - go test -v + depends_on: + - build + +depends_on: +- foo +- bar + +... diff --git a/yaml/pretty/testdata/pipeline_node.yml b/yaml/pretty/testdata/pipeline_node.yml new file mode 100644 index 0000000..7ca2aba --- /dev/null +++ b/yaml/pretty/testdata/pipeline_node.yml @@ -0,0 +1,19 @@ +kind: pipeline +name: default + +steps: + - name: build + image: golang + commands: + - go build + + - name: test + image: golang + commands: + - go build + - go test -v + depends_on: + - build + +node: + disk: ssd diff --git a/yaml/pretty/testdata/pipeline_node.yml.golden b/yaml/pretty/testdata/pipeline_node.yml.golden new file mode 100644 index 0000000..3f0c4b8 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_node.yml.golden @@ -0,0 +1,26 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: build + image: golang + commands: + - go build + +- name: test + image: golang + commands: + - go build + - go test -v + depends_on: + - build + +node: + disk: ssd + +... diff --git a/yaml/pretty/testdata/pipeline_ports.yml b/yaml/pretty/testdata/pipeline_ports.yml new file mode 100644 index 0000000..efc3dad --- /dev/null +++ b/yaml/pretty/testdata/pipeline_ports.yml @@ -0,0 +1,13 @@ +kind: pipeline +name: default + +steps: +- name: redis + pull: always + detatch: true + image: redis:latest + ports: + - 6379 + - port: 6379 + host: 6379 + protocol: TCP diff --git a/yaml/pretty/testdata/pipeline_ports.yml.golden b/yaml/pretty/testdata/pipeline_ports.yml.golden new file mode 100644 index 0000000..ce2ba71 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_ports.yml.golden @@ -0,0 +1,19 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: redis + pull: always + image: redis:latest + ports: + - 6379 + - port: 6379 + host: 6379 + protocol: TCP + +... diff --git a/yaml/pretty/testdata/pipeline_push.yml b/yaml/pretty/testdata/pipeline_push.yml new file mode 100644 index 0000000..52e26e3 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_push.yml @@ -0,0 +1,6 @@ +kind: pipeline +name: default + +steps: +- name: test_build + push: octocat/hello-world \ No newline at end of file diff --git a/yaml/pretty/testdata/pipeline_push.yml.golden b/yaml/pretty/testdata/pipeline_push.yml.golden new file mode 100644 index 0000000..0b13292 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_push.yml.golden @@ -0,0 +1,13 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: test_build + push: octocat/hello-world + +... diff --git a/yaml/pretty/testdata/pipeline_resources.yml b/yaml/pretty/testdata/pipeline_resources.yml new file mode 100644 index 0000000..1233c25 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_resources.yml @@ -0,0 +1,15 @@ +kind: pipeline +name: default + +steps: + - name: build + image: golang + commands: + - go build + resources: + limits: + cpu: 2 + memory: '100Mi' + requests: + cpu: 100m + memory: '50Mi' diff --git a/yaml/pretty/testdata/pipeline_resources.yml.golden b/yaml/pretty/testdata/pipeline_resources.yml.golden new file mode 100644 index 0000000..c592200 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_resources.yml.golden @@ -0,0 +1,22 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: build + image: golang + commands: + - go build + resources: + limits: + cpu: 2000 + memory: 100MiB + requests: + cpu: 100 + memory: 50MiB + +... diff --git a/yaml/pretty/testdata/pipeline_settings.yml b/yaml/pretty/testdata/pipeline_settings.yml new file mode 100644 index 0000000..b6bbf05 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_settings.yml @@ -0,0 +1,14 @@ +kind: pipeline +name: default + +steps: +- name: notify + image: plugins/slack + settings: + token: + from_secret: token + root: general + labels: + - foo + - bar + - baz diff --git a/yaml/pretty/testdata/pipeline_settings.yml.golden b/yaml/pretty/testdata/pipeline_settings.yml.golden new file mode 100644 index 0000000..f2af22b --- /dev/null +++ b/yaml/pretty/testdata/pipeline_settings.yml.golden @@ -0,0 +1,21 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: notify + image: plugins/slack + settings: + labels: + - foo + - bar + - baz + root: general + token: + from_secret: token + +... diff --git a/yaml/pretty/testdata/pipeline_trigger.yml b/yaml/pretty/testdata/pipeline_trigger.yml new file mode 100644 index 0000000..6d639de --- /dev/null +++ b/yaml/pretty/testdata/pipeline_trigger.yml @@ -0,0 +1,20 @@ +kind: pipeline +name: default + +steps: + - name: test + image: golang + commands: + - go build + - go test -v + +trigger: + branch: + - master + - develop + event: + include: + - push + - pull_request + exclude: + - tag diff --git a/yaml/pretty/testdata/pipeline_trigger.yml.golden b/yaml/pretty/testdata/pipeline_trigger.yml.golden new file mode 100644 index 0000000..79fe59c --- /dev/null +++ b/yaml/pretty/testdata/pipeline_trigger.yml.golden @@ -0,0 +1,27 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: test + image: golang + commands: + - go build + - go test -v + +trigger: + branch: + - master + - develop + event: + include: + - push + - pull_request + exclude: + - tag + +... diff --git a/yaml/pretty/testdata/pipeline_volumes.yml b/yaml/pretty/testdata/pipeline_volumes.yml new file mode 100644 index 0000000..ec7eaeb --- /dev/null +++ b/yaml/pretty/testdata/pipeline_volumes.yml @@ -0,0 +1,30 @@ +--- +kind: pipeline +name: default + +steps: +- name: compile + image: golang + commands: + - go test + - go build + +- name: build + image: docker + commands: + - docker build . + volumes: + - name: sock + path: /var/run/docker.sock + +volumes: +- name: temp + temp: + medium: memory +- name: sock + host: + path: /var/run/docker.sock + +depends_on: +- foo +- bar diff --git a/yaml/pretty/testdata/pipeline_volumes.yml.golden b/yaml/pretty/testdata/pipeline_volumes.yml.golden new file mode 100644 index 0000000..6039de8 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_volumes.yml.golden @@ -0,0 +1,36 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: compile + image: golang + commands: + - go test + - go build + +- name: build + image: docker + commands: + - docker build . + volumes: + - name: sock + path: /var/run/docker.sock + +volumes: +- name: temp + temp: + medium: memory +- name: sock + host: + path: /var/run/docker.sock + +depends_on: +- foo +- bar + +... diff --git a/yaml/pretty/testdata/pipeline_workspace.yml b/yaml/pretty/testdata/pipeline_workspace.yml new file mode 100644 index 0000000..c171387 --- /dev/null +++ b/yaml/pretty/testdata/pipeline_workspace.yml @@ -0,0 +1,13 @@ +kind: pipeline +name: default + +workspace: + base: /go + path: src/github.com/octocat/hello-world + +steps: +- name: test + image: golang + commands: + - go build + - go test -v diff --git a/yaml/pretty/testdata/pipeline_workspace.yml.golden b/yaml/pretty/testdata/pipeline_workspace.yml.golden new file mode 100644 index 0000000..93d6f5d --- /dev/null +++ b/yaml/pretty/testdata/pipeline_workspace.yml.golden @@ -0,0 +1,20 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +workspace: + base: /go + path: src/github.com/octocat/hello-world + +steps: +- name: test + image: golang + commands: + - go build + - go test -v + +... diff --git a/yaml/pretty/testdata/registry.yml b/yaml/pretty/testdata/registry.yml new file mode 100644 index 0000000..4a44cdf --- /dev/null +++ b/yaml/pretty/testdata/registry.yml @@ -0,0 +1,4 @@ +kind: registry +type: encrypted +data: + index.docker.io: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK diff --git a/yaml/pretty/testdata/registry.yml.golden b/yaml/pretty/testdata/registry.yml.golden new file mode 100644 index 0000000..9b6ee8f --- /dev/null +++ b/yaml/pretty/testdata/registry.yml.golden @@ -0,0 +1,8 @@ +--- +kind: registry +type: encrypted +data: + index.docker.io: > + N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + +... diff --git a/yaml/pretty/testdata/secret.yml b/yaml/pretty/testdata/secret.yml new file mode 100644 index 0000000..0553b00 --- /dev/null +++ b/yaml/pretty/testdata/secret.yml @@ -0,0 +1,6 @@ +kind: secret +type: general + +data: + username: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + password: NGZhNjY5YWMxZjhlYzJkNzE1ODlkZDliN2I4MDMwOTEzNGZhZTk3ZjcyNzk5NzNmZmQ3ZWRmNGY0YWJmYjFlMGY3ZmI2MmQ2MmNjMDQ1NDQwNmU5Nzc5NTlmNDEyYzM2YzI1ZjdhOWVkOTc1OTI5YmE5OTY1ZGRhOTk3NTQ1NDAK diff --git a/yaml/pretty/testdata/secret.yml.golden b/yaml/pretty/testdata/secret.yml.golden new file mode 100644 index 0000000..d9b9628 --- /dev/null +++ b/yaml/pretty/testdata/secret.yml.golden @@ -0,0 +1,12 @@ +--- +kind: secret +type: general +data: + password: > + NGZhNjY5YWMxZjhlYzJkNzE1ODlkZDliN2I4MDMwOTEzNGZhZTk3ZjcyNzk5 + NzNmZmQ3ZWRmNGY0YWJmYjFlMGY3ZmI2MmQ2MmNjMDQ1NDQwNmU5Nzc5NTlm + NDEyYzM2YzI1ZjdhOWVkOTc1OTI5YmE5OTY1ZGRhOTk3NTQ1NDAK + username: > + N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + +... diff --git a/yaml/pretty/testdata/secret_extern.yml b/yaml/pretty/testdata/secret_extern.yml new file mode 100644 index 0000000..585a18f --- /dev/null +++ b/yaml/pretty/testdata/secret_extern.yml @@ -0,0 +1,10 @@ +kind: secret + +external_data: + username: + path: secrets/data/docker + name: username + + password: + path: secrets/data/docker + name: password diff --git a/yaml/pretty/testdata/secret_extern.yml.golden b/yaml/pretty/testdata/secret_extern.yml.golden new file mode 100644 index 0000000..d2b9b75 --- /dev/null +++ b/yaml/pretty/testdata/secret_extern.yml.golden @@ -0,0 +1,12 @@ +--- +kind: secret +type: general +external_data: + password: + path: secrets/data/docker + name: password + username: + path: secrets/data/docker + name: username + +... diff --git a/yaml/pretty/testdata/services.yml b/yaml/pretty/testdata/services.yml new file mode 100644 index 0000000..c82d0b7 --- /dev/null +++ b/yaml/pretty/testdata/services.yml @@ -0,0 +1,32 @@ +kind: pipeline +name: default + +steps: +- name: test + image: golang + commands: + - go build + - go test -v + +services: +- name: redis + image: redis:localhost + ports: + - 6379 + entrypoint: + - /bin/redis-server + command: + - --port + - 6380 +- name: mysql + image: mysql:latest + ports: + - 3306 + environment: + MYSQL_USERNAME: root + MYSQL_PASSWORD: + from_secret: password + +depends_on: + - foo + - bar diff --git a/yaml/pretty/testdata/services.yml.golden b/yaml/pretty/testdata/services.yml.golden new file mode 100644 index 0000000..dd5a697 --- /dev/null +++ b/yaml/pretty/testdata/services.yml.golden @@ -0,0 +1,40 @@ +--- +kind: pipeline +name: default + +platform: + os: linux + arch: amd64 + +steps: +- name: test + image: golang + commands: + - go build + - go test -v + +services: +- name: redis + image: redis:localhost + entrypoint: + - /bin/redis-server + command: + - --port + - 6380 + ports: + - 6379 + +- name: mysql + image: mysql:latest + environment: + MYSQL_PASSWORD: + from_secret: password + MYSQL_USERNAME: root + ports: + - 3306 + +depends_on: +- foo +- bar + +... diff --git a/yaml/pretty/testdata/signature.yml b/yaml/pretty/testdata/signature.yml new file mode 100644 index 0000000..c08bc25 --- /dev/null +++ b/yaml/pretty/testdata/signature.yml @@ -0,0 +1,2 @@ +kind: signature +hmac: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK diff --git a/yaml/pretty/testdata/signature.yml.golden b/yaml/pretty/testdata/signature.yml.golden new file mode 100644 index 0000000..3e26f1c --- /dev/null +++ b/yaml/pretty/testdata/signature.yml.golden @@ -0,0 +1,5 @@ +--- +kind: signature +hmac: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + +... diff --git a/yaml/pretty/util.go b/yaml/pretty/util.go new file mode 100644 index 0000000..9987946 --- /dev/null +++ b/yaml/pretty/util.go @@ -0,0 +1,78 @@ +package pretty + +import "github.com/drone/drone-yaml/yaml" + +func isPrimative(v interface{}) bool { + switch v.(type) { + case bool, string, int, float64: + return true + case yaml.BytesSize: + return true + case yaml.MilliSize: + return true + default: + return false + } +} + +func isSlice(v interface{}) bool { + switch v.(type) { + case []interface{}: + return true + case []string: + return true + default: + return false + } +} + +func isZero(v interface{}) bool { + switch v := v.(type) { + case bool: + return v == false + case string: + return len(v) == 0 + case int: + return v == 0 + case float64: + return v == 0 + case []interface{}: + return len(v) == 0 + case []string: + return len(v) == 0 + case map[interface{}]interface{}: + return len(v) == 0 + case map[string]string: + return len(v) == 0 + case yaml.BytesSize: + return int64(v) == 0 + default: + return false + } +} + +func isQuoted(b rune) bool { + switch b { + case '#', ',', '[', ']', '{', '}', '&', '*', '!', '|', '>', '\'', '"', '%', '@', '`': + return true + case '\a', '\b', '\f', '\n', '\r', '\t', '\v': + return true + default: + return false + } +} + +func chunk(s string, chunkSize int) []string { + if len(s) == 0 { + return []string{s} + } + var chunks []string + for i := 0; i < len(s); i += chunkSize { + nn := i + chunkSize + if nn > len(s) { + nn = len(s) + } + chunks = append(chunks, s[i:nn]) + } + return chunks +} diff --git a/yaml/pretty/util_test.go b/yaml/pretty/util_test.go new file mode 100644 index 0000000..d5ff48d --- /dev/null +++ b/yaml/pretty/util_test.go @@ -0,0 +1,28 @@ +package pretty + +import ( + "strings" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestChunk(t *testing.T) { + s := strings.Join(testChunk, "") + got, want := chunk(s, 64), testChunk + if diff := cmp.Diff(got, want); diff != "" { + t.Errorf("Unexpected chunk value") + t.Log(diff) + } +} + +var testChunk = []string{ + "ZDllMjFjZDg3Zjk0ZWFjZDRhMjdhMTA1ZDQ1OTVkYTA1ODBjMTk0ZWVlZjQyNmU4", + "N2RiNTIwZjg0NWQwYjcyYjE3MmFmZDIyYzg3NTQ1N2YyYzgxODhjYjJmNDhhOTFj", + "ZjdhMzA0YjEzYWFlMmYxMTIwMmEyM2Q1YjQ5Yjg2ZmMK", +} + +var testScalar = `> + ZDllMjFjZDg3Zjk0ZWFjZDRhMjdhMTA1ZDQ1OTVkYTA1ODBjMTk0ZWVlZjQyNmU4 + N2RiNTIwZjg0NWQwYjcyYjE3MmFmZDIyYzg3NTQ1N2YyYzgxODhjYjJmNDhhOTFj + ZjdhMzA0YjEzYWFlMmYxMTIwMmEyM2Q1YjQ5Yjg2ZmMK` diff --git a/yaml/pretty/writer.go b/yaml/pretty/writer.go new file mode 100644 index 0000000..b7e8051 --- /dev/null +++ b/yaml/pretty/writer.go @@ -0,0 +1,319 @@ +package pretty + +import ( + "bytes" + "fmt" + "sort" + "strconv" + + "github.com/drone/drone-yaml/yaml" +) + +// TODO rename WriteTag to WriteKey +// TODO rename WriteTagValue to WriteKeyValue + +// ESCAPING: +// +// The string starts with a special character: +// One of !#%@&*`?|>{[ or -. +// The string starts or ends with whitespace characters. +// The string contains : or # character sequences. +// The string ends with a colon. +// The value looks like a number or boolean (123, 1.23, true, false, null) but should be a string. + +// Implement state pooling. See: +// https://golang.org/src/fmt/print.go#L131 + +type writer interface { + // Indent appends padding to the buffer. + Indent() + + // IndentIncrease inreases indentation. + IndentIncrease() + + // IndentDecrese decreases indentation. + IndentDecrease() + + // Write appends the contents of p to the buffer. + Write(p []byte) (n int, err error) + + // WriteByte appends the contents of b to the buffer. + WriteByte(b byte) error + + // WriteString appends the contents of s to the buffer. + WriteString(s string) (n int, err error) + + // WriteTag appends the key to the buffer. + WriteTag(v interface{}) + + // WriteTag appends the keypair to the buffer. + WriteTagValue(k, v interface{}) +} + +// +// node writer +// + +type baseWriter struct { + bytes.Buffer + depth int +} + +func (w *baseWriter) Indent() { + for i := 0; i < w.depth; i++ { + w.WriteString(" ") + } +} + +func (w *baseWriter) IndentIncrease() { + w.depth++ +} + +func (w *baseWriter) IndentDecrease() { + w.depth-- +} + +func (w *baseWriter) WriteTag(v interface{}) { + w.WriteByte('\n') + w.Indent() + writeValue(w, v) + w.WriteByte(':') +} + +func (w *baseWriter) WriteTagValue(k, v interface{}) { + if isZero(v) { + return + } + w.WriteTag(k) + if isPrimative(v) { + w.WriteByte(' ') + writeValue(w, v) + } else if isSlice(v) { + w.WriteByte('\n') + w.Indent() + writeValue(w, v) + } else { + w.depth++ + w.WriteByte('\n') + w.Indent() + writeValue(w, v) + w.depth-- + } +} + +// +// sequence writer +// + +type indexWriter struct { + writer + index int +} + +func (w *indexWriter) WriteTag(v interface{}) { + w.WriteByte('\n') + if w.index == 0 { + w.IndentDecrease() + w.Indent() + w.IndentIncrease() + w.WriteByte('-') + w.WriteByte(' ') + } else { + w.Indent() + } + writeValue(w, v) + w.WriteByte(':') + w.index++ +} + +func (w *indexWriter) WriteTagValue(k, v interface{}) { + if isZero(v) { + return + } + w.WriteTag(k) + if isPrimative(v) { + w.WriteByte(' ') + writeValue(w, v) + } else if isSlice(v) { + w.WriteByte('\n') + w.Indent() + writeValue(w, v) + } else { + w.IndentIncrease() + w.WriteByte('\n') + w.Indent() + writeValue(w, v) + w.IndentDecrease() + } +} + +// +// helper functions +// + +func writeBool(w writer, v bool) { + w.WriteString( + strconv.FormatBool(v), + ) +} + +func writeFloat(w writer, v float64) { + w.WriteString( + strconv.FormatFloat(v, 'g', -1, 64), + ) +} + +func writeInt(w writer, v int) { + w.WriteString( + strconv.Itoa(v), + ) +} + +func writeEncode(w writer, v string) { + if len(v) == 0 { + w.WriteByte('"') + w.WriteByte('"') + return + } + for _, b := range v { + if isQuoted(b) { + fmt.Fprintf(w, "%q", v) + return + } + } + w.WriteString(v) +} + +func writeValue(w writer, v interface{}) { + if v == nil { + w.WriteByte('~') + return + } + switch v := v.(type) { + case bool, int, float64, string: + writeScalar(w, v) + case []interface{}: + writeSequence(w, v) + case []string: + writeSequenceStr(w, v) + case map[interface{}]interface{}: + writeMapping(w, v) + case map[string]string: + writeMappingStr(w, v) + case yaml.BytesSize: + writeValue(w, v.String()) + case yaml.MilliSize: + writeValue(w, v.String()) + } +} + +func writeScalar(w writer, v interface{}) { + switch v := v.(type) { + case bool: + writeBool(w, v) + case int: + writeInt(w, v) + case float64: + writeFloat(w, v) + case string: + writeEncode(w, v) + } +} + +func writeSequence(w writer, v []interface{}) { + if len(v) == 0 { + w.WriteByte('[') + w.WriteByte(']') + return + } + for i, v := range v { + if i != 0 { + w.WriteByte('\n') + w.Indent() + } + w.WriteByte('-') + w.WriteByte(' ') + w.IndentIncrease() + writeValue(w, v) + w.IndentDecrease() + } +} + +func writeSequenceStr(w writer, v []string) { + if len(v) == 0 { + w.WriteByte('[') + w.WriteByte(']') + return + } + for i, v := range v { + if i != 0 { + w.WriteByte('\n') + w.Indent() + } + w.WriteByte('-') + w.WriteByte(' ') + writeEncode(w, v) + } +} + +func writeMapping(w writer, v map[interface{}]interface{}) { + if len(v) == 0 { + w.WriteByte('{') + w.WriteByte('}') + return + } + var keys []string + for k := range v { + s := fmt.Sprint(k) + keys = append(keys, s) + } + sort.Strings(keys) + for i, k := range keys { + v := v[k] + if i != 0 { + w.WriteByte('\n') + w.Indent() + } + writeEncode(w, k) + w.WriteByte(':') + if v == nil || isPrimative(v) || isZero(v) { + w.WriteByte(' ') + writeValue(w, v) + } else { + slice := isSlice(v) + if !slice { + w.IndentIncrease() + } + w.WriteByte('\n') + w.Indent() + writeValue(w, v) + if !slice { + w.IndentDecrease() + } + } + } +} + +func writeMappingStr(w writer, v map[string]string) { + if len(v) == 0 { + w.WriteByte('{') + w.WriteByte('}') + return + } + var keys []string + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + for i, k := range keys { + v := v[k] + if i != 0 { + w.WriteByte('\n') + w.Indent() + } + writeEncode(w, k) + w.WriteByte(':') + w.WriteByte(' ') + writeEncode(w, v) + } +} diff --git a/yaml/pretty/writer_test.go b/yaml/pretty/writer_test.go new file mode 100644 index 0000000..9a8fc15 --- /dev/null +++ b/yaml/pretty/writer_test.go @@ -0,0 +1,54 @@ +package pretty + +import ( + "strings" + "testing" + + "gopkg.in/yaml.v2" +) + +// this unit tests pretty prints a complex yaml structure +// to ensure we have common use cases covered. +func TestWriteComplexValue(t *testing.T) { + block := map[interface{}]interface{}{} + err := yaml.Unmarshal([]byte(testComplexValue), &block) + if err != nil { + t.Error(err) + return + } + + b := new(baseWriter) + writeValue(b, block) + got, want := b.String(), strings.TrimSpace(testComplexValue) + if got != want { + t.Errorf("Unexpected block format") + print(got) + } +} + +var testComplexValue = ` +a: b +c: +- d +- e +f: + g: h + i: + - j + - k + - l: m + o: p + q: + - r + - s: ~ + - {} + - [] + - ~ +t: {} +u: [] +v: 1 +w: true +x: ~ +z: "#y" +zz: "\nz\n" +"{z}": z` diff --git a/yaml/push.go b/yaml/push.go new file mode 100644 index 0000000..d65b2b2 --- /dev/null +++ b/yaml/push.go @@ -0,0 +1,25 @@ +package yaml + +type ( + // Push configures a Docker push. + Push struct { + Image string `json:"image,omitempty"` + } + + // push is a tempoary type used to unmarshal + // the Push struct when long format is used. + push struct { + Image string `json:"image,omitempty"` + } +) + +// UnmarshalYAML implements yaml unmarshalling. +func (p *Push) UnmarshalYAML(unmarshal func(interface{}) error) error { + d := new(push) + err := unmarshal(&d.Image) + if err != nil { + err = unmarshal(d) + } + p.Image = d.Image + return err +} diff --git a/yaml/push_test.go b/yaml/push_test.go new file mode 100644 index 0000000..a7901b1 --- /dev/null +++ b/yaml/push_test.go @@ -0,0 +1,44 @@ +package yaml + +import ( + "testing" + + "gopkg.in/yaml.v2" +) + +func TestPush(t *testing.T) { + tests := []struct { + yaml string + image string + }{ + { + yaml: "foo", + image: "foo", + }, + { + yaml: "{ image: foo }", + image: "foo", + }, + } + for _, test := range tests { + in := []byte(test.yaml) + out := new(Push) + err := yaml.Unmarshal(in, out) + if err != nil { + t.Error(err) + return + } + if got, want := out.Image, test.image; got != want { + t.Errorf("Want Image %q, got %q", want, got) + } + } +} + +func TestPushError(t *testing.T) { + in := []byte("[]") + out := new(Push) + err := yaml.Unmarshal(in, out) + if err == nil { + t.Errorf("Expect unmarshal error") + } +} diff --git a/yaml/registry.go b/yaml/registry.go new file mode 100644 index 0000000..40c68dd --- /dev/null +++ b/yaml/registry.go @@ -0,0 +1,30 @@ +package yaml + +import "errors" + +type ( + // Registry is a resource that provides encrypted + // registry credentials and pointers to external + // registry credentials (e.g. from vault). + Registry struct { + Version string `json:"version,omitempt"` + Kind string `json:"kind,omitempty"` + Type string `json:"type,omitempty"` + + Data map[string]string `json:"data,omitempty"` + } +) + +// GetVersion returns the resource version. +func (r *Registry) GetVersion() string { return r.Version } + +// GetKind returns the resource kind. +func (r *Registry) GetKind() string { return r.Kind } + +// Validate returns an error if the registry is invalid. +func (r *Registry) Validate() error { + if len(r.Data) == 0 { + return errors.New("yaml: invalid registry resource") + } + return nil +} diff --git a/yaml/registry_test.go b/yaml/registry_test.go new file mode 100644 index 0000000..8655b65 --- /dev/null +++ b/yaml/registry_test.go @@ -0,0 +1,34 @@ +package yaml + +import "testing" + +func TestRegistryUnmarshal(t *testing.T) { + diff, err := diff("testdata/registry.yml") + if err != nil { + t.Error(err) + } + if diff != "" { + t.Error("Failed to parse registry") + t.Log(diff) + } +} + +func TestRegistryValidate(t *testing.T) { + registry := new(Registry) + + registry.Data = map[string]string{"index.drone.io": ""} + if err := registry.Validate(); err != nil { + t.Error(err) + return + } + + registry.Data = map[string]string{} + if err := registry.Validate(); err == nil { + t.Errorf("Expect invalid registry error") + } + + registry.Data = nil + if err := registry.Validate(); err == nil { + t.Errorf("Expect invalid registry error") + } +} diff --git a/yaml/secret.go b/yaml/secret.go new file mode 100644 index 0000000..d2b1e24 --- /dev/null +++ b/yaml/secret.go @@ -0,0 +1,37 @@ +package yaml + +import "errors" + +type ( + // Secret is a resource that provides encrypted data + // and pointers to external data (i.e. from vault). + Secret struct { + Version string `json:"version,omitempty"` + Kind string `json:"kind,omitempty"` + Type string `json:"type,omitempty"` + + Data map[string]string `json:"data,omitempty"` + External map[string]ExternalData `json:"external_data,omitempty" yaml:"external_data"` + } + + // ExternalData defines the path and name of external + // data located in an external or remote storage system. + ExternalData struct { + Path string `json:"path,omitempty"` + Name string `json:"name,omitempty"` + } +) + +// GetVersion returns the resource version. +func (s *Secret) GetVersion() string { return s.Version } + +// GetKind returns the resource kind. +func (s *Secret) GetKind() string { return s.Kind } + +// Validate returns an error if the secret is invalid. +func (s *Secret) Validate() error { + if len(s.Data) == 0 && len(s.External) == 0 { + return errors.New("yaml: invalid secret resource") + } + return nil +} diff --git a/yaml/secret_test.go b/yaml/secret_test.go new file mode 100644 index 0000000..dfb00aa --- /dev/null +++ b/yaml/secret_test.go @@ -0,0 +1,36 @@ +package yaml + +import ( + "testing" +) + +func TestSecretUnmarshal(t *testing.T) { + diff, err := diff("testdata/secret.yml") + if err != nil { + t.Error(err) + } + if diff != "" { + t.Error("Failed to parse secret") + t.Log(diff) + } +} + +func TestSecretValidate(t *testing.T) { + secret := new(Secret) + + secret.Data = map[string]string{"foo": "bar"} + if err := secret.Validate(); err != nil { + t.Error(err) + return + } + + secret.Data = map[string]string{} + if err := secret.Validate(); err == nil { + t.Errorf("Expect invalid secret error") + } + + secret.Data = nil + if err := secret.Validate(); err == nil { + t.Errorf("Expect invalid secret error") + } +} diff --git a/yaml/signature.go b/yaml/signature.go new file mode 100644 index 0000000..90d3265 --- /dev/null +++ b/yaml/signature.go @@ -0,0 +1,29 @@ +package yaml + +import "errors" + +type ( + // Signature is a resource that provides an hmac + // signature of combined resources. This signature + // can be used to validate authenticity and prevent + // tampering. + Signature struct { + Version string `json:"version,omitempty"` + Kind string `json:"kind"` + Hmac string `json:"hmac"` + } +) + +// GetVersion returns the resource version. +func (s *Signature) GetVersion() string { return s.Version } + +// GetKind returns the resource kind. +func (s *Signature) GetKind() string { return s.Kind } + +// Validate returns an error if the signature is invalid. +func (s Signature) Validate() error { + if s.Hmac == "" { + return errors.New("yaml: invalid signature. missing hash") + } + return nil +} diff --git a/yaml/signature_test.go b/yaml/signature_test.go new file mode 100644 index 0000000..3455fdc --- /dev/null +++ b/yaml/signature_test.go @@ -0,0 +1,27 @@ +package yaml + +import "testing" + +func TestSignatureUnmarshal(t *testing.T) { + diff, err := diff("testdata/signature.yml") + if err != nil { + t.Error(err) + } + if diff != "" { + t.Error("Failed to parse signature") + t.Log(diff) + } +} + +func TestSignatureValidate(t *testing.T) { + sig := Signature{Hmac: "1234"} + if err := sig.Validate(); err != nil { + t.Error(err) + return + } + + sig.Hmac = "" + if err := sig.Validate(); err == nil { + t.Errorf("Expect invalid signature error") + } +} diff --git a/yaml/signer/signer.go b/yaml/signer/signer.go new file mode 100644 index 0000000..9528571 --- /dev/null +++ b/yaml/signer/signer.go @@ -0,0 +1,151 @@ +package signer + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "errors" + + "github.com/drone/drone-yaml/yaml" + + goyaml "gopkg.in/yaml.v2" +) + +// ErrInvalidKey is returned when the key is missing or +// is less than 32-bytes. +var ErrInvalidKey = errors.New("signer: key must be 32-bytes") + +// Key represents 32-byte signature. +type Key []byte + +// KeyString is a helper function that returns a Key +// from a string. +func KeyString(s string) Key { + return []byte(s) +} + +// Sign calculates and returns the hmac signature of the +// parsed yaml file. +func Sign(data []byte, key Key) (string, error) { + res, err := yaml.ParseRawBytes(data) + if err != nil { + return "", err + } + hmac, err := sign(res, key) + return hex.EncodeToString(hmac), err +} + +// SignUpdate calculates the hmac signature of the parsed +// yaml file and adds a signature resource. If a signature +// resource already exists, it is replaced. +func SignUpdate(data []byte, key Key) ([]byte, error) { + res, err := yaml.ParseRawBytes(data) + if err != nil { + return nil, err + } + hmac, err := sign(res, key) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + for _, r := range res { + if r.Kind != yaml.KindSignature { + buf.WriteString("---") + buf.WriteByte('\n') + buf.Write(r.Data) + } + } + + buf.WriteString("---") + buf.WriteByte('\n') + buf.WriteString("kind: signature") + buf.WriteByte('\n') + buf.WriteString("hmac: " + hex.EncodeToString(hmac)) + buf.WriteByte('\n') + buf.WriteByte('\n') + buf.WriteString("...") + buf.WriteByte('\n') + return buf.Bytes(), nil +} + +// Verify returns true if the signature of the parsed +// yaml file can be verified. +func Verify(data []byte, key Key) (bool, error) { + res, err := yaml.ParseRawBytes(data) + if err != nil { + return false, err + } + mac1, err := extract(res) + if err != nil { + return false, nil + } + mac2, err := sign(res, key) + if err != nil { + return false, err + } + return hmac.Equal(mac1, mac2), nil +} + +// WriteTo writes the signature to the yaml file. If the +// signature already exists it is removed, and the new +// signature is appended to the end of the document. +func WriteTo(data []byte, hmac string) ([]byte, error) { + res, err := yaml.ParseRawBytes(data) + return upsert(res, hmac), err +} + +// helper function extracts the hex-encoded signature +// resource from the parsed resource list. +func extract(res []*yaml.RawResource) ([]byte, error) { + for _, r := range res { + if r.Kind == yaml.KindSignature { + out := new(yaml.Signature) + err := goyaml.Unmarshal(r.Data, out) + if err != nil { + return nil, err + } + return hex.DecodeString(out.Hmac) + } + } + return nil, errors.New("yaml: missing signature") +} + +// helper function generates a hex-encoded signature +// based on the parsed resource list. +func sign(resources []*yaml.RawResource, key Key) ([]byte, error) { + if len(key) < 32 { + return nil, ErrInvalidKey + } + h := hmac.New(sha256.New, key) + for _, r := range resources { + if r.Kind != yaml.KindSignature { + h.Write(r.Data) + } + } + return h.Sum(nil), nil +} + +// helper function inserts or updates the hmac signature +// into the yaml document, and returns an updated copy. +func upsert(res []*yaml.RawResource, hmac string) []byte { + var buf bytes.Buffer + for _, r := range res { + if r.Kind != yaml.KindSignature { + buf.WriteString("---") + buf.WriteByte('\n') + buf.Write(r.Data) + } + } + buf.WriteString("---") + buf.WriteByte('\n') + buf.WriteString("kind: signature") + buf.WriteByte('\n') + buf.WriteString("hmac: " + hmac) + buf.WriteByte('\n') + buf.WriteByte('\n') + buf.WriteString("...") + buf.WriteByte('\n') + return buf.Bytes() +} diff --git a/yaml/signer/signer_test.go b/yaml/signer/signer_test.go new file mode 100644 index 0000000..864d7fb --- /dev/null +++ b/yaml/signer/signer_test.go @@ -0,0 +1,142 @@ +package signer + +import ( + "io/ioutil" + "testing" +) + +func TestSign(t *testing.T) { + in, err := ioutil.ReadFile("testdata/signed.yml") + if err != nil { + t.Error(err) + return + } + + key := KeyString("589396227fff5a93ba934965e8735f88") + want := "389cf92a7472870783a9a5ea77b4abe58a4bb67ba58e1e7293e943aee314aedc" + got, err := Sign(in, key) + if err != nil { + t.Error(err) + } + if got != want { + t.Errorf("Expected hash %s, got %s", want, got) + } + + verified, err := Verify(in, key) + if err != nil { + t.Error(err) + } + if !verified { + t.Errorf("Expected signature verified") + } +} + +func TestVerify_Invalid(t *testing.T) { + in, err := ioutil.ReadFile("testdata/invalid_signature.yml") + if err != nil { + t.Error(err) + return + } + key := KeyString("c953bd41ad0f75848a78ccd54d3861fa") + verified, err := Verify(in, key) + if err != nil { + t.Error(err) + } + if verified { + t.Errorf("Expected signature verification failure") + } +} + +func TestVerify_InvalidKey(t *testing.T) { + in, err := ioutil.ReadFile("testdata/signed.yml") + if err != nil { + t.Error(err) + return + } + key := []byte("this-is-an-invalid-key") + _, err = Verify(in, key) + if err != ErrInvalidKey { + t.Errorf("Expected ErrInvalidKey") + } +} + +// This test verifies that signature verification +// fails if no signature is present in the yaml. +func TestVerify_MissingSignature(t *testing.T) { + in, err := ioutil.ReadFile("testdata/missing_signature.yml") + if err != nil { + t.Error(err) + return + } + key := KeyString("589396227fff5a93ba934965e8735f88") + verified, err := Verify(in, key) + if err != nil { + t.Error(err) + } + if verified { + t.Errorf("Expected signature verification failure") + } +} + +// The test verifies that the SignUpdate function signs the +// configuraiton and appends the signature. +func TestSignUpdate(t *testing.T) { + before, err := ioutil.ReadFile("testdata/invalid_signature.yml") + if err != nil { + t.Error(err) + return + } + + key := KeyString("589396227fff5a93ba934965e8735f88") + after, err := SignUpdate(before, key) + if err != nil { + t.Error(err) + return + } + + verified, err := Verify(after, key) + if err != nil { + t.Error(err) + } + if !verified { + t.Errorf("Expected signature verified") + } +} + +// The test verifies that the SignUpdate function signs the +// configuraiton and appends the signature. +func TestSignUpdate_Append(t *testing.T) { + before, err := ioutil.ReadFile("testdata/missing_signature.yml") + if err != nil { + t.Error(err) + return + } + + key := KeyString("589396227fff5a93ba934965e8735f88") + after, err := SignUpdate(before, key) + if err != nil { + t.Error(err) + return + } + + verified, err := Verify(after, key) + if err != nil { + t.Error(err) + } + if !verified { + t.Errorf("Expected signature verified") + } +} + +func TestSignUpdate_InvalidKey(t *testing.T) { + in, err := ioutil.ReadFile("testdata/signed.yml") + if err != nil { + t.Error(err) + return + } + key := KeyString("this-is-an-invalid-key") + _, err = SignUpdate(in, key) + if err != ErrInvalidKey { + t.Errorf("Expected ErrInvalidKey") + } +} diff --git a/yaml/signer/testdata/invalid_signature.yml b/yaml/signer/testdata/invalid_signature.yml new file mode 100644 index 0000000..7f274ae --- /dev/null +++ b/yaml/signer/testdata/invalid_signature.yml @@ -0,0 +1,22 @@ +--- +kind: pipeline +name: default + +pipeline: +- name: build + image: golang + commands: + - go build + - go test + +--- +kind: secret +data: + password: YjgwNDc4ZDY4NmQzNzQzYjNkYmUwYmE3YjMwOTM2OWUK + username: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + +--- +kind: signature +hmac: 8a4f92a79a5eabb67ba5389c77b4abe58472870783ae1e7293e943aee314aedc + +... diff --git a/yaml/signer/testdata/missing_signature.yml b/yaml/signer/testdata/missing_signature.yml new file mode 100644 index 0000000..4d5ac4d --- /dev/null +++ b/yaml/signer/testdata/missing_signature.yml @@ -0,0 +1,12 @@ +--- +kind: pipeline +name: default + +pipeline: +- name: build + image: golang + commands: + - go build + - go test + +... diff --git a/yaml/signer/testdata/signed.yml b/yaml/signer/testdata/signed.yml new file mode 100644 index 0000000..154a82c --- /dev/null +++ b/yaml/signer/testdata/signed.yml @@ -0,0 +1,22 @@ +--- +kind: pipeline +name: default + +pipeline: +- name: build + image: golang + commands: + - go build + - go test + +--- +kind: secret +data: + password: YjgwNDc4ZDY4NmQzNzQzYjNkYmUwYmE3YjMwOTM2OWUK + username: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + +--- +kind: signature +hmac: 389cf92a7472870783a9a5ea77b4abe58a4bb67ba58e1e7293e943aee314aedc + +... diff --git a/yaml/testdata/cron.yml b/yaml/testdata/cron.yml new file mode 100644 index 0000000..6d86a22 --- /dev/null +++ b/yaml/testdata/cron.yml @@ -0,0 +1,8 @@ +kind: cron +name: nightly + +spec: + schedule: "1 * * * *" + branch: master + deployment: + target: production diff --git a/yaml/testdata/cron.yml.golden b/yaml/testdata/cron.yml.golden new file mode 100644 index 0000000..43ad74f --- /dev/null +++ b/yaml/testdata/cron.yml.golden @@ -0,0 +1,13 @@ +[ + { + "kind": "cron", + "name": "nightly", + "spec": { + "schedule": "1 * * * *", + "branch": "master", + "deployment": { + "target": "production" + } + } + } +] \ No newline at end of file diff --git a/yaml/testdata/manifest.yml b/yaml/testdata/manifest.yml new file mode 100644 index 0000000..a024724 --- /dev/null +++ b/yaml/testdata/manifest.yml @@ -0,0 +1,12 @@ +--- +kind: pipeline +--- +kind: registry +--- +kind: secret +--- +kind: cron +--- +kind: signature +hmac: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK +... \ No newline at end of file diff --git a/yaml/testdata/manifest.yml.golden b/yaml/testdata/manifest.yml.golden new file mode 100644 index 0000000..dd09e62 --- /dev/null +++ b/yaml/testdata/manifest.yml.golden @@ -0,0 +1,18 @@ +[ + { + "kind": "pipeline" + }, + { + "kind": "registry" + }, + { + "kind": "secret" + }, + { + "kind": "cron" + }, + { + "kind": "signature", + "hmac": "N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK" + } +] \ No newline at end of file diff --git a/yaml/testdata/pipeline.yml b/yaml/testdata/pipeline.yml new file mode 100644 index 0000000..98d29f9 --- /dev/null +++ b/yaml/testdata/pipeline.yml @@ -0,0 +1,81 @@ +kind: pipeline +name: default + +platform: + os: windows + arch: arm64 + variant: 7 + version: 1803 + +workspace: + base: /go + path: src/github.com/drone/go-yaml + +clone: + disable: true + depth: 50 + +steps: +- name: test + image: golang + pull: always + shell: bash + commands: + - go get + - go test + environment: + GOOS: windows + GOARCH: arm + privileged: true + when: + branch: + - master + - develop + +- name: build + build: + image: octocat/hello-world + context: . + dockerfile: Dockerfile + labels: + foo: bar + baz: qux + cache_from: + - octocat/knife-spoon + +- name: push + push: octocat/hello-world + +services: +- name: redis + image: redis:latest + detach: true + entrypoint: [ /bin/redis-server ] + command: [ --port, 6380 ] + ports: + - 6380 + working_dir: /data + volumes: + - name: data + path: /data + failure: ignore + environment: + REDIS_USERNAME: foo + REDIS_PASSWORD: bar + +volumes: +- name: data + temp: {} +- name: other + host: + path: /tmp/data + +trigger: + branch: + include: + - master + - develop + +depends_on: +- foo +- bar diff --git a/yaml/testdata/pipeline.yml.golden b/yaml/testdata/pipeline.yml.golden new file mode 100644 index 0000000..c76bb0c --- /dev/null +++ b/yaml/testdata/pipeline.yml.golden @@ -0,0 +1,129 @@ +[ + { + "kind": "pipeline", + "name": "default", + "clone": { + "disable": true, + "depth": 50 + }, + "depends_on": [ + "foo", + "bar" + ], + "platform": { + "os": "windows", + "arch": "arm64", + "variant": "7", + "version": "1803" + }, + "services": [ + { + "command": [ + "--port", + "6380" + ], + "detach": true, + "entrypoint": [ + "/bin/redis-server" + ], + "environment": { + "REDIS_PASSWORD": { + "value": "bar" + }, + "REDIS_USERNAME": { + "value": "foo" + } + }, + "failure": "ignore", + "image": "redis:latest", + "name": "redis", + "ports": [ + { + "port": 6380 + } + ], + "volumes": [ + { + "name": "data", + "path": "/data" + } + ], + "working_dir": "/data" + } + ], + "steps": [ + { + "commands": [ + "go get", + "go test" + ], + "environment": { + "GOARCH": { + "value": "arm" + }, + "GOOS": { + "value": "windows" + } + }, + "image": "golang", + "name": "test", + "privileged": true, + "pull": "always", + "shell": "bash", + "when": { + "branch": { + "include": [ + "master", + "develop" + ] + } + } + }, + { + "build": { + "cache_from": [ + "octocat/knife-spoon" + ], + "context": ".", + "dockerfile": "Dockerfile", + "image": "octocat/hello-world", + "labels": { + "baz": "qux", + "foo": "bar" + } + }, + "name": "build" + }, + { + "name": "push", + "push": { + "image": "octocat/hello-world" + } + } + ], + "trigger": { + "branch": { + "include": [ + "master", + "develop" + ] + } + }, + "volumes": [ + { + "name": "data", + "temp": {} + }, + { + "name": "other", + "host": { + "path": "/tmp/data" + } + } + ], + "workspace": { + "base": "/go", + "path": "src/github.com/drone/go-yaml" + } + } +] \ No newline at end of file diff --git a/yaml/testdata/registry.yml b/yaml/testdata/registry.yml new file mode 100644 index 0000000..6833265 --- /dev/null +++ b/yaml/testdata/registry.yml @@ -0,0 +1,5 @@ +--- +kind: registry +type: encrypted +data: + index.docker.io: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK diff --git a/yaml/testdata/registry.yml.golden b/yaml/testdata/registry.yml.golden new file mode 100644 index 0000000..1ed670b --- /dev/null +++ b/yaml/testdata/registry.yml.golden @@ -0,0 +1,9 @@ +[ + { + "kind": "registry", + "type": "encrypted", + "data": { + "index.docker.io": "N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK" + } + } +] \ No newline at end of file diff --git a/yaml/testdata/secret.yml b/yaml/testdata/secret.yml new file mode 100644 index 0000000..0c92e0d --- /dev/null +++ b/yaml/testdata/secret.yml @@ -0,0 +1,7 @@ +--- +kind: secret +type: encrypted + +data: + username: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK + password: YjgwNDc4ZDY4NmQzNzQzYjNkYmUwYmE3YjMwOTM2OWUK diff --git a/yaml/testdata/secret.yml.golden b/yaml/testdata/secret.yml.golden new file mode 100644 index 0000000..93007fe --- /dev/null +++ b/yaml/testdata/secret.yml.golden @@ -0,0 +1,10 @@ +[ + { + "kind": "secret", + "type": "encrypted", + "data": { + "password": "YjgwNDc4ZDY4NmQzNzQzYjNkYmUwYmE3YjMwOTM2OWUK", + "username": "N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK" + } + } +] \ No newline at end of file diff --git a/yaml/testdata/signature.yml b/yaml/testdata/signature.yml new file mode 100644 index 0000000..2212c6a --- /dev/null +++ b/yaml/testdata/signature.yml @@ -0,0 +1,3 @@ +--- +kind: signature +hmac: N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK diff --git a/yaml/testdata/signature.yml.golden b/yaml/testdata/signature.yml.golden new file mode 100644 index 0000000..0b1e0b6 --- /dev/null +++ b/yaml/testdata/signature.yml.golden @@ -0,0 +1,6 @@ +[ + { + "kind": "signature", + "hmac": "N2NmYjA3ODQwNTY1ODFlY2E5MGJmOWI1NDk0NDFhMTEK" + } +] \ No newline at end of file diff --git a/yaml/unit.go b/yaml/unit.go new file mode 100644 index 0000000..f55e17b --- /dev/null +++ b/yaml/unit.go @@ -0,0 +1,75 @@ +package yaml + +import ( + "strconv" + "strings" + + "github.com/docker/go-units" +) +// BytesSize stores a human-readable size in bytes, +// kibibytes, mebibytes, gibibytes, or tebibytes +// (eg. "44kiB", "17MiB"). +type BytesSize int64 + +// UnmarshalYAML implements yaml unmarshalling. +func (b *BytesSize) UnmarshalYAML(unmarshal func(interface{}) error) error { + var intType int64 + if err := unmarshal(&intType); err == nil { + *b = BytesSize(intType) + return nil + } + + var stringType string + if err := unmarshal(&stringType); err != nil { + return err + } + + intType, err := units.RAMInBytes(stringType) + if err == nil { + *b = BytesSize(intType) + } + return err +} + +// String returns a human-readable size in bytes, +// kibibytes, mebibytes, gibibytes, or tebibytes +// (eg. "44kiB", "17MiB"). +func (b BytesSize) String() string { + return units.BytesSize(float64(b)) +} + +// MilliSize will convert cpus to millicpus as int64. +// for instance "1" will be converted to 1000 and "100m" to 100 +type MilliSize int64 + +// UnmarshalYAML implements yaml unmarshalling. +func (m *MilliSize) UnmarshalYAML(unmarshal func(interface{}) error) error { + var intType int64 + if err := unmarshal(&intType); err == nil { + *m = MilliSize(intType * 1000) + return nil + } + + var stringType string + if err := unmarshal(&stringType); err != nil { + return err + } + if len(stringType) > 0 { + lastChar := string(stringType[len(stringType)-1:]) + if lastChar == "m" { + // convert to int64 + i, err := strconv.ParseInt(strings.TrimSuffix(stringType, "m"), 10, 64) + if err != nil { + return err + } + *m = MilliSize(i) + } + } + return nil +} + +// String returns a human-readable cpu millis, +// (eg. "1000", "10"). +func (m MilliSize) String() string { + return strconv.FormatInt(int64(m), 10) +} diff --git a/yaml/unit_test.go b/yaml/unit_test.go new file mode 100644 index 0000000..566691f --- /dev/null +++ b/yaml/unit_test.go @@ -0,0 +1,85 @@ +package yaml + +import ( + "testing" + + "gopkg.in/yaml.v2" +) + +func TestBytesSize(t *testing.T) { + tests := []struct { + yaml string + size int64 + text string + }{ + { + yaml: "1KiB", + size: 1024, + text: "1KiB", + }, + { + yaml: "100Mi", + size: 104857600, + text: "100MiB", + }, + { + yaml: "1024", + size: 1024, + text: "1KiB", + }, + } + for _, test := range tests { + in := []byte(test.yaml) + out := BytesSize(0) + err := yaml.Unmarshal(in, &out) + if err != nil { + t.Error(err) + return + } + if got, want := int64(out), test.size; got != want { + t.Errorf("Want byte size %d, got %d", want, got) + } + if got, want := out.String(), test.text; got != want { + t.Errorf("Want byte text %s, got %s", want, got) + } + } +} + +func TestMilliSize(t *testing.T) { + tests := []struct { + yaml string + size int64 + text string + }{ + { + yaml: "100m", + size: 100, + text: "100", + }, + { + yaml: "1", + size: 1000, + text: "1000", + }, + { + yaml: "0m", + size: 0, + text: "0", + }, + } + for _, test := range tests { + in := []byte(test.yaml) + out := MilliSize(0) + err := yaml.Unmarshal(in, &out) + if err != nil { + t.Error(err) + return + } + if got, want := int64(out), test.size; got != want { + t.Errorf("Want millis %d, got %d", want, got) + } + if got, want := out.String(), test.text; got != want { + t.Errorf("Want text %s, got %s", want, got) + } + } +}