From c22000f4bc6e7e8910467c924405838fe29932cf Mon Sep 17 00:00:00 2001 From: Robert Kaussow Date: Fri, 29 Nov 2019 14:45:28 +0100 Subject: [PATCH] initial commit --- .drone.jsonnet | 236 +++++++++++++++++++++++++++ .drone.yml | 363 ++++++++++++++++++++++++++++++++++++++++++ .gitignore | 103 ++++++++++++ Dockerfile | 24 +++ README.md | 1 + bin/git-batch | 7 + gitbatch/__init__.py | 9 ++ gitbatch/__main__.py | 99 ++++++++++++ manifest.tmpl | 24 +++ setup.py | 68 ++++++++ test-requirements.txt | 19 +++ 11 files changed, 953 insertions(+) create mode 100644 .drone.jsonnet create mode 100644 .drone.yml create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 README.md create mode 100755 bin/git-batch create mode 100644 gitbatch/__init__.py create mode 100644 gitbatch/__main__.py create mode 100644 manifest.tmpl create mode 100644 setup.py create mode 100644 test-requirements.txt diff --git a/.drone.jsonnet b/.drone.jsonnet new file mode 100644 index 0000000..e8f4480 --- /dev/null +++ b/.drone.jsonnet @@ -0,0 +1,236 @@ +local PythonVersion(pyversion="3.5") = { + name: "python" + std.strReplace(pyversion, '.', '') + "-pytest", + image: "python:" + pyversion, + pull: "always", + environment: { + PY_COLORS: 1 + }, + commands: [ + "pip install -r test-requirements.txt -qq", + "pip install -qq .", + "git-batch --help", + ], + depends_on: [ + "clone", + ], +}; + +local PipelineLint = { + kind: "pipeline", + name: "lint", + platform: { + os: "linux", + arch: "amd64", + }, + steps: [ + { + name: "flake8", + image: "python:3.7", + pull: "always", + environment: { + PY_COLORS: 1 + }, + commands: [ + "pip install -r test-requirements.txt -qq", + "pip install -qq .", + "flake8 ./gitbatch", + ], + }, + ], + trigger: { + ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"], + }, +}; + +local PipelineTest = { + kind: "pipeline", + name: "test", + platform: { + os: "linux", + arch: "amd64", + }, + steps: [ + PythonVersion(pyversion="3.5"), + PythonVersion(pyversion="3.6"), + PythonVersion(pyversion="3.7"), + PythonVersion(pyversion="3.8"), + ], + trigger: { + ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"], + }, + depends_on: [ + "lint", + ], +}; + +local PipelineSecurity = { + kind: "pipeline", + name: "security", + platform: { + os: "linux", + arch: "amd64", + }, + steps: [ + { + name: "bandit", + image: "python:3.7", + pull: "always", + environment: { + PY_COLORS: 1 + }, + commands: [ + "pip install -r test-requirements.txt -qq", + "pip install -qq .", + "bandit -r ./gitbatch -x ./gitbatch/tests", + ], + }, + ], + depends_on: [ + "test", + ], + trigger: { + ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"], + }, +}; + +local PipelineBuildContainer(arch="amd64") = { + kind: "pipeline", + name: "build-container-" + arch, + platform: { + os: "linux", + arch: arch, + }, + steps: [ + { + name: "build", + image: "python:3.7", + pull: "always", + commands: [ + "python setup.py bdist_wheel", + ] + }, + { + name: "dryrun", + image: "plugins/docker:18-linux-" + arch, + pull: "always", + settings: { + dry_run: true, + dockerfile: "Dockerfile", + repo: "xoxys/git-batch", + username: { "from_secret": "docker_username" }, + password: { "from_secret": "docker_password" }, + }, + when: { + ref: ["refs/pull/**"], + }, + }, + { + name: "publish", + image: "plugins/docker:18-linux-" + arch, + pull: "always", + settings: { + auto_tag: true, + auto_tag_suffix: arch, + dockerfile: "Dockerfile", + repo: "xoxys/git-batch", + username: { "from_secret": "docker_username" }, + password: { "from_secret": "docker_password" }, + }, + when: { + ref: ["refs/heads/master", "refs/tags/**"], + }, + }, + ], + depends_on: [ + "security", + ], + trigger: { + ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"], + }, +}; + +local PipelineNotifications = { + kind: "pipeline", + name: "notifications", + platform: { + os: "linux", + arch: "amd64", + }, + steps: [ + { + image: "plugins/manifest", + name: "manifest", + pull: "always", + settings: { + ignore_missing: true, + auto_tag: true, + username: { from_secret: "docker_username" }, + password: { from_secret: "docker_password" }, + spec: "manifest.tmpl", + }, + when: { + ref: [ + 'refs/heads/master', + 'refs/tags/**', + ], + }, + }, + { + name: "readme", + image: "sheogorath/readme-to-dockerhub", + pull: "always", + environment: { + DOCKERHUB_USERNAME: { from_secret: "docker_username" }, + DOCKERHUB_PASSWORD: { from_secret: "docker_password" }, + DOCKERHUB_REPO_PREFIX: "xoxys", + DOCKERHUB_REPO_NAME: "git-batch", + README_PATH: "README.md", + SHORT_DESCRIPTION: "git-batch" + }, + when: { + ref: [ + 'refs/heads/master', + 'refs/tags/**', + ], + }, + }, + { + name: "microbadger", + image: "plugins/webhook", + pull: "always", + settings: { + urls: { from_secret: "microbadger_url" }, + }, + }, + { + name: "matrix", + image: "plugins/matrix", + settings: { + template: "Status: **{{ build.status }}**
Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}
Message: {{ build.message }}", + roomid: { "from_secret": "matrix_roomid" }, + homeserver: { "from_secret": "matrix_homeserver" }, + username: { "from_secret": "matrix_username" }, + password: { "from_secret": "matrix_password" }, + }, + }, + ], + depends_on: [ + "build-container-amd64", + "build-container-arm64", + "build-container-arm" + ], + trigger: { + ref: ["refs/heads/master", "refs/tags/**"], + status: [ "success", "failure" ], + }, +}; + +[ + PipelineLint, + PipelineTest, + PipelineSecurity, + PipelineBuildContainer(arch="amd64"), + PipelineBuildContainer(arch="arm64"), + PipelineBuildContainer(arch="arm"), + PipelineNotifications, +] diff --git a/.drone.yml b/.drone.yml new file mode 100644 index 0000000..683ad54 --- /dev/null +++ b/.drone.yml @@ -0,0 +1,363 @@ +--- +kind: pipeline +name: lint + +platform: + os: linux + arch: amd64 + +steps: +- name: flake8 + pull: always + image: python:3.7 + commands: + - pip install -r test-requirements.txt -qq + - pip install -qq . + - flake8 ./gitbatch + environment: + PY_COLORS: 1 + +trigger: + ref: + - refs/heads/master + - refs/tags/** + - refs/pull/** + +--- +kind: pipeline +name: test + +platform: + os: linux + arch: amd64 + +steps: +- name: python35-pytest + pull: always + image: python:3.5 + commands: + - pip install -r test-requirements.txt -qq + - pip install -qq . + - git-batch --help + environment: + PY_COLORS: 1 + depends_on: + - clone + +- name: python36-pytest + pull: always + image: python:3.6 + commands: + - pip install -r test-requirements.txt -qq + - pip install -qq . + - git-batch --help + environment: + PY_COLORS: 1 + depends_on: + - clone + +- name: python37-pytest + pull: always + image: python:3.7 + commands: + - pip install -r test-requirements.txt -qq + - pip install -qq . + - git-batch --help + environment: + PY_COLORS: 1 + depends_on: + - clone + +- name: python38-pytest + pull: always + image: python:3.8 + commands: + - pip install -r test-requirements.txt -qq + - pip install -qq . + - git-batch --help + environment: + PY_COLORS: 1 + depends_on: + - clone + +trigger: + ref: + - refs/heads/master + - refs/tags/** + - refs/pull/** + +depends_on: +- lint + +--- +kind: pipeline +name: security + +platform: + os: linux + arch: amd64 + +steps: +- name: bandit + pull: always + image: python:3.7 + commands: + - pip install -r test-requirements.txt -qq + - pip install -qq . + - bandit -r ./gitbatch -x ./gitbatch/tests + environment: + PY_COLORS: 1 + +trigger: + ref: + - refs/heads/master + - refs/tags/** + - refs/pull/** + +depends_on: +- test + +--- +kind: pipeline +name: build-container-amd64 + +platform: + os: linux + arch: amd64 + +steps: +- name: build + pull: always + image: python:3.7 + commands: + - python setup.py bdist_wheel + +- name: dryrun + pull: always + image: plugins/docker:18-linux-amd64 + settings: + dockerfile: Dockerfile + dry_run: true + password: + from_secret: docker_password + repo: xoxys/git-batch + username: + from_secret: docker_username + when: + ref: + - refs/pull/** + +- name: publish + pull: always + image: plugins/docker:18-linux-amd64 + settings: + auto_tag: true + auto_tag_suffix: amd64 + dockerfile: Dockerfile + password: + from_secret: docker_password + repo: xoxys/git-batch + username: + from_secret: docker_username + when: + ref: + - refs/heads/master + - refs/tags/** + +trigger: + ref: + - refs/heads/master + - refs/tags/** + - refs/pull/** + +depends_on: +- security + +--- +kind: pipeline +name: build-container-arm64 + +platform: + os: linux + arch: arm64 + +steps: +- name: build + pull: always + image: python:3.7 + commands: + - python setup.py bdist_wheel + +- name: dryrun + pull: always + image: plugins/docker:18-linux-arm64 + settings: + dockerfile: Dockerfile + dry_run: true + password: + from_secret: docker_password + repo: xoxys/git-batch + username: + from_secret: docker_username + when: + ref: + - refs/pull/** + +- name: publish + pull: always + image: plugins/docker:18-linux-arm64 + settings: + auto_tag: true + auto_tag_suffix: arm64 + dockerfile: Dockerfile + password: + from_secret: docker_password + repo: xoxys/git-batch + username: + from_secret: docker_username + when: + ref: + - refs/heads/master + - refs/tags/** + +trigger: + ref: + - refs/heads/master + - refs/tags/** + - refs/pull/** + +depends_on: +- security + +--- +kind: pipeline +name: build-container-arm + +platform: + os: linux + arch: arm + +steps: +- name: build + pull: always + image: python:3.7 + commands: + - python setup.py bdist_wheel + +- name: dryrun + pull: always + image: plugins/docker:18-linux-arm + settings: + dockerfile: Dockerfile + dry_run: true + password: + from_secret: docker_password + repo: xoxys/git-batch + username: + from_secret: docker_username + when: + ref: + - refs/pull/** + +- name: publish + pull: always + image: plugins/docker:18-linux-arm + settings: + auto_tag: true + auto_tag_suffix: arm + dockerfile: Dockerfile + password: + from_secret: docker_password + repo: xoxys/git-batch + username: + from_secret: docker_username + when: + ref: + - refs/heads/master + - refs/tags/** + +trigger: + ref: + - refs/heads/master + - refs/tags/** + - refs/pull/** + +depends_on: +- security + +--- +kind: pipeline +name: notifications + +platform: + os: linux + arch: amd64 + +steps: +- name: manifest + pull: always + image: plugins/manifest + settings: + auto_tag: true + ignore_missing: true + password: + from_secret: docker_password + spec: manifest.tmpl + username: + from_secret: docker_username + when: + ref: + - refs/heads/master + - refs/tags/** + +- name: readme + pull: always + image: sheogorath/readme-to-dockerhub + environment: + DOCKERHUB_PASSWORD: + from_secret: docker_password + DOCKERHUB_REPO_NAME: git-batch + DOCKERHUB_REPO_PREFIX: xoxys + DOCKERHUB_USERNAME: + from_secret: docker_username + README_PATH: README.md + SHORT_DESCRIPTION: git-batch + when: + ref: + - refs/heads/master + - refs/tags/** + +- name: microbadger + pull: always + image: plugins/webhook + settings: + urls: + from_secret: microbadger_url + +- name: matrix + image: plugins/matrix + settings: + homeserver: + from_secret: matrix_homeserver + password: + from_secret: matrix_password + roomid: + from_secret: matrix_roomid + template: "Status: **{{ build.status }}**
Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}
Message: {{ build.message }}" + username: + from_secret: matrix_username + +trigger: + ref: + - refs/heads/master + - refs/tags/** + status: + - success + - failure + +depends_on: +- build-container-amd64 +- build-container-arm64 +- build-container-arm + +... diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..620329a --- /dev/null +++ b/.gitignore @@ -0,0 +1,103 @@ +# ---> Python +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ +env/ +env*/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject + +# Ignore ide addons +.server-script +.on-save.json +.vscode +.pytest_cache + +pip-wheel-metadata diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6831c82 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.7-alpine + +LABEL maintainer="Robert Kaussow " \ + org.label-schema.name="git-batch" \ + org.label-schema.vcs-url="https://github.com/xoxys/git-batch" \ + org.label-schema.vendor="Robert Kaussow" \ + org.label-schema.schema-version="1.0" + +ENV PY_COLORS=1 + +ADD dist/git_batch-*.whl / + +RUN apk --update add --virtual .build-deps build-base libffi-dev libressl-dev && \ + apk --update add git && \ + pip install --upgrade --no-cache-dir pip && \ + pip install --no-cache-dir --find-links=. git-batch && \ + apk del .build-deps && \ + rm -rf /var/cache/apk/* && \ + rm -rf /root/.cache/ && \ + rm -f git_batch-*.whl + +USER root +CMD [] +ENTRYPOINT ["/usr/local/bin/git-batch"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..bcb3ca4 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# git-batch diff --git a/bin/git-batch b/bin/git-batch new file mode 100755 index 0000000..c40dff2 --- /dev/null +++ b/bin/git-batch @@ -0,0 +1,7 @@ +#!/usr/bin/env python3 + +import sys + +import gitbatch.__main__ + +sys.exit(gitbatch.__main__.main()) diff --git a/gitbatch/__init__.py b/gitbatch/__init__.py new file mode 100644 index 0000000..7bc2316 --- /dev/null +++ b/gitbatch/__init__.py @@ -0,0 +1,9 @@ +"""Default package.""" + +__author__ = "Robert Kaussow" +__project__ = "git-batch" +__version__ = "0.1.0" +__license__ = "MIT" +__maintainer__ = "Robert Kaussow" +__email__ = "mail@geeklabor.de" +__url__ = "https://github.com/xoxys/git-batch" diff --git a/gitbatch/__main__.py b/gitbatch/__main__.py new file mode 100644 index 0000000..f035dc8 --- /dev/null +++ b/gitbatch/__main__.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 + +import argparse +import os +import logging +import sys +import git + +from gitbatch import __version__ +from urllib.parse import urlparse + +logger = logging.getLogger("gitbatch") +formatter = logging.Formatter("[%(levelname)s] %(message)s") + +cmdlog = logging.StreamHandler() +cmdlog.setLevel(logging.ERROR) +cmdlog.setFormatter(formatter) +logger.addHandler(cmdlog) + + +def sysexit(self, code=1): + sys.exit(code) + + +def sysexit_with_message(msg, code=1): + logger.error(str(msg)) + sysexit(code) + + +def normalize_path(path): + if path: + return os.path.abspath(os.path.expanduser(os.path.expandvars(path))) + + +def repos_from_file(src): + repos = [] + with open(src, 'r') as f: + for num, line in enumerate(f, start=1): + repo = {} + line = line.strip() + if line and not line.startswith("#"): + try: + url, branch, dest = [x.strip() for x in line.split(";")] + except ValueError as e: + sysexit_with_message("Wrong numer of delimiters in line {line_num}: {exp}".format( + line_num=num, exp=e)) + + if url: + url_parts = urlparse(url) + + repo["url"] = url + repo["branch"] = branch or "master" + repo["name"] = os.path.basename(url_parts.path) + repo["dest"] = normalize_path(dest) or normalize_path("./{}".format(repo["name"])) + + repos.append(repo) + else: + sysexit_with_message("Repository Url is not set on line {line_num}".format( + line_num=num)) + return repos + + +def repos_clone(repos, ignore_existing): + for repo in repos: + print(repo) + try: + git.Repo.clone_from(repo["url"], repo["dest"], multi_options=["--branch=docs", "--single-branch"]) + except git.exc.GitCommandError as e: + if not ignore_existing: + err_raw = [x.strip() for x in e.stderr.split(":")][2] + err = err_raw.splitlines()[0].split(".")[0] + sysexit_with_message("Git error: {}".format(err)) + else: + pass + + +def main(): + """Run main program.""" + parser = argparse.ArgumentParser( + description=("Clone single branch from all repositories listed in a file")) + parser.add_argument("--version", action="version", version="%(prog)s {}".format(__version__)) + + parser.parse_args() + + input_file_raw = os.environ.get("GIT_BATCH_INPUT_FILE", "./batchfile") + input_file = normalize_path(input_file_raw) + + ignore_existing = os.environ.get("GIT_BATCH_IGNORE_EXISTING", True) + + if os.path.isfile(input_file): + repos = repos_from_file(input_file) + repos_clone(repos, ignore_existing) + else: + sysexit_with_message("The given batch file at '{}' does not exist".format( + os.path.relpath(os.path.join("./", input_file)))) + + +if __name__ == "__main__": + main() diff --git a/manifest.tmpl b/manifest.tmpl new file mode 100644 index 0000000..c4d86a1 --- /dev/null +++ b/manifest.tmpl @@ -0,0 +1,24 @@ +image: xoxys/git-batch:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}} +{{#if build.tags}} +tags: +{{#each build.tags}} + - {{this}} +{{/each}} +{{/if}} +manifests: + - image: xoxys/git-batch:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}amd64 + platform: + architecture: amd64 + os: linux + + - image: xoxys/git-batch:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm64 + platform: + architecture: arm64 + os: linux + variant: v8 + + - image: xoxys/git-batch:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm + platform: + architecture: arm + os: linux + variant: v7 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..f2f5b94 --- /dev/null +++ b/setup.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Setup script for the package.""" + +import io +import os +import re + +from setuptools import find_packages +from setuptools import setup + +PACKAGE_NAME = "gitbatch" + + +def get_property(prop, project): + current_dir = os.path.dirname(os.path.realpath(__file__)) + result = re.search( + r'{}\s*=\s*[\'"]([^\'"]*)[\'"]'.format(prop), + open(os.path.join(current_dir, project, "__init__.py")).read()) + return result.group(1) + + +def get_readme(filename="README.md"): + this = os.path.abspath(os.path.dirname(__file__)) + with io.open(os.path.join(this, filename), encoding="utf-8") as f: + long_description = f.read() + return long_description + + +setup( + name=get_property("__project__", PACKAGE_NAME), + version=get_property("__version__", PACKAGE_NAME), + description="Clone single branch from all repositories listed in a file.", + author=get_property("__author__", PACKAGE_NAME), + author_email=get_property("__email__", PACKAGE_NAME), + url=get_property("__url__", PACKAGE_NAME), + license=get_property("__license__", PACKAGE_NAME), + long_description=get_readme(), + long_description_content_type="text/markdown", + packages=find_packages(exclude=["*.tests", "tests", "tests.*"]), + include_package_data=True, + zip_safe=False, + python_requires=">=3.5", + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", + "Natural Language :: English", + "Operating System :: POSIX", + "Programming Language :: Python :: 3 :: Only", + "Topic :: System :: Installation/Setup", + "Topic :: System :: Systems Administration", + "Topic :: Utilities", + "Topic :: Software Development", + "Topic :: Software Development :: Documentation", + ], + install_requires=[ + "gitpython", + ], + entry_points={ + "console_scripts": [ + "git-batch = gitbatch.__main__:main" + ] + }, + test_suite="tests" +) diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..25c2aaf --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,19 @@ +# open issue +# https://gitlab.com/pycqa/flake8-docstrings/issues/36 +pydocstyle<4.0.0 +flake8 +flake8-colors +flake8-blind-except +flake8-builtins +flake8-colors +flake8-docstrings<=3.0.0 +flake8-isort +flake8-logging-format +flake8-polyfill +flake8-quotes +pep8-naming +wheel +pytest +pytest-mock +pytest-cov +bandit