initial commit

.drone.jsonnet

@@ -0,0 +1,236 @@
+local PythonVersion(pyversion="3.5") = {
+    name: "python" + std.strReplace(pyversion, '.', '') + "-pytest",
+    image: "python:" + pyversion,
+    pull: "always",
+    environment: {
+        PY_COLORS: 1
+    },
+    commands: [
+        "pip install -r test-requirements.txt -qq",
+        "pip install -qq .",
+        "git-batch --help",
+    ],
+    depends_on: [
+        "clone",
+    ],
+};
+
+local PipelineLint = {
+    kind: "pipeline",
+    name: "lint",
+    platform: {
+        os: "linux",
+        arch: "amd64",
+    },
+    steps: [
+        {
+            name: "flake8",
+            image: "python:3.7",
+            pull: "always",
+            environment: {
+                PY_COLORS: 1
+            },
+            commands: [
+                "pip install -r test-requirements.txt -qq",
+                "pip install -qq .",
+                "flake8 ./gitbatch",
+            ],
+        },
+    ],
+    trigger: {
+        ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
+    },
+};
+
+local PipelineTest = {
+    kind: "pipeline",
+    name: "test",
+    platform: {
+        os: "linux",
+        arch: "amd64",
+    },
+    steps: [
+        PythonVersion(pyversion="3.5"),
+        PythonVersion(pyversion="3.6"),
+        PythonVersion(pyversion="3.7"),
+        PythonVersion(pyversion="3.8"),
+    ],
+    trigger: {
+        ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
+    },
+    depends_on: [
+        "lint",
+    ],
+};
+
+local PipelineSecurity = {
+    kind: "pipeline",
+    name: "security",
+    platform: {
+        os: "linux",
+        arch: "amd64",
+    },
+    steps: [
+        {
+            name: "bandit",
+            image: "python:3.7",
+            pull: "always",
+            environment: {
+                PY_COLORS: 1
+            },
+            commands: [
+                "pip install -r test-requirements.txt -qq",
+                "pip install -qq .",
+                "bandit -r ./gitbatch -x ./gitbatch/tests",
+            ],
+        },
+    ],
+    depends_on: [
+        "test",
+    ],
+    trigger: {
+        ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
+    },
+};
+
+local PipelineBuildContainer(arch="amd64") = {
+  kind: "pipeline",
+  name: "build-container-" + arch,
+  platform: {
+    os: "linux",
+    arch: arch,
+  },
+  steps: [
+    {
+      name: "build",
+      image: "python:3.7",
+      pull: "always",
+      commands: [
+          "python setup.py bdist_wheel",
+      ]
+    },
+    {
+      name: "dryrun",
+      image: "plugins/docker:18-linux-" + arch,
+      pull: "always",
+      settings: {
+        dry_run: true,
+        dockerfile: "Dockerfile",
+        repo: "xoxys/git-batch",
+        username: { "from_secret": "docker_username" },
+        password: { "from_secret": "docker_password" },
+      },
+      when: {
+        ref: ["refs/pull/**"],
+      },
+    },
+    {
+      name: "publish",
+      image: "plugins/docker:18-linux-" + arch,
+      pull: "always",
+      settings: {
+        auto_tag: true,
+        auto_tag_suffix: arch,
+        dockerfile: "Dockerfile",
+        repo: "xoxys/git-batch",
+        username: { "from_secret": "docker_username" },
+        password: { "from_secret": "docker_password" },
+      },
+      when: {
+          ref: ["refs/heads/master", "refs/tags/**"],
+      },
+    },
+  ],
+  depends_on: [
+    "security",
+  ],
+  trigger: {
+      ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
+  },
+};
+
+local PipelineNotifications = {
+  kind: "pipeline",
+  name: "notifications",
+  platform: {
+    os: "linux",
+    arch: "amd64",
+  },
+  steps: [
+    {
+      image: "plugins/manifest",
+      name: "manifest",
+      pull: "always",
+      settings: {
+        ignore_missing: true,
+        auto_tag: true,
+        username: { from_secret: "docker_username" },
+        password: { from_secret: "docker_password" },
+        spec: "manifest.tmpl",
+      },
+      when: {
+        ref: [
+          'refs/heads/master',
+          'refs/tags/**',
+        ],
+      },
+    },
+    {
+      name: "readme",
+      image: "sheogorath/readme-to-dockerhub",
+      pull: "always",
+      environment: {
+        DOCKERHUB_USERNAME: { from_secret: "docker_username" },
+        DOCKERHUB_PASSWORD: { from_secret: "docker_password" },
+        DOCKERHUB_REPO_PREFIX: "xoxys",
+        DOCKERHUB_REPO_NAME: "git-batch",
+        README_PATH: "README.md",
+        SHORT_DESCRIPTION: "git-batch"
+      },
+      when: {
+        ref: [
+          'refs/heads/master',
+          'refs/tags/**',
+        ],
+      },
+    },
+    {
+      name: "microbadger",
+      image: "plugins/webhook",
+      pull: "always",
+      settings: {
+        urls: { from_secret: "microbadger_url" },
+      },
+    },
+    {
+      name: "matrix",
+      image: "plugins/matrix",
+      settings: {
+        template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}",
+        roomid: { "from_secret": "matrix_roomid" },
+        homeserver: { "from_secret": "matrix_homeserver" },
+        username: { "from_secret": "matrix_username" },
+        password: { "from_secret": "matrix_password" },
+      },
+    },
+  ],
+  depends_on: [
+    "build-container-amd64",
+    "build-container-arm64",
+    "build-container-arm"
+  ],
+  trigger: {
+    ref: ["refs/heads/master", "refs/tags/**"],
+    status: [ "success", "failure" ],
+  },
+};
+
+[
+    PipelineLint,
+    PipelineTest,
+    PipelineSecurity,
+    PipelineBuildContainer(arch="amd64"),
+    PipelineBuildContainer(arch="arm64"),
+    PipelineBuildContainer(arch="arm"),
+    PipelineNotifications,
+]

.drone.yml

@@ -0,0 +1,363 @@
+---
+kind: pipeline
+name: lint
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: flake8
+  pull: always
+  image: python:3.7
+  commands:
+  - pip install -r test-requirements.txt -qq
+  - pip install -qq .
+  - flake8 ./gitbatch
+  environment:
+    PY_COLORS: 1
+
+trigger:
+  ref:
+  - refs/heads/master
+  - refs/tags/**
+  - refs/pull/**
+
+---
+kind: pipeline
+name: test
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: python35-pytest
+  pull: always
+  image: python:3.5
+  commands:
+  - pip install -r test-requirements.txt -qq
+  - pip install -qq .
+  - git-batch --help
+  environment:
+    PY_COLORS: 1
+  depends_on:
+  - clone
+
+- name: python36-pytest
+  pull: always
+  image: python:3.6
+  commands:
+  - pip install -r test-requirements.txt -qq
+  - pip install -qq .
+  - git-batch --help
+  environment:
+    PY_COLORS: 1
+  depends_on:
+  - clone
+
+- name: python37-pytest
+  pull: always
+  image: python:3.7
+  commands:
+  - pip install -r test-requirements.txt -qq
+  - pip install -qq .
+  - git-batch --help
+  environment:
+    PY_COLORS: 1
+  depends_on:
+  - clone
+
+- name: python38-pytest
+  pull: always
+  image: python:3.8
+  commands:
+  - pip install -r test-requirements.txt -qq
+  - pip install -qq .
+  - git-batch --help
+  environment:
+    PY_COLORS: 1
+  depends_on:
+  - clone
+
+trigger:
+  ref:
+  - refs/heads/master
+  - refs/tags/**
+  - refs/pull/**
+
+depends_on:
+- lint
+
+---
+kind: pipeline
+name: security
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: bandit
+  pull: always
+  image: python:3.7
+  commands:
+  - pip install -r test-requirements.txt -qq
+  - pip install -qq .
+  - bandit -r ./gitbatch -x ./gitbatch/tests
+  environment:
+    PY_COLORS: 1
+
+trigger:
+  ref:
+  - refs/heads/master
+  - refs/tags/**
+  - refs/pull/**
+
+depends_on:
+- test
+
+---
+kind: pipeline
+name: build-container-amd64
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: build
+  pull: always
+  image: python:3.7
+  commands:
+  - python setup.py bdist_wheel
+
+- name: dryrun
+  pull: always
+  image: plugins/docker:18-linux-amd64
+  settings:
+    dockerfile: Dockerfile
+    dry_run: true
+    password:
+      from_secret: docker_password
+    repo: xoxys/git-batch
+    username:
+      from_secret: docker_username
+  when:
+    ref:
+    - refs/pull/**
+
+- name: publish
+  pull: always
+  image: plugins/docker:18-linux-amd64
+  settings:
+    auto_tag: true
+    auto_tag_suffix: amd64
+    dockerfile: Dockerfile
+    password:
+      from_secret: docker_password
+    repo: xoxys/git-batch
+    username:
+      from_secret: docker_username
+  when:
+    ref:
+    - refs/heads/master
+    - refs/tags/**
+
+trigger:
+  ref:
+  - refs/heads/master
+  - refs/tags/**
+  - refs/pull/**
+
+depends_on:
+- security
+
+---
+kind: pipeline
+name: build-container-arm64
+
+platform:
+  os: linux
+  arch: arm64
+
+steps:
+- name: build
+  pull: always
+  image: python:3.7
+  commands:
+  - python setup.py bdist_wheel
+
+- name: dryrun
+  pull: always
+  image: plugins/docker:18-linux-arm64
+  settings:
+    dockerfile: Dockerfile
+    dry_run: true
+    password:
+      from_secret: docker_password
+    repo: xoxys/git-batch
+    username:
+      from_secret: docker_username
+  when:
+    ref:
+    - refs/pull/**
+
+- name: publish
+  pull: always
+  image: plugins/docker:18-linux-arm64
+  settings:
+    auto_tag: true
+    auto_tag_suffix: arm64
+    dockerfile: Dockerfile
+    password:
+      from_secret: docker_password
+    repo: xoxys/git-batch
+    username:
+      from_secret: docker_username
+  when:
+    ref:
+    - refs/heads/master
+    - refs/tags/**
+
+trigger:
+  ref:
+  - refs/heads/master
+  - refs/tags/**
+  - refs/pull/**
+
+depends_on:
+- security
+
+---
+kind: pipeline
+name: build-container-arm
+
+platform:
+  os: linux
+  arch: arm
+
+steps:
+- name: build
+  pull: always
+  image: python:3.7
+  commands:
+  - python setup.py bdist_wheel
+
+- name: dryrun
+  pull: always
+  image: plugins/docker:18-linux-arm
+  settings:
+    dockerfile: Dockerfile
+    dry_run: true
+    password:
+      from_secret: docker_password
+    repo: xoxys/git-batch
+    username:
+      from_secret: docker_username
+  when:
+    ref:
+    - refs/pull/**
+
+- name: publish
+  pull: always
+  image: plugins/docker:18-linux-arm
+  settings:
+    auto_tag: true
+    auto_tag_suffix: arm
+    dockerfile: Dockerfile
+    password:
+      from_secret: docker_password
+    repo: xoxys/git-batch
+    username:
+      from_secret: docker_username
+  when:
+    ref:
+    - refs/heads/master
+    - refs/tags/**
+
+trigger:
+  ref:
+  - refs/heads/master
+  - refs/tags/**
+  - refs/pull/**
+
+depends_on:
+- security
+
+---
+kind: pipeline
+name: notifications
+
+platform:
+  os: linux
+  arch: amd64
+
+steps:
+- name: manifest
+  pull: always
+  image: plugins/manifest
+  settings:
+    auto_tag: true
+    ignore_missing: true
+    password:
+      from_secret: docker_password
+    spec: manifest.tmpl
+    username:
+      from_secret: docker_username
+  when:
+    ref:
+    - refs/heads/master
+    - refs/tags/**
+
+- name: readme
+  pull: always
+  image: sheogorath/readme-to-dockerhub
+  environment:
+    DOCKERHUB_PASSWORD:
+      from_secret: docker_password
+    DOCKERHUB_REPO_NAME: git-batch
+    DOCKERHUB_REPO_PREFIX: xoxys
+    DOCKERHUB_USERNAME:
+      from_secret: docker_username
+    README_PATH: README.md
+    SHORT_DESCRIPTION: git-batch
+  when:
+    ref:
+    - refs/heads/master
+    - refs/tags/**
+
+- name: microbadger
+  pull: always
+  image: plugins/webhook
+  settings:
+    urls:
+      from_secret: microbadger_url
+
+- name: matrix
+  image: plugins/matrix
+  settings:
+    homeserver:
+      from_secret: matrix_homeserver
+    password:
+      from_secret: matrix_password
+    roomid:
+      from_secret: matrix_roomid
+    template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}"
+    username:
+      from_secret: matrix_username
+
+trigger:
+  ref:
+  - refs/heads/master
+  - refs/tags/**
+  status:
+  - success
+  - failure
+
+depends_on:
+- build-container-amd64
+- build-container-arm64
+- build-container-arm
+
+...

.gitignore

@@ -0,0 +1,103 @@
+# ---> Python
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+env/
+env*/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+# Ignore ide addons
+.server-script
+.on-save.json
+.vscode
+.pytest_cache
+
+pip-wheel-metadata

Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.7-alpine
+  
+LABEL maintainer="Robert Kaussow <mail@geeklabor.de>" \
+    org.label-schema.name="git-batch" \
+    org.label-schema.vcs-url="https://github.com/xoxys/git-batch" \
+    org.label-schema.vendor="Robert Kaussow" \
+    org.label-schema.schema-version="1.0"
+
+ENV PY_COLORS=1
+
+ADD dist/git_batch-*.whl /
+
+RUN apk --update add --virtual .build-deps build-base libffi-dev libressl-dev && \
+    apk --update add git && \
+    pip install --upgrade --no-cache-dir pip && \
+    pip install --no-cache-dir --find-links=. git-batch && \
+    apk del .build-deps && \
+    rm -rf /var/cache/apk/* && \
+    rm -rf /root/.cache/  && \
+    rm -f git_batch-*.whl
+
+USER root
+CMD []
+ENTRYPOINT ["/usr/local/bin/git-batch"]

README.md

@@ -0,0 +1 @@
+# git-batch

bin/git-batch

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+  
+import sys
+
+import gitbatch.__main__
+
+sys.exit(gitbatch.__main__.main())

gitbatch/__init__.py

@@ -0,0 +1,9 @@
+"""Default package."""
+
+__author__ = "Robert Kaussow"
+__project__ = "git-batch"
+__version__ = "0.1.0"
+__license__ = "MIT"
+__maintainer__ = "Robert Kaussow"
+__email__ = "mail@geeklabor.de"
+__url__ = "https://github.com/xoxys/git-batch"

gitbatch/__main__.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+
+import argparse
+import os
+import logging
+import sys
+import git
+
+from gitbatch import __version__
+from urllib.parse import urlparse
+
+logger = logging.getLogger("gitbatch")
+formatter = logging.Formatter("[%(levelname)s] %(message)s")
+
+cmdlog = logging.StreamHandler()
+cmdlog.setLevel(logging.ERROR)
+cmdlog.setFormatter(formatter)
+logger.addHandler(cmdlog)
+
+
+def sysexit(self, code=1):
+    sys.exit(code)
+
+
+def sysexit_with_message(msg, code=1):
+    logger.error(str(msg))
+    sysexit(code)
+
+
+def normalize_path(path):
+    if path:
+        return os.path.abspath(os.path.expanduser(os.path.expandvars(path)))
+
+
+def repos_from_file(src):
+    repos = []
+    with open(src, 'r') as f:
+        for num, line in enumerate(f, start=1):
+            repo = {}
+            line = line.strip()
+            if line and not line.startswith("#"):
+                try:
+                    url, branch, dest = [x.strip() for x in line.split(";")]
+                except ValueError as e:
+                    sysexit_with_message("Wrong numer of delimiters in line {line_num}: {exp}".format(
+                        line_num=num, exp=e))
+
+                if url:
+                    url_parts = urlparse(url)
+
+                    repo["url"] = url
+                    repo["branch"] = branch or "master"
+                    repo["name"] = os.path.basename(url_parts.path)
+                    repo["dest"] = normalize_path(dest) or normalize_path("./{}".format(repo["name"]))
+
+                    repos.append(repo)
+                else:
+                    sysexit_with_message("Repository Url is not set on line {line_num}".format(
+                        line_num=num))
+    return repos
+
+
+def repos_clone(repos, ignore_existing):
+    for repo in repos:
+        print(repo)
+        try:
+            git.Repo.clone_from(repo["url"], repo["dest"], multi_options=["--branch=docs", "--single-branch"])
+        except git.exc.GitCommandError as e:
+            if not ignore_existing:
+                err_raw = [x.strip() for x in e.stderr.split(":")][2]
+                err = err_raw.splitlines()[0].split(".")[0]
+                sysexit_with_message("Git error: {}".format(err))
+            else:
+                pass
+
+
+def main():
+    """Run main program."""
+    parser = argparse.ArgumentParser(
+        description=("Clone single branch from all repositories listed in a file"))
+    parser.add_argument("--version", action="version", version="%(prog)s {}".format(__version__))
+
+    parser.parse_args()
+
+    input_file_raw = os.environ.get("GIT_BATCH_INPUT_FILE", "./batchfile")
+    input_file = normalize_path(input_file_raw)
+
+    ignore_existing = os.environ.get("GIT_BATCH_IGNORE_EXISTING", True)
+
+    if os.path.isfile(input_file):
+        repos = repos_from_file(input_file)
+        repos_clone(repos, ignore_existing)
+    else:
+        sysexit_with_message("The given batch file at '{}' does not exist".format(
+            os.path.relpath(os.path.join("./", input_file))))
+
+
+if __name__ == "__main__":
+    main()

manifest.tmpl

@@ -0,0 +1,24 @@
+image: xoxys/git-batch:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
+{{#if build.tags}}
+tags:
+{{#each build.tags}}
+  - {{this}}
+{{/each}}
+{{/if}}
+manifests:
+  - image: xoxys/git-batch:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}amd64
+    platform:
+      architecture: amd64
+      os: linux
+
+  - image: xoxys/git-batch:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm64
+    platform:
+      architecture: arm64
+      os: linux
+      variant: v8
+
+  - image: xoxys/git-batch:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm
+    platform:
+      architecture: arm
+      os: linux
+      variant: v7

setup.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""Setup script for the package."""
+
+import io
+import os
+import re
+
+from setuptools import find_packages
+from setuptools import setup
+
+PACKAGE_NAME = "gitbatch"
+
+
+def get_property(prop, project):
+    current_dir = os.path.dirname(os.path.realpath(__file__))
+    result = re.search(
+        r'{}\s*=\s*[\'"]([^\'"]*)[\'"]'.format(prop),
+        open(os.path.join(current_dir, project, "__init__.py")).read())
+    return result.group(1)
+
+
+def get_readme(filename="README.md"):
+    this = os.path.abspath(os.path.dirname(__file__))
+    with io.open(os.path.join(this, filename), encoding="utf-8") as f:
+        long_description = f.read()
+    return long_description
+
+
+setup(
+    name=get_property("__project__", PACKAGE_NAME),
+    version=get_property("__version__", PACKAGE_NAME),
+    description="Clone single branch from all repositories listed in a file.",
+    author=get_property("__author__", PACKAGE_NAME),
+    author_email=get_property("__email__", PACKAGE_NAME),
+    url=get_property("__url__", PACKAGE_NAME),
+    license=get_property("__license__", PACKAGE_NAME),
+    long_description=get_readme(),
+    long_description_content_type="text/markdown",
+    packages=find_packages(exclude=["*.tests", "tests", "tests.*"]),
+    include_package_data=True,
+    zip_safe=False,
+    python_requires=">=3.5",
+    classifiers=[
+        "Development Status :: 5 - Production/Stable",
+        "Environment :: Console",
+        "Intended Audience :: Developers",
+        "Intended Audience :: Information Technology",
+        "Intended Audience :: System Administrators",
+        "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
+        "Natural Language :: English",
+        "Operating System :: POSIX",
+        "Programming Language :: Python :: 3 :: Only",
+        "Topic :: System :: Installation/Setup",
+        "Topic :: System :: Systems Administration",
+        "Topic :: Utilities",
+        "Topic :: Software Development",
+        "Topic :: Software Development :: Documentation",
+    ],
+    install_requires=[
+        "gitpython",
+    ],
+    entry_points={
+        "console_scripts": [
+            "git-batch = gitbatch.__main__:main"
+        ]
+    },
+    test_suite="tests"
+)

test-requirements.txt

@@ -0,0 +1,19 @@
+# open issue
+# https://gitlab.com/pycqa/flake8-docstrings/issues/36
+pydocstyle<4.0.0
+flake8
+flake8-colors
+flake8-blind-except
+flake8-builtins
+flake8-colors
+flake8-docstrings<=3.0.0
+flake8-isort
+flake8-logging-format
+flake8-polyfill
+flake8-quotes
+pep8-naming
+wheel
+pytest
+pytest-mock
+pytest-cov
+bandit