initial commit

This commit is contained in:
Robert Kaussow 2019-11-29 14:45:28 +01:00
commit c22000f4bc
11 changed files with 953 additions and 0 deletions

236
.drone.jsonnet Normal file
View File

@ -0,0 +1,236 @@
local PythonVersion(pyversion="3.5") = {
name: "python" + std.strReplace(pyversion, '.', '') + "-pytest",
image: "python:" + pyversion,
pull: "always",
environment: {
PY_COLORS: 1
},
commands: [
"pip install -r test-requirements.txt -qq",
"pip install -qq .",
"git-batch --help",
],
depends_on: [
"clone",
],
};
local PipelineLint = {
kind: "pipeline",
name: "lint",
platform: {
os: "linux",
arch: "amd64",
},
steps: [
{
name: "flake8",
image: "python:3.7",
pull: "always",
environment: {
PY_COLORS: 1
},
commands: [
"pip install -r test-requirements.txt -qq",
"pip install -qq .",
"flake8 ./gitbatch",
],
},
],
trigger: {
ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
},
};
local PipelineTest = {
kind: "pipeline",
name: "test",
platform: {
os: "linux",
arch: "amd64",
},
steps: [
PythonVersion(pyversion="3.5"),
PythonVersion(pyversion="3.6"),
PythonVersion(pyversion="3.7"),
PythonVersion(pyversion="3.8"),
],
trigger: {
ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
},
depends_on: [
"lint",
],
};
local PipelineSecurity = {
kind: "pipeline",
name: "security",
platform: {
os: "linux",
arch: "amd64",
},
steps: [
{
name: "bandit",
image: "python:3.7",
pull: "always",
environment: {
PY_COLORS: 1
},
commands: [
"pip install -r test-requirements.txt -qq",
"pip install -qq .",
"bandit -r ./gitbatch -x ./gitbatch/tests",
],
},
],
depends_on: [
"test",
],
trigger: {
ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
},
};
local PipelineBuildContainer(arch="amd64") = {
kind: "pipeline",
name: "build-container-" + arch,
platform: {
os: "linux",
arch: arch,
},
steps: [
{
name: "build",
image: "python:3.7",
pull: "always",
commands: [
"python setup.py bdist_wheel",
]
},
{
name: "dryrun",
image: "plugins/docker:18-linux-" + arch,
pull: "always",
settings: {
dry_run: true,
dockerfile: "Dockerfile",
repo: "xoxys/git-batch",
username: { "from_secret": "docker_username" },
password: { "from_secret": "docker_password" },
},
when: {
ref: ["refs/pull/**"],
},
},
{
name: "publish",
image: "plugins/docker:18-linux-" + arch,
pull: "always",
settings: {
auto_tag: true,
auto_tag_suffix: arch,
dockerfile: "Dockerfile",
repo: "xoxys/git-batch",
username: { "from_secret": "docker_username" },
password: { "from_secret": "docker_password" },
},
when: {
ref: ["refs/heads/master", "refs/tags/**"],
},
},
],
depends_on: [
"security",
],
trigger: {
ref: ["refs/heads/master", "refs/tags/**", "refs/pull/**"],
},
};
local PipelineNotifications = {
kind: "pipeline",
name: "notifications",
platform: {
os: "linux",
arch: "amd64",
},
steps: [
{
image: "plugins/manifest",
name: "manifest",
pull: "always",
settings: {
ignore_missing: true,
auto_tag: true,
username: { from_secret: "docker_username" },
password: { from_secret: "docker_password" },
spec: "manifest.tmpl",
},
when: {
ref: [
'refs/heads/master',
'refs/tags/**',
],
},
},
{
name: "readme",
image: "sheogorath/readme-to-dockerhub",
pull: "always",
environment: {
DOCKERHUB_USERNAME: { from_secret: "docker_username" },
DOCKERHUB_PASSWORD: { from_secret: "docker_password" },
DOCKERHUB_REPO_PREFIX: "xoxys",
DOCKERHUB_REPO_NAME: "git-batch",
README_PATH: "README.md",
SHORT_DESCRIPTION: "git-batch"
},
when: {
ref: [
'refs/heads/master',
'refs/tags/**',
],
},
},
{
name: "microbadger",
image: "plugins/webhook",
pull: "always",
settings: {
urls: { from_secret: "microbadger_url" },
},
},
{
name: "matrix",
image: "plugins/matrix",
settings: {
template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}",
roomid: { "from_secret": "matrix_roomid" },
homeserver: { "from_secret": "matrix_homeserver" },
username: { "from_secret": "matrix_username" },
password: { "from_secret": "matrix_password" },
},
},
],
depends_on: [
"build-container-amd64",
"build-container-arm64",
"build-container-arm"
],
trigger: {
ref: ["refs/heads/master", "refs/tags/**"],
status: [ "success", "failure" ],
},
};
[
PipelineLint,
PipelineTest,
PipelineSecurity,
PipelineBuildContainer(arch="amd64"),
PipelineBuildContainer(arch="arm64"),
PipelineBuildContainer(arch="arm"),
PipelineNotifications,
]

363
.drone.yml Normal file
View File

@ -0,0 +1,363 @@
---
kind: pipeline
name: lint
platform:
os: linux
arch: amd64
steps:
- name: flake8
pull: always
image: python:3.7
commands:
- pip install -r test-requirements.txt -qq
- pip install -qq .
- flake8 ./gitbatch
environment:
PY_COLORS: 1
trigger:
ref:
- refs/heads/master
- refs/tags/**
- refs/pull/**
---
kind: pipeline
name: test
platform:
os: linux
arch: amd64
steps:
- name: python35-pytest
pull: always
image: python:3.5
commands:
- pip install -r test-requirements.txt -qq
- pip install -qq .
- git-batch --help
environment:
PY_COLORS: 1
depends_on:
- clone
- name: python36-pytest
pull: always
image: python:3.6
commands:
- pip install -r test-requirements.txt -qq
- pip install -qq .
- git-batch --help
environment:
PY_COLORS: 1
depends_on:
- clone
- name: python37-pytest
pull: always
image: python:3.7
commands:
- pip install -r test-requirements.txt -qq
- pip install -qq .
- git-batch --help
environment:
PY_COLORS: 1
depends_on:
- clone
- name: python38-pytest
pull: always
image: python:3.8
commands:
- pip install -r test-requirements.txt -qq
- pip install -qq .
- git-batch --help
environment:
PY_COLORS: 1
depends_on:
- clone
trigger:
ref:
- refs/heads/master
- refs/tags/**
- refs/pull/**
depends_on:
- lint
---
kind: pipeline
name: security
platform:
os: linux
arch: amd64
steps:
- name: bandit
pull: always
image: python:3.7
commands:
- pip install -r test-requirements.txt -qq
- pip install -qq .
- bandit -r ./gitbatch -x ./gitbatch/tests
environment:
PY_COLORS: 1
trigger:
ref:
- refs/heads/master
- refs/tags/**
- refs/pull/**
depends_on:
- test
---
kind: pipeline
name: build-container-amd64
platform:
os: linux
arch: amd64
steps:
- name: build
pull: always
image: python:3.7
commands:
- python setup.py bdist_wheel
- name: dryrun
pull: always
image: plugins/docker:18-linux-amd64
settings:
dockerfile: Dockerfile
dry_run: true
password:
from_secret: docker_password
repo: xoxys/git-batch
username:
from_secret: docker_username
when:
ref:
- refs/pull/**
- name: publish
pull: always
image: plugins/docker:18-linux-amd64
settings:
auto_tag: true
auto_tag_suffix: amd64
dockerfile: Dockerfile
password:
from_secret: docker_password
repo: xoxys/git-batch
username:
from_secret: docker_username
when:
ref:
- refs/heads/master
- refs/tags/**
trigger:
ref:
- refs/heads/master
- refs/tags/**
- refs/pull/**
depends_on:
- security
---
kind: pipeline
name: build-container-arm64
platform:
os: linux
arch: arm64
steps:
- name: build
pull: always
image: python:3.7
commands:
- python setup.py bdist_wheel
- name: dryrun
pull: always
image: plugins/docker:18-linux-arm64
settings:
dockerfile: Dockerfile
dry_run: true
password:
from_secret: docker_password
repo: xoxys/git-batch
username:
from_secret: docker_username
when:
ref:
- refs/pull/**
- name: publish
pull: always
image: plugins/docker:18-linux-arm64
settings:
auto_tag: true
auto_tag_suffix: arm64
dockerfile: Dockerfile
password:
from_secret: docker_password
repo: xoxys/git-batch
username:
from_secret: docker_username
when:
ref:
- refs/heads/master
- refs/tags/**
trigger:
ref:
- refs/heads/master
- refs/tags/**
- refs/pull/**
depends_on:
- security
---
kind: pipeline
name: build-container-arm
platform:
os: linux
arch: arm
steps:
- name: build
pull: always
image: python:3.7
commands:
- python setup.py bdist_wheel
- name: dryrun
pull: always
image: plugins/docker:18-linux-arm
settings:
dockerfile: Dockerfile
dry_run: true
password:
from_secret: docker_password
repo: xoxys/git-batch
username:
from_secret: docker_username
when:
ref:
- refs/pull/**
- name: publish
pull: always
image: plugins/docker:18-linux-arm
settings:
auto_tag: true
auto_tag_suffix: arm
dockerfile: Dockerfile
password:
from_secret: docker_password
repo: xoxys/git-batch
username:
from_secret: docker_username
when:
ref:
- refs/heads/master
- refs/tags/**
trigger:
ref:
- refs/heads/master
- refs/tags/**
- refs/pull/**
depends_on:
- security
---
kind: pipeline
name: notifications
platform:
os: linux
arch: amd64
steps:
- name: manifest
pull: always
image: plugins/manifest
settings:
auto_tag: true
ignore_missing: true
password:
from_secret: docker_password
spec: manifest.tmpl
username:
from_secret: docker_username
when:
ref:
- refs/heads/master
- refs/tags/**
- name: readme
pull: always
image: sheogorath/readme-to-dockerhub
environment:
DOCKERHUB_PASSWORD:
from_secret: docker_password
DOCKERHUB_REPO_NAME: git-batch
DOCKERHUB_REPO_PREFIX: xoxys
DOCKERHUB_USERNAME:
from_secret: docker_username
README_PATH: README.md
SHORT_DESCRIPTION: git-batch
when:
ref:
- refs/heads/master
- refs/tags/**
- name: microbadger
pull: always
image: plugins/webhook
settings:
urls:
from_secret: microbadger_url
- name: matrix
image: plugins/matrix
settings:
homeserver:
from_secret: matrix_homeserver
password:
from_secret: matrix_password
roomid:
from_secret: matrix_roomid
template: "Status: **{{ build.status }}**<br/> Build: [{{ repo.Owner }}/{{ repo.Name }}]({{ build.link }}) ({{ build.branch }}) by {{ build.author }}<br/> Message: {{ build.message }}"
username:
from_secret: matrix_username
trigger:
ref:
- refs/heads/master
- refs/tags/**
status:
- success
- failure
depends_on:
- build-container-amd64
- build-container-arm64
- build-container-arm
...

103
.gitignore vendored Normal file
View File

@ -0,0 +1,103 @@
# ---> Python
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
env/
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*,cover
.hypothesis/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# dotenv
.env
# virtualenv
.venv
venv/
ENV/
env/
env*/
# Spyder project settings
.spyderproject
# Rope project settings
.ropeproject
# Ignore ide addons
.server-script
.on-save.json
.vscode
.pytest_cache
pip-wheel-metadata

24
Dockerfile Normal file
View File

@ -0,0 +1,24 @@
FROM python:3.7-alpine
LABEL maintainer="Robert Kaussow <mail@geeklabor.de>" \
org.label-schema.name="git-batch" \
org.label-schema.vcs-url="https://github.com/xoxys/git-batch" \
org.label-schema.vendor="Robert Kaussow" \
org.label-schema.schema-version="1.0"
ENV PY_COLORS=1
ADD dist/git_batch-*.whl /
RUN apk --update add --virtual .build-deps build-base libffi-dev libressl-dev && \
apk --update add git && \
pip install --upgrade --no-cache-dir pip && \
pip install --no-cache-dir --find-links=. git-batch && \
apk del .build-deps && \
rm -rf /var/cache/apk/* && \
rm -rf /root/.cache/ && \
rm -f git_batch-*.whl
USER root
CMD []
ENTRYPOINT ["/usr/local/bin/git-batch"]

1
README.md Normal file
View File

@ -0,0 +1 @@
# git-batch

7
bin/git-batch Executable file
View File

@ -0,0 +1,7 @@
#!/usr/bin/env python3
import sys
import gitbatch.__main__
sys.exit(gitbatch.__main__.main())

9
gitbatch/__init__.py Normal file
View File

@ -0,0 +1,9 @@
"""Default package."""
__author__ = "Robert Kaussow"
__project__ = "git-batch"
__version__ = "0.1.0"
__license__ = "MIT"
__maintainer__ = "Robert Kaussow"
__email__ = "mail@geeklabor.de"
__url__ = "https://github.com/xoxys/git-batch"

99
gitbatch/__main__.py Normal file
View File

@ -0,0 +1,99 @@
#!/usr/bin/env python3
import argparse
import os
import logging
import sys
import git
from gitbatch import __version__
from urllib.parse import urlparse
logger = logging.getLogger("gitbatch")
formatter = logging.Formatter("[%(levelname)s] %(message)s")
cmdlog = logging.StreamHandler()
cmdlog.setLevel(logging.ERROR)
cmdlog.setFormatter(formatter)
logger.addHandler(cmdlog)
def sysexit(self, code=1):
sys.exit(code)
def sysexit_with_message(msg, code=1):
logger.error(str(msg))
sysexit(code)
def normalize_path(path):
if path:
return os.path.abspath(os.path.expanduser(os.path.expandvars(path)))
def repos_from_file(src):
repos = []
with open(src, 'r') as f:
for num, line in enumerate(f, start=1):
repo = {}
line = line.strip()
if line and not line.startswith("#"):
try:
url, branch, dest = [x.strip() for x in line.split(";")]
except ValueError as e:
sysexit_with_message("Wrong numer of delimiters in line {line_num}: {exp}".format(
line_num=num, exp=e))
if url:
url_parts = urlparse(url)
repo["url"] = url
repo["branch"] = branch or "master"
repo["name"] = os.path.basename(url_parts.path)
repo["dest"] = normalize_path(dest) or normalize_path("./{}".format(repo["name"]))
repos.append(repo)
else:
sysexit_with_message("Repository Url is not set on line {line_num}".format(
line_num=num))
return repos
def repos_clone(repos, ignore_existing):
for repo in repos:
print(repo)
try:
git.Repo.clone_from(repo["url"], repo["dest"], multi_options=["--branch=docs", "--single-branch"])
except git.exc.GitCommandError as e:
if not ignore_existing:
err_raw = [x.strip() for x in e.stderr.split(":")][2]
err = err_raw.splitlines()[0].split(".")[0]
sysexit_with_message("Git error: {}".format(err))
else:
pass
def main():
"""Run main program."""
parser = argparse.ArgumentParser(
description=("Clone single branch from all repositories listed in a file"))
parser.add_argument("--version", action="version", version="%(prog)s {}".format(__version__))
parser.parse_args()
input_file_raw = os.environ.get("GIT_BATCH_INPUT_FILE", "./batchfile")
input_file = normalize_path(input_file_raw)
ignore_existing = os.environ.get("GIT_BATCH_IGNORE_EXISTING", True)
if os.path.isfile(input_file):
repos = repos_from_file(input_file)
repos_clone(repos, ignore_existing)
else:
sysexit_with_message("The given batch file at '{}' does not exist".format(
os.path.relpath(os.path.join("./", input_file))))
if __name__ == "__main__":
main()

24
manifest.tmpl Normal file
View File

@ -0,0 +1,24 @@
image: xoxys/git-batch:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
{{#if build.tags}}
tags:
{{#each build.tags}}
- {{this}}
{{/each}}
{{/if}}
manifests:
- image: xoxys/git-batch:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}amd64
platform:
architecture: amd64
os: linux
- image: xoxys/git-batch:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm64
platform:
architecture: arm64
os: linux
variant: v8
- image: xoxys/git-batch:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}arm
platform:
architecture: arm
os: linux
variant: v7

68
setup.py Normal file
View File

@ -0,0 +1,68 @@
#!/usr/bin/env python3
"""Setup script for the package."""
import io
import os
import re
from setuptools import find_packages
from setuptools import setup
PACKAGE_NAME = "gitbatch"
def get_property(prop, project):
current_dir = os.path.dirname(os.path.realpath(__file__))
result = re.search(
r'{}\s*=\s*[\'"]([^\'"]*)[\'"]'.format(prop),
open(os.path.join(current_dir, project, "__init__.py")).read())
return result.group(1)
def get_readme(filename="README.md"):
this = os.path.abspath(os.path.dirname(__file__))
with io.open(os.path.join(this, filename), encoding="utf-8") as f:
long_description = f.read()
return long_description
setup(
name=get_property("__project__", PACKAGE_NAME),
version=get_property("__version__", PACKAGE_NAME),
description="Clone single branch from all repositories listed in a file.",
author=get_property("__author__", PACKAGE_NAME),
author_email=get_property("__email__", PACKAGE_NAME),
url=get_property("__url__", PACKAGE_NAME),
license=get_property("__license__", PACKAGE_NAME),
long_description=get_readme(),
long_description_content_type="text/markdown",
packages=find_packages(exclude=["*.tests", "tests", "tests.*"]),
include_package_data=True,
zip_safe=False,
python_requires=">=3.5",
classifiers=[
"Development Status :: 5 - Production/Stable",
"Environment :: Console",
"Intended Audience :: Developers",
"Intended Audience :: Information Technology",
"Intended Audience :: System Administrators",
"License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)",
"Natural Language :: English",
"Operating System :: POSIX",
"Programming Language :: Python :: 3 :: Only",
"Topic :: System :: Installation/Setup",
"Topic :: System :: Systems Administration",
"Topic :: Utilities",
"Topic :: Software Development",
"Topic :: Software Development :: Documentation",
],
install_requires=[
"gitpython",
],
entry_points={
"console_scripts": [
"git-batch = gitbatch.__main__:main"
]
},
test_suite="tests"
)

19
test-requirements.txt Normal file
View File

@ -0,0 +1,19 @@
# open issue
# https://gitlab.com/pycqa/flake8-docstrings/issues/36
pydocstyle<4.0.0
flake8
flake8-colors
flake8-blind-except
flake8-builtins
flake8-colors
flake8-docstrings<=3.0.0
flake8-isort
flake8-logging-format
flake8-polyfill
flake8-quotes
pep8-naming
wheel
pytest
pytest-mock
pytest-cov
bandit