""" Frank can be configured using the following yaml:: .frank.yml commads: - test - deploy - publish newtag: - python: - frank.actions:detect_new_tag deploy: - cmd - runif: - test - newtag publish: - shell: - cd docs; make html - python: - frank.actions:recursive_copy The sections commands is a simple list of command you would like to define. This section is optional. It's main purpose is for you to decalre which commands frank should execute upon recieving a load. Each command is a dictionary with the following possible keys:: cmd_name: - cmd # not mandatory if you include it in the list of commands - runif # a conditional to determine whether to run or skip the command # it can contain multiple directives which can be python # code to execute # or shell code For example, let's say you would like to build the sphinx documentation of your project after every push. You start by defining a command build_docs in the following way:: build_docs: - cmd - shell: - cd docs - make html You could also specify:: build_docs: - cmd - shell: cd docs; make html or:: build_docs: - shell: - cwd: docs - cmd: make html This tells that `build_docs` is a command that executes after every push. It will execute a shell and will run the shell commands `cd docs; make html`. Pretty straight forward! Now, let's refine. Suppose we want to build the docs only if the docs changed, thus ignoring completely, changes in the embeded docs strings*:: build_docs: - cmd - runif: - shell: - git --no-pager diff --name-status HEAD~1 | grep -v docs - shell: cd docs; make html Now, the command will run if the latest git commit have changed the docs directory. Since the grep command will return exit with 1. Alternatively, the conditional to run can be some python code that returns True or False. Here is how to specify this: build_docs: - cmd - runif: - python: frank.githubparsers:detect_changes_in_docs - shell: cd docs; make html This, will invoke the method ``detect_changes_in_docs`` which is found in the module `frank.githubparsers`. If this function will return True the shell command will execute. If the methods takes some arguments, they could also be given. Suppose the function is called `detect_changes_in` and this function takes a single paramter called path we can configure the build_docs command in the following way: build_docs: - cmd - runif: - python: - function: frank.githubparsers:detect_changes_in - args: - path: docs * This is probably a lame idea, but it's good for demonstration. So, do write doc-strings, and do build your docs often. """ import hmac import os import subprocess import subprocess as sp import hashlib import yaml import click from flask import Flask, request, abort import conf from shell import Shell from collections import OrderedDict, namedtuple import importlib PythonCode = namedtuple('PythonCode', ['path', 'args', 'kwargs', 'code']) def override_run(self, command, **kwargs): """ Override Shell.run to handle exceptions and accept kwargs that Popen accepts """ self.last_command = command command_bits = self._split_command(command) _kwargs = { 'stdout': subprocess.PIPE, 'stderr': subprocess.PIPE, 'universal_newlines': True, } if kwargs: for kw in kwargs: _kwargs[kw] = kwargs[kw] _kwargs['shell'] = True if self.has_input: _kwargs['stdin'] = subprocess.PIPE try: self._popen = subprocess.Popen( command_bits, **_kwargs ) except Exception as E: self.exception = E return self self.pid = self._popen.pid if not self.has_input: self._communicate() return self Shell.run = override_run def ordered_load(stream, Loader=yaml.Loader, selfect_pairs_hook=OrderedDict): class OrderedLoader(Loader): pass def construct_mapping(loader, node): loader.flatten_mapping(node) return selfect_pairs_hook(loader.construct_pairs(node)) OrderedLoader.add_constructor( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) return yaml.load(stream, OrderedLoader) app = Flask(__name__) app.config.from_object(conf) def parse_branch_gh(request_json): """ parse the branch to clone from a github payload "ref": "refs/heads/develop", -> should return develop """ return request_json['ref'].split('/')[-1] def parse_yaml(clone_dest): os.chdir(clone_dest) if os.path.exists('.frank.yaml'): with open('.frank.yaml') as f: y = ordered_load(f, yaml.SafeLoader) return y def load_actions(yaml): pass def report_success(results): pass def report_failure(results): pass def run_action(axn): results = [] # run shell or python callable object without arguments if isinstance(axn, list): if axn[0] == 'shell': for cmd in axn[1:]: sh = Shell() assert isinstance(cmd, str) sh.run(cmd) results.append(sh) if sh.code: break if axn[0] == 'python': for func in axn[1:]: mod, f = func.split(':') mod = importlib.import_module(mod) f = getattr(mod, f) res = f() results.append(PythonCode(func, None, None, res)) # run shell or python callable object arguments elif isinstance(axn, OrderedDict): if 'shell' in axn: sh = Shell() cmd = axn['shell'].pop('cmd') assert isinstance(cmd, str) kwargs = axn['shell'] sh.run(cmd, **kwargs) results.append(sh) if 'python' in axn: callables = axn['python'] for func in callables: mod, f = func.split(':') mod = importlib.import_module(mod) try: f = getattr(mod, f) res = f() except AttributeError as E: res = E results.append(PythonCode(func, None, None, res)) return results def clone(clone_url, branch, depth=1): cmd = ('git clone --depth={d} -b {branch} --single-branch ' '{git_url} {dir}'.format(d=depth, branch=branch, git_url=clone_url, dir=branch)) pull = sp.Popen(cmd, stderr=sp.STDOUT, shell=True) out, err = pull.communicate() return out, err @app.route('/', methods=['POST']) def start(): """ main logic: 1. listen to post 2a if authenticated post do: 3. clone the latest commit (--depth 1) 4. parse yaml config 5. for each command in the config 7. run command 8. report success or failure """ # This is authentication for github only # We could\should check for other hostings ans = hmac.new(app.config['POST_KEY'], request.data, hashlib.sha1).hexdigest() secret = request.headers['X-Hub-Signature'].split('=')[-1] if ans != secret: return abort(500) request_as_json = request.get_json() clone_dest = parse_branch_gh(request_as_json) repo_name = request_as_json["repository"]['name'] try: o, e = clone(request_as_json['repository']['ssh_url'], clone_dest) except Exception as E: print E, E.message # parse yaml is still very crude ... # it could yield selfect with a run method # thus: # for action in parse_yaml(clone_dest): # action.run() # # this should also handle dependencies, # the following implementation is very crud failed = None for action in parse_yaml(clone_dest): # if config says we use huey, we should modiy run_action results = run_action(action) if any([result.code for result in results]): report_failure(results) else: report_success(results) @click.group() @click.option('--debug/--no-debug', default=False) def cli(debug): pass @cli.command() def web(): app.run(host='0.0.0.0', debug=True) @cli.command() def worker(): click.echo('will start the consumer...') @cli.command() def queue(): click.echo('will start the queue')