""" Frank-ci - A simple CI for Pythonistas, based on Flask + Huey. For more information see the docs/frank.rst """ import hmac import os import subprocess import subprocess as sp import logging import hashlib import yaml import click from flask import Flask, request, abort import conf from shell import Shell from collections import OrderedDict, namedtuple import importlib from conf import taskq import types import pickle # monkey patch data store _list = "select * FROM {0}" def list_results(obj): with obj._db.get_connection() as conn: try: return list(conn.execute(obj._list.format(obj.name))) except: return None taskq.result_store._list = _list taskq.result_store.list_results = types.MethodType(list_results, taskq.result_store) PythonCode = namedtuple('PythonCode', ['path', 'args', 'kwargs', 'code']) def override_run(self, command, **kwargs): """ Override Shell.run to handle exceptions and accept kwargs that Popen accepts """ self.last_command = command command_bits = self._split_command(command) _kwargs = { 'stdout': subprocess.PIPE, 'stderr': subprocess.PIPE, 'universal_newlines': True, } if kwargs: for kw in kwargs: _kwargs[kw] = kwargs[kw] _kwargs['shell'] = True if self.has_input: _kwargs['stdin'] = subprocess.PIPE try: self._popen = subprocess.Popen( command_bits, **_kwargs ) except Exception as E: self.exception = E return self self.pid = self._popen.pid if not self.has_input: self._communicate() return self Shell.run = override_run def ordered_load(stream, Loader=yaml.Loader, selfect_pairs_hook=OrderedDict): class OrderedLoader(Loader): pass def construct_mapping(loader, node): loader.flatten_mapping(node) return selfect_pairs_hook(loader.construct_pairs(node)) OrderedLoader.add_constructor( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) return yaml.load(stream, OrderedLoader) app = Flask(__name__) app.config.from_object(conf) def parse_branch_gh(request_json): """ parse the branch to clone from a github payload "ref": "refs/heads/develop", -> should return develop """ return request_json['ref'].split('/')[-1] def parse_yaml(clone_dest): os.chdir(clone_dest) if os.path.exists('.frank.yaml'): with open('.frank.yaml') as f: y = ordered_load(f, yaml.SafeLoader) return y def load_actions(yaml): pass def report_success(results): pass def report_failure(results): pass def run_action(axn): results = [] # run shell or python callable object without arguments if isinstance(axn, list): if axn[0] == 'shell': for cmd in axn[1:]: sh = Shell() assert isinstance(cmd, str) sh.run(cmd) results.append(sh) if sh.code: break if axn[0] == 'python': for func in axn[1:]: mod, f = func.split(':') mod = importlib.import_module(mod) f = getattr(mod, f) res = f() results.append(PythonCode(func, None, None, res)) # run shell or python callable object arguments elif isinstance(axn, OrderedDict): if 'shell' in axn: sh = Shell() cmd = axn['shell'].pop('cmd') assert isinstance(cmd, str) kwargs = axn['shell'] sh.run(cmd, **kwargs) results.append(sh) if 'python' in axn: callables = axn['python'] for func in callables: mod, f = func.split(':') mod = importlib.import_module(mod) try: f = getattr(mod, f) res = f() except AttributeError as E: res = E results.append(PythonCode(func, None, None, res)) return results def clone(clone_url, branch, depth=1): cmd = ('git clone --depth={d} -b {branch} --single-branch ' '{git_url} {dir}'.format(d=depth, branch=branch, git_url=clone_url, dir=branch)) pull = sp.Popen(cmd, stderr=sp.STDOUT, shell=True) out, err = pull.communicate() return out, err @taskq.task() def count_beans(): print "count_12" return "count_12" @taskq.task() def build_task(request_json): """ . clone the latest commit (--depth 1) . parse yaml config . for each command in the config . run command . report success or failure """ clone_dest = parse_branch_gh(request_json) repo_name = request_as_json["repository"]['name'] try: o, e = clone(request_as_json['repository']['ssh_url'], clone_dest) except Exception as E: print E, E.message # parse yaml is still very crude ... # it could yield selfect with a run method # thus: # for action in parse_yaml(clone_dest): # action.run() # # this should also handle dependencies, # the following implementation is very crud failed = None for action in parse_yaml(clone_dest): # if config says we use huey, we should modiy run_action results = run_action(action) if any([result.code for result in results]): report_failure(results) else: report_success(results) @app.route('/beans') def do_beans(): a = count_beans() ans = a.get(blocking=True) return ans @app.route('/results') def show_resutls(): """ TODO: Make a nicer web page with jinja2 """ res = {r[1]:r[2] for r in taskq.result_store.list_results()} return ''.join(['

'+str(k) + ': ' + str(pickle.loads(v)) + '

\n' for (k, v) in res.iteritems()]) @app.route('/', methods=['POST']) def start(): """ main logic: 1. listen to post 2. if authenticated post do: enqueue task to build or test the code # This is authentication for github only # We could\should check for other hostings """ ans = hmac.new(app.config['POST_KEY'], request.data, hashlib.sha1).hexdigest() secret = request.headers[app.config['SECRET_KEY_NAME']].split('=')[-1] if ans != secret: return abort(500) request_as_json = request.get_json() build_task(request_as_json) return "OK" @app.route('/status') def status(): return "Frank is alive\n" @click.group() def cli(): pass @cli.command('web', short_help='start the web service') @click.option('--port','-p', default=8080) @click.option('--debug', default=False, is_flag=True) def web(port, debug): click.echo("DEBUG: %s" % debug) app.run(host='0.0.0.0',port=port, debug=debug) @cli.command(context_settings=dict( ignore_unknown_options=True, allow_extra_args=True)) @click.argument('worker_args', nargs=-1, type=click.UNPROCESSED) def worker(worker_args, short_help='start the consumer of tasks'): from huey.bin.huey_consumer import (get_option_parser, Consumer, setup_logger, RotatingFileHandler) from conf import taskq parser = get_option_parser() opts, args = parser.parse_args(list(worker_args)) setup_logger(logging.INFO, opts.logfile) consumer = Consumer(taskq, 2, opts.periodic, opts.initial_delay, opts.backoff, opts.max_delay, opts.utc, opts.scheduler_interval, opts.periodic_task_interval) consumer.run()