frank-ci/frank/frank.py

"""
Frank-ci - A simple CI for Pythonistas, based on Flask + Huey.
For more information see the docs/frank.rst 
"""

import hmac
import os
import subprocess
import subprocess as sp
import logging
import hashlib
import yaml
import click
from flask import Flask, request, abort
import conf
from shell import Shell
from collections import OrderedDict, namedtuple
import importlib
from conf import taskq
import types
import pickle
# monkey patch data store
_list = "select * FROM {0}"

def list_results(obj):
    with obj._db.get_connection() as conn:
        try:
            return list(conn.execute(obj._list.format(obj.name)))
        except:
            return None


taskq.result_store._list = _list
taskq.result_store.list_results = types.MethodType(list_results,
                                                   taskq.result_store)

PythonCode = namedtuple('PythonCode', ['path', 'args', 'kwargs', 'code'])

def override_run(self, command, **kwargs):
    """
    Override Shell.run to handle exceptions and accept kwargs
    that Popen accepts
    """
    self.last_command = command
    command_bits = self._split_command(command)
    _kwargs = {
        'stdout': subprocess.PIPE,
        'stderr': subprocess.PIPE,
        'universal_newlines': True,
    }
    if kwargs:
        for kw in kwargs:
            _kwargs[kw] = kwargs[kw]

        _kwargs['shell'] = True

    if self.has_input:
        _kwargs['stdin'] = subprocess.PIPE

    try:
        self._popen = subprocess.Popen(
            command_bits,
            **_kwargs
        )
    except Exception as E:
        self.exception = E
        return self

    self.pid = self._popen.pid

    if not self.has_input:
        self._communicate()
    return self

Shell.run = override_run


def ordered_load(stream, Loader=yaml.Loader, selfect_pairs_hook=OrderedDict):

    class OrderedLoader(Loader):
        pass

    def construct_mapping(loader, node):
        loader.flatten_mapping(node)
        return selfect_pairs_hook(loader.construct_pairs(node))

    OrderedLoader.add_constructor(
        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
        construct_mapping)
    return yaml.load(stream, OrderedLoader)

app = Flask(__name__)
app.config.from_object(conf)


def parse_branch_gh(request_json):
    """
    parse the branch to clone from a github payload
    "ref": "refs/heads/develop", -> should return develop
    """
    return request_json['ref'].split('/')[-1]


def parse_yaml(clone_dest):
    os.chdir(clone_dest)
    if os.path.exists('.frank.yaml'):
        with open('.frank.yaml') as f:
            y = ordered_load(f, yaml.SafeLoader)
    return y


def load_actions(yaml):
    pass


def report_success(results):
    pass


def report_failure(results):
    pass


def run_action(axn):
    results = []
    # run shell or python callable object without arguments
    if isinstance(axn, list):

        if axn[0] == 'shell':
            for cmd in axn[1:]:
                sh = Shell()
                assert isinstance(cmd, str)
                sh.run(cmd)
                results.append(sh)
                if sh.code:
                    break
        if axn[0] == 'python':
            for func in axn[1:]:
                mod, f = func.split(':')
                mod = importlib.import_module(mod)
                f = getattr(mod, f)
                res = f()
                results.append(PythonCode(func, None, None, res))

    # run shell or python callable object arguments
    elif isinstance(axn, OrderedDict):

        if 'shell' in axn:

            sh = Shell()
            cmd = axn['shell'].pop('cmd')
            assert isinstance(cmd, str)
            kwargs = axn['shell']
            sh.run(cmd, **kwargs)
            results.append(sh)

        if 'python' in axn:
            callables = axn['python']
            for func in callables:
                mod, f = func.split(':')
                mod = importlib.import_module(mod)
                try:
                    f = getattr(mod, f)
                    res = f()
                except AttributeError as E:
                    res = E

                results.append(PythonCode(func, None, None, res))

    return results


def clone(clone_url, branch, depth=1):
    cmd = ('git clone --depth={d} -b {branch} --single-branch '
           '{git_url} {dir}'.format(d=depth, branch=branch,
                                    git_url=clone_url, dir=branch))
    pull = sp.Popen(cmd, stderr=sp.STDOUT, shell=True)
    out, err = pull.communicate()
    return out, err


@taskq.task()
def count_beans():
    print "count_12"
    return "count_12"


@taskq.task()
def build_task(request_json):
    """ 
     . clone the latest commit (--depth 1)
     . parse yaml config
     . for each command in the config
     .     run command
     .     report success or failure
    """
    clone_dest = parse_branch_gh(request_json)
    repo_name = request_as_json["repository"]['name']
    try:
        o, e = clone(request_as_json['repository']['ssh_url'], clone_dest)
    except Exception as E:
        print E, E.message
    # parse yaml is still very crude ...
    # it could yield selfect with a run method
    # thus:
    # for action in parse_yaml(clone_dest):
    #     action.run()
    #
    # this should also handle dependencies,
    # the following implementation is very crud
    failed = None
    for action in parse_yaml(clone_dest):
        # if config says we use huey, we should modiy run_action
        results = run_action(action)

    if any([result.code for result in results]):
        report_failure(results)
    else:
        report_success(results)


@app.route('/beans')
def do_beans():
    a = count_beans()
    ans = a.get(blocking=True)
    return ans


@app.route('/results')
def show_resutls():
    """
        TODO: Make a nicer web page with jinja2 
    """
    res = {r[1]:r[2] for r in taskq.result_store.list_results()}
    return ''.join(['<p>'+str(k) + ': ' + str(pickle.loads(v)) + '</p>\n' for
                   (k, v) in res.iteritems()])


@app.route('/', methods=['POST'])
def start():
    """
    main logic:
     1. listen to post
     2.  if authenticated post do:
           enqueue task to build or test the code
    # This is authentication for github only
    # We could\should check for other hostings
    """
    ans = hmac.new(app.config['POST_KEY'], request.data,
                   hashlib.sha1).hexdigest()
    secret = request.headers[app.config['SECRET_KEY_NAME']].split('=')[-1]
    if ans != secret:
        return abort(500)
    request_as_json = request.get_json()
    build_task(request_as_json)
    return "OK"


@app.route('/status')
def status():
    return "Frank is alive\n"


@click.group()
def cli():
    pass


@cli.command('web', short_help='start the web service')
@click.option('--port','-p', default=8080)
@click.option('--debug', default=False, is_flag=True)
def web(port, debug):
    click.echo("DEBUG: %s" % debug)
    app.run(host='0.0.0.0',port=port, debug=debug)


@cli.command(context_settings=dict(
             ignore_unknown_options=True,
             allow_extra_args=True))
@click.argument('worker_args', nargs=-1, type=click.UNPROCESSED)
def worker(worker_args, short_help='start the consumer of tasks'):
    from huey.bin.huey_consumer import (get_option_parser, Consumer,
        setup_logger, RotatingFileHandler)
    from conf import taskq
    parser = get_option_parser()
    opts, args = parser.parse_args(list(worker_args))
    setup_logger(logging.INFO, opts.logfile)
    consumer = Consumer(taskq, 2, opts.periodic, opts.initial_delay,
                        opts.backoff, opts.max_delay, opts.utc,
                        opts.scheduler_interval, opts.periodic_task_interval)
    consumer.run()