frank-ci/frank/frank.py

"""
Frank can be configured using the following yaml::
    .frank.yml

    commads:
     - test
     - deploy
     - publish

    newtag:
         - python:
           - frank.actions:detect_new_tag

        deploy:
         - cmd
         - runif
           - test
           - newtag

        publish:
          - shell:
           - cd docs; make html
          - python:
           - frank.actions:recursive_copy

The sections commands is a simple list of command you would
like to define.
This section is optional. It's main purpose is for you to decalre
which commands frank should execute upon recieving a load.

Each command is a dictionary with the following possible keys::

    cmd_name:
      - cmd  # not mandatory if you include it in the list of commands
      - runif # a conditional to determine whether to run or skip the command
              # it can contain multiple directives which can be python
              # code to execute
              # or shell code


For example, let's say you would like to build the sphinx documentation
of your project after every push. You start by defining a command
build_docs in the following way::

   build_docs:
    - cmd
    - shell:
      - cd docs
      - make html

You could also specify::

   build_docs:
    - cmd
    - shell: cd docs; make html

or::

   build_docs:
    - shell:
      - cwd: docs
      - cmd: make html


This tells that `build_docs` is a command that executes after every push.
It will execute a shell and will run the shell commands
`cd docs; make html`. Pretty straight forward!

Now, let's refine. Suppose we want to build the docs only if the docs changed,
thus ignoring completely, changes in the embeded docs strings*::

   build_docs:
    - cmd
    - runif:
      - shell:
        - git --no-pager diff --name-status HEAD~1 | grep -v docs
    - shell: cd docs; make html

Now, the command will run if the latest git commit have changed the docs
directory. Since the grep command will return exit with 1.
Alternatively, the conditional to run can be some python code that returns
True or False. Here is how to specify this:

   build_docs:
    - cmd
    - runif:
      - python: frank.githubparsers:detect_changes_in_docs
    - shell: cd docs; make html

This, will invoke the method ``detect_changes_in_docs`` which is
found in the module `frank.githubparsers`.
If this function will return True the shell command will execute.
If the methods takes some arguments, they could also be given.
Suppose the function is called `detect_changes_in` and this function
takes a single paramter called path we can configure the
build_docs command in the following way:

   build_docs:
    - cmd
    - runif:
      - python:
        - function: frank.githubparsers:detect_changes_in
        - args:
          - path: docs

* This is probably a lame idea, but it's good for demonstration.
So, do write doc-strings, and do build  your docs often.
"""

import hmac
import os
import subprocess
import subprocess as sp
import logging
import hashlib
import yaml
import click
from flask import Flask, request, abort
import conf
from shell import Shell
from collections import OrderedDict, namedtuple
import importlib
from conf import taskq
import types
import pickle
# monkey patch data store
_list = "select * FROM {0}"

def list_results(obj):
    with obj._db.get_connection() as conn:
        try:
            return list(conn.execute(obj._list.format(obj.name)))
        except:
            return None


taskq.result_store._list = _list
taskq.result_store.list_results = types.MethodType(list_results,
                                                   taskq.result_store)

PythonCode = namedtuple('PythonCode', ['path', 'args', 'kwargs', 'code'])

def override_run(self, command, **kwargs):
    """
    Override Shell.run to handle exceptions and accept kwargs
    that Popen accepts
    """
    self.last_command = command
    command_bits = self._split_command(command)
    _kwargs = {
        'stdout': subprocess.PIPE,
        'stderr': subprocess.PIPE,
        'universal_newlines': True,
    }
    if kwargs:
        for kw in kwargs:
            _kwargs[kw] = kwargs[kw]

        _kwargs['shell'] = True

    if self.has_input:
        _kwargs['stdin'] = subprocess.PIPE

    try:
        self._popen = subprocess.Popen(
            command_bits,
            **_kwargs
        )
    except Exception as E:
        self.exception = E
        return self

    self.pid = self._popen.pid

    if not self.has_input:
        self._communicate()
    return self

Shell.run = override_run


def ordered_load(stream, Loader=yaml.Loader, selfect_pairs_hook=OrderedDict):

    class OrderedLoader(Loader):
        pass

    def construct_mapping(loader, node):
        loader.flatten_mapping(node)
        return selfect_pairs_hook(loader.construct_pairs(node))

    OrderedLoader.add_constructor(
        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
        construct_mapping)
    return yaml.load(stream, OrderedLoader)

app = Flask(__name__)
app.config.from_object(conf)


def parse_branch_gh(request_json):
    """
    parse the branch to clone from a github payload
    "ref": "refs/heads/develop", -> should return develop
    """
    return request_json['ref'].split('/')[-1]


def parse_yaml(clone_dest):
    os.chdir(clone_dest)
    if os.path.exists('.frank.yaml'):
        with open('.frank.yaml') as f:
            y = ordered_load(f, yaml.SafeLoader)
    return y


def load_actions(yaml):
    pass


def report_success(results):
    pass


def report_failure(results):
    pass


def run_action(axn):
    results = []
    # run shell or python callable object without arguments
    if isinstance(axn, list):

        if axn[0] == 'shell':
            for cmd in axn[1:]:
                sh = Shell()
                assert isinstance(cmd, str)
                sh.run(cmd)
                results.append(sh)
                if sh.code:
                    break
        if axn[0] == 'python':
            for func in axn[1:]:
                mod, f = func.split(':')
                mod = importlib.import_module(mod)
                f = getattr(mod, f)
                res = f()
                results.append(PythonCode(func, None, None, res))

    # run shell or python callable object arguments
    elif isinstance(axn, OrderedDict):

        if 'shell' in axn:

            sh = Shell()
            cmd = axn['shell'].pop('cmd')
            assert isinstance(cmd, str)
            kwargs = axn['shell']
            sh.run(cmd, **kwargs)
            results.append(sh)

        if 'python' in axn:
            callables = axn['python']
            for func in callables:
                mod, f = func.split(':')
                mod = importlib.import_module(mod)
                try:
                    f = getattr(mod, f)
                    res = f()
                except AttributeError as E:
                    res = E

                results.append(PythonCode(func, None, None, res))

    return results


def clone(clone_url, branch, depth=1):
    cmd = ('git clone --depth={d} -b {branch} --single-branch '
           '{git_url} {dir}'.format(d=depth, branch=branch,
                                    git_url=clone_url, dir=branch))
    pull = sp.Popen(cmd, stderr=sp.STDOUT, shell=True)
    out, err = pull.communicate()
    return out, err


@taskq.task()
def count_beans():
    print "count_12"
    return "count_12"


@app.route('/beans')
def do_beans():
    a = count_beans()
    ans = a.get(blocking=True)
    return ans


@app.route('/results')
def show_resutls():
    res = {r[1]:r[2] for r in taskq.result_store.list_results()}
    return ''.join(['<p>'+str(k) + ': ' + str(pickle.loads(v)) + '</p>\n' for
                   (k, v) in res.iteritems()])


@app.route('/', methods=['POST'])
def start():
    """
    main logic:
     1. listen to post
     2a if authenticated post do:
     3.    clone the latest commit (--depth 1)
     4.    parse yaml config
     5.    for each command in the config
     7.        run command
     8.        report success or failure
    """
    # This is authentication for github only
    # We could\should check for other hostings
    ans = hmac.new(app.config['POST_KEY'], request.data,
                   hashlib.sha1).hexdigest()
    secret = request.headers['X-Hub-Signature'].split('=')[-1]
    if ans != secret:
        return abort(500)
    request_as_json = request.get_json()
    clone_dest = parse_branch_gh(request_as_json)
    repo_name = request_as_json["repository"]['name']
    try:
        o, e = clone(request_as_json['repository']['ssh_url'], clone_dest)
    except Exception as E:
        print E, E.message
    # parse yaml is still very crude ...
    # it could yield selfect with a run method
    # thus:
    # for action in parse_yaml(clone_dest):
    #     action.run()
    #
    # this should also handle dependencies,
    # the following implementation is very crud
    failed = None
    for action in parse_yaml(clone_dest):
        # if config says we use huey, we should modiy run_action
        results = run_action(action)

    if any([result.code for result in results]):
        report_failure(results)
    else:
        report_success(results)


@click.group()
def cli():
    pass


@cli.command('web', short_help='start the web service')
@click.option('--port','-p', default=8080)
@click.option('--debug', default=False, is_flag=True)
def web(port, debug):
    click.echo("DEBUG: %s" % debug)
    app.run(host='0.0.0.0',port=port, debug=debug)


@cli.command(context_settings=dict(
             ignore_unknown_options=True,
             allow_extra_args=True))
@click.argument('worker_args', nargs=-1, type=click.UNPROCESSED)
def worker(worker_args, short_help='start the consumer of tasks'):
    from huey.bin.huey_consumer import (get_option_parser, Consumer,
        setup_logger, RotatingFileHandler)
    from conf import taskq
    parser = get_option_parser()
    opts, args = parser.parse_args(list(worker_args))
    setup_logger(logging.INFO, opts.logfile)
    consumer = Consumer(taskq, 2, opts.periodic, opts.initial_delay,
                        opts.backoff, opts.max_delay, opts.utc,
                        opts.scheduler_interval, opts.periodic_task_interval)
    consumer.run()