gitlabircsender/code/apihandler.py

#! /usr/bin/env python
# -*- coding: utf8 -*-
#
#  gitlab irc sender
#  Copyright (C) 2016-2017  Andrei Karas (4144)
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 3 of the License, or
#  any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.

import requests
import threading
import time

from code.configuration import Configuration
from code.logger import Logger


class ApiHandler(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)
        self.die = False
        self.requests = []
        self.retry_builds = dict()


    def run(self):
        while (self.die == False):
            while len(self.requests) > 0:
                data = self.requests.pop()
                if len(data) < 2:
                    continue
                self.parse_command(data);
            time.sleep(Configuration.api_sleep_time)


    def add(self, data):
        self.requests.insert(0, data)


    def send_request(self, url, token, flag, cnt):
        cnt = cnt + 1
        while cnt > 0:
            Logger.addLog("Send get api request: {0}, retry count: {1}".format(url, cnt))
            cnt = cnt - 1
            try:
                if flag == True:
                    func = requests.post
                else:
                    func = requests.get
                request = func(url,
                    headers = {
                        "PRIVATE-TOKEN": token,
                        "Accept": "application/vnd.gitlab.v4+json"
                    }
                );
                request.raise_for_status()
                Logger.addLog("Request status: {0}".format(request.status_code))
                if request.status_code >= 200 and request.status_code <= 210:
                    return request
            except Exception as e:
                Logger.addError("Request exception: {0}".format(e))
            Logger.addError("Retry gitlab api request: " + url)
            time.sleep(Configuration.api_sleep_time)
        return None


    def parse_command(self, data):
        command = data[0]
        options = data[1]
        if command == "getlog":
            project_id = data[2]
            build_id = data[3]
            sha = data[4]
            request = self.request_build_log(project_id, build_id, options)
            if request is None:
                Logger.addError("Api request failed")
                return
            if len(request.text) == 0:
                Logger.addLog("Gitlab stalled build issue detected: {0}, sha: {1}".format(build_id, sha))
                request = self.request_build_retry(project_id, build_id, sha, options)
            elif request.text.find("ERROR: Job failed (system failure):") >= 0 or \
                request.text.find("ERROR: Build failed (system failure):") >= 0 or \
                request.text.find("\nFATAL: invalid argument                            \nERROR: Build failed: exit code 1\n") > 0 or \
                request.text.find("\nFATAL: invalid argument                            \nERROR: Job failed: exit code 1\n") > 0 or \
                request.text.find("ERROR: Job failed: canceled\n") > 0:
                Logger.addLog("Gitlab build issue detected. Retry build: {0}, sha: {1}".format(build_id, sha))
                request = self.request_build_retry(project_id, build_id, sha, options)
            elif (request.text.find("Using Shell executor...\nNo passwd entry for user 'gitlab-runner'\n") >= 0 or \
                request.text.find("ERROR: Job failed: exit code 137\n") >= 0):
                Logger.addLog("Gitlab runner issue detected. Retry build: {0}, sha: {1}".format(build_id, sha))
                request = self.request_build_retry(project_id, build_id, sha, options)
            elif request.text.find("remote: GitLab is not responding\nfatal: unable to access 'https://gitlab-ci-token:xxxxxxxxxxxxxxxxxxxx@gitlab.com/") >= 0:
                Logger.addLog("Gitlab runner issue detected. Retry build: {0}, sha: {1}".format(build_id, sha))
                request = self.request_build_retry(project_id, build_id, sha, options)
            elif request.text.find("ERROR: Job failed") < 0:
                Logger.addLog("Gitlab runner issue detected. Retry build: {0}, sha: {1}".format(build_id, sha))
                request = self.request_build_retry(project_id, build_id, sha, options)
            elif request.text.find("fatal: unable to access 'https://gitlab-ci-token:xxxxxxxxxxxxxxxxxxxx@") > 0 and \
                request.text.find(".git/': The requested URL returned error: 500\n/bin/bash: line ") > 0:
                Logger.addLog("Gitlab cloning issue detected. Retry build: {0}, sha: {1}".format(build_id, sha))
                request = self.request_build_retry(project_id, build_id, sha, options)
            elif request.text.find("ERROR: Uploading artifacts to coordinator... error  error=couldn't execute POST against http") >= 0 and \
                (request.text.find("FATAL: invalid argument                            \nERROR: Job failed: exit code 1") > 0 or \
                request.text.find("FATAL: invalid argument                            \nERROR: Job failed: exit status 1")) > 0:
                Logger.addLog("Gitlab artifacts upload issue detected. Retry build: {0}, sha: {1}".format(build_id, sha))
                request = self.request_build_retry(project_id, build_id, sha, options)
            elif request.text.find("ERROR: Job failed (system failure): Error: No such image: ") >= 0:
                Logger.addLog("Gitlab docker image download fail detected. Retry build: {0}, sha: {1}".format(build_id, sha))
                request = self.request_build_retry(project_id, build_id, sha, options)
            elif request.text.find("FATAL: invalid argument                           ") >= 0 and \
                request.text.find("[0KERROR: Job failed: exit code ") >= 0:
                Logger.addLog("Gitlab build issue detected. Retry build: {0}, sha: {1}".format(build_id, sha))
                request = self.request_build_retry(project_id, build_id, sha, options)
            elif request.text.find("[0KERROR: Job failed: execution took longer than 3h0m0s seconds") >= 0:
                Logger.addLog("Gitlab build issue detected. Retry build: {0}, sha: {1}".format(build_id, sha))
                request = self.request_build_retry(project_id, build_id, sha, options)
            elif request.text.find("ERROR: Job failed: Error response from daemon: error parsing "
                                   "HTTP 404 response body: invalid character 'p' after top-level value:") >= 0:
                Logger.addLog("Gitlab docker image download fail detected. Retry build: {0}, sha: {1}".format(build_id, sha))
                request = self.request_build_retry(project_id, build_id, sha, options)
        else:
            Logger.addError("ApiHandler: wrong command: {0}".format(data[0]))


    def request_build_log(self, project, build, options):
        return self.send_request(
            "{url}/v4/projects/{project}/jobs/{build}/trace".format(
                url = options.gitlab_api_url,
                project = project,
                build = build
            ),
            options.gitlab_api_token,
            False,
            options.gitlab_api_retry_count)


    def check_retry_build_count(self, sha, options):
        if sha not in self.retry_builds:
            self.retry_builds[sha] = 0
        return self.retry_builds[sha] < options.build_retry_count


    def request_build_retry(self, project, build, sha, options):
        if self.check_retry_build_count(sha, options) == False:
            Logger.addLog("Retry limit reached for: {0}".format(sha))
            return
        self.retry_builds[sha] = self.retry_builds[sha] + 1
        response = self.send_request(
            "{url}/v4/projects/{project}/jobs/{build}/retry".format(
                url = options.gitlab_api_url,
                project = project,
                build = build
            ),
            options.gitlab_api_token,
            True,
            options.gitlab_api_retry_count)
        return response


    def retry_build(self, project_id, build_id, sha, options):
        if self.check_retry_build_count(sha, options) == False:
            Logger.addLog("Retry limit reached for: {0}".format(sha))
            return
        self.add(("getlog", options, project_id, build_id, sha))