searx-qt/searxqt/core/guard.py

########################################################################
#  Searx-Qt - Lightweight desktop application for Searx.
#  Copyright (C) 2020-2022  CYBERDEViL
#
#  This file is part of Searx-Qt.
#
#  Searx-Qt is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  Searx-Qt is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program.  If not, see <https://www.gnu.org/licenses/>.
#
########################################################################

""" Conditionally place failing instances on the blacklist or on a timeout (the
temporary blacklist).
"""

from copy import deepcopy

from searxqt.core.requests import ErrorType as RequestErrorType

from searxqt.utils.time import nowInMinutes

from searxqt.translations import _


class Condition:
    """ This describes the condition of a rule to trigger.
    """
    def __init__(
        self, errorType, amount=0,
        period=60, status=None
    ):
        """
        @param errorType: Error type to meet this conditiion.
        @type errorType: RequestErrorType

        @param amount: Amount of failed searches (in a row) of this errorType
                       to meet this condition.
        @type amount: uint

        @param period: The last x minutes where the fails have to occur in.
                       Set to 0 for forever.
        @type period: uint

        @param status: Status code of the failed search request. Set to None
                       when irrelevant.
        @type status: uint or None
        """
        self.__errorType = errorType
        self.__amount = amount
        self.__period = period
        self.__status = status

    @property
    def errorType(self):
        return self.__errorType

    @errorType.setter
    def errorType(self, errorType):
        self.__errorType = errorType

    @property
    def amount(self):
        return self.__amount

    @amount.setter
    def amount(self, amount):
        self.__amount = amount

    @property
    def period(self):
        return self.__period

    @period.setter
    def period(self, period):
        self.__period = period

    @property
    def status(self):
        return self.__status

    @status.setter
    def status(self, status):
        self.__status = status

    def serialize(self):
        return {
            "errorType": self.__errorType,
            "amount": self.__amount,
            "period": self.__period,
            "status": self.__status
        }

    def deserialize(self, data):
        self.__errorType = data.get('errorType', RequestErrorType.Other)
        self.__amount = data.get('amount', 0)
        self.__period = data.get('period', 0)
        self.__status = data.get('status', None)

    def evaluate(self, instanceLog):
        # First check if our errorType is present in the instanceLog
        if self.__errorType not in instanceLog:
            return False

        amountPeroidCount = 0
        startTime = nowInMinutes() - self.__period
        for logTime, statusCode, errorMsg in instanceLog[self.__errorType]:
            # Response status code
            if self.__status is not None:
                # This rule has defined a specific status code.
                if self.__status != statusCode:
                    # Don't count incidents that have different status code.
                    continue

            # Count occurrences in time-frame.
            if self.__period:
                if logTime - startTime > 0:
                    amountPeroidCount += 1
            elif self.__amount:
                amountPeroidCount += 1
            else:
                break

        return True if amountPeroidCount >= self.__amount else False


class ConsequenceType:
    Blacklist = 0
    Timeout = 1


ConsequenceTypeStr = {
    ConsequenceType.Blacklist: _("Blacklist"),
    ConsequenceType.Timeout: _("Timeout")
}


class Consequence:
    """ This describes the consequence for a instance of a rule that has it's
    condition met.
    """
    def __init__(self, type_=ConsequenceType.Timeout, duration=0):
        """
        @param consequence: Put the failing instance on the blacklist or
                            timeout? See class Consequences
        @type consequence: uint

        @param duration: Only used when consequence == Consequences.Timeout. It
                         is the duration in minutes the instance should be on
                         timeout.
        @type duration: uint
        """
        self.__type = type_
        self.__duration = duration

    @property
    def type(self):
        return self.__type

    @type.setter
    def type(self, type_):
        self.__type = type_

    @property
    def duration(self):
        return self.__duration

    @duration.setter
    def duration(self, duration):
        self.__duration = duration

    def serialize(self):
        return {
            "type": self.__type,
            "duration": self.__duration
        }

    def deserialize(self, data):
        self.__type = data.get('type', ConsequenceType.Timeout)
        self.__duration = data.get('duration', 0)


class Rule:
    def __init__(
        self,
        errorType=RequestErrorType.Other,
        consequenceType=ConsequenceType.Timeout,
        amount=0,
        period=0,
        duration=0,
        status=None
    ):
        self.__condition = Condition(
            errorType,
            amount=amount,
            period=period,
            status=status
        )
        self.__consequence = Consequence(consequenceType, duration)

    @property
    def condition(self):
        return self.__condition

    @property
    def consequence(self):
        return self.__consequence

    def meetsConditions(self, instanceLog):
        return self.__condition.evaluate(instanceLog)

    def serialize(self):
        return {
            "condition": self.__condition.serialize(),
            "consequence": self.__consequence.serialize()
        }

    def deserialize(self, data):
        self.__condition.deserialize(data.get('condition', {}))
        self.__consequence.deserialize(data.get('consequence', {}))


class Guard:
    """ Guard can have rules (condition and consequence) for failing searches.

    When enabled it logs failing instances so from that log can be evaluated
    (by the rules) whether the failing instance should be places on a timeout
    or the blacklist (the consequence).

    Guard itself doesn't handle the consequence itself but the consequence can
    be requested by other objects to handle.

    When enabled, each search response should be reported by calling
    `reportSearchResult()` for Guard to properly handle.

    The fail log of a instance will be cleared when a valid search response is
    reported to Guard, so instances have to fail in a row!

    The order of rules does matter! Rules with a lower index have higher
    priority.
    """
    # Defaults
    Enabled = False
    StoreLog = False
    LogPeriod = 7  # In days

    def __init__(self):
        self.__enabled = Guard.Enabled
        # Store logs on disk when True (bool).
        self.__storeLog = Guard.StoreLog
        # Max log period in days (uint) from now.
        self.__logPeriod = Guard.LogPeriod
        self.__log = {}
        self.__rules = []

    def reset(self):
        """ Reset Guard to default values, this will also clear the log and
        made rules.
        """
        self.__enabled = Guard.Enabled
        self.__storeLog = Guard.StoreLog
        self.clear()

    def clear(self):
        """ Clear the log and rules.
        """
        self.clearLog()
        self.__rules.clear()

    def clearLog(self):
        """ Clear the whole log.
        """
        self.__log.clear()

    def clearInstanceLog(self, instanceUrl):
        """ Clear the log of a specific instance by url.

        @param instanceUrl: Url of the instance
        @type instanceUrl: str
        """
        if instanceUrl in self.__log:
            del self.__log[instanceUrl]

    def doesStoreLog(self):
        """
        @return: Whether the log is stored on disk or not.
        @rtype: bool
        """
        return self.__storeLog

    def setStoreLog(self, state):
        """
        @param state: Store log on disk?
        @type state: bool
        """
        self.__storeLog = state

    def maxLogPeriod(self):
        """ Maximum log period in days.
        """
        return self.__logPeriod

    def setMaxLogPeriod(self, days):
        """ Set the maximum log period in days. This is only relevant when
        doesStoreLog() returns True.

        @param days: For how many days should the logs be stored.
        @type days: uint
        """
        self.__logPeriod = days

    def isEnabled(self):
        """ Returns whether Guard is enabled or not.
        """
        return self.__enabled

    def setEnabled(self, state):
        """ Enable/disable Guard.

        @param state: Enabled or disabled state of Guard as a bool.
        @type state: bool
        """
        self.__enabled = state

    def rules(self):
        """ Returns a list with Rules

        @rtype: list
        """
        return self.__rules

    def log(self):
        """ Returns the log

        @rtype: dict
        """
        return self.__log

    def addRule(
        self,
        errorType,
        consequenceType,
        amount=0,
        period=0,
        duration=0,
        status=None
    ):
        """ Add a new rule.

        A rule will only trigger when searches of a specific instance fails
        `amount` times in the last `period`, the fails have to be from the
        same `errorType`.

        The `consequenceType` defines what to do with the instance when this
        rule gets triggered, for now it can be put on a timeout or on the
        blacklist. When the type is
        `searxqt.core.guard.ConsequenceType.Timeout` a `duration` in minutes
        can be given to define for how long the timeout should last. When the
        `duration` is left to `0` with `Timeout` type it will be put on the
        timeout list until restart/switch-profile or manual removal.

        @param errorType: The search error type for this rule to trigger.
        @type errorType: searxqt.core.requests.ErrorType

        @param consequenceType: The action that will be taken on trigger.
        @type consequenceType: searxqt.core.guard.ConsequenceType

        @param amount: The amount of failures with the set errorType of this
                       rule that have to occur in a row to trigger this rule.
        @type amount: uint

        @param period: Period in minutes where the fails have to occur in.
                       `0` is always.
        @type period: uint

        """
        rule = Rule(
            errorType,
            consequenceType,
            amount=amount,
            period=period,
            duration=duration,
            status=status
        )
        self.__rules.append(rule)

    def moveRule(self, index, toIndex):
        """ Move a rule from index to a new index.

        @param index: Rule index
        @type index: uint

        @param toIndex: New Rule index
        @type toIndex: uint
        """
        rule = self.__rules.pop(index)
        self.__rules.insert(toIndex, rule)

    def delRule(self, index):
        """ Delete a rule by it's index

        @param index: Rule index
        @type index: uint
        """
        del self.__rules[index]

    def popRule(self, index=0):
        """ Pops a rule

        @param index: Rule index
        @type index: uint
        """
        return self.__rules.pop(index)

    def getRule(self, index):
        """ Get a rule by index

        @param index: Rule index
        @type index: uint

        @return: Guard Rule
        @rtype: searxqt.core.guard.Rule
        """
        return self.__rules[index]

    def serialize(self):
        """ Serialize this object.

        @return: Current data of this object.
        @rtype: dict
        """
        return {
            "rules": [rule.serialize() for rule in self.__rules],
            "log": self.__log if self.doesStoreLog() else {},
            "enabled": self.__enabled,
            "storeLog": self.doesStoreLog(),
            "maxLogDays": self.maxLogPeriod()
        }

    def deserialize(self, data):
        """ Deserialize data into this object.

        @param data: Data to set.
        @type data: dict
        """
        self.reset()
        for ruleData in data.get("rules", []):
            rule = Rule()
            rule.deserialize(ruleData)
            self.__rules.append(rule)

        self.setEnabled(data.get("enabled", Guard.Enabled))
        self.setStoreLog(data.get("storeLog", Guard.StoreLog))
        self.setMaxLogPeriod(data.get("maxLogDays", Guard.LogPeriod))

        if self.doesStoreLog():
            dataLog = deepcopy(data.get("log", {}))

            # Remove old logs
            deltaMax = self.__logPeriod * 24 * 60
            now = nowInMinutes()
            for url in dataLog:
                for errorType in dataLog[url]:

                    if not dataLog[url][errorType]:
                        # No log entries for this error type.
                        continue

                    index = len(dataLog[url][errorType]) - 1
                    while True:
                        logEntry = dataLog[url][errorType][index]
                        delta = int(now - logEntry[0])
                        if delta > deltaMax:
                            # This log enrtry is older then the max set log
                            # time, so delete this entry.
                            del dataLog[url][errorType][index]

                        if not index:
                            # Processed last log entry for this errorType.
                            break

                        index -= 1

            # Update log
            self.__log.update(dataLog)

    def getConsequence(self, instanceUrl):
        """ Get consequence for a instance by url. It will return `None` when
        none of the rules triggered.

        @param instanceUrl: url of the instance.
        @type instanceUrl: str

        @return: Consequence for this instance, should it be put on the
                 blacklist, a timeout or should nothing be done?
        @rtype: searxqt.core.guard.Consequence or None
        """
        if instanceUrl in self.__log:
            instanceLog = self.__log[instanceUrl]
            for rule in self.__rules:
                if rule.meetsConditions(instanceLog):
                    return rule.consequence
        return None

    def reportSearchResult(self, instanceUrl, searchResult):
        """ Search results (failed or not) should be reported to Guard through
        this method so Guard can evaluate with `getConsequence`. When the
        search succeeded, previous fail logs for this instance will be removed.
        So search fails have to occur in a row. Failed searches will be
        logged.

        @param instanceUrl: url of the instance.
        @type instanceUrl: str

        @param searchResult: Search result
        @type searchResult: searxqt.core.requests.Result
        """
        if bool(searchResult):
            # Clear the log for given instanceUrl when we have a valid result.
            # This will reset the counting of failures for the instance. This
            # also means that search failures have to occur in a row for one of
            # the rules to trigger.
            self.clearInstanceLog(instanceUrl)
        else:
            # Search failed; add the incident to the log.
            self.reportSearchFail(instanceUrl, searchResult)

    def reportSearchFail(self, instanceUrl, searchResult):
        """ Log failed search

        @param instanceUrl: url of the instance.
        @type instanceUrl: str

        @param searchResult: Search result
        @type searchResult: searxqt.core.requests.Result
        """
        if instanceUrl not in self.__log:
            # No previous log for this instance
            self.__log.update({instanceUrl: {}})

        errorType = searchResult.errorType()
        if errorType not in self.__log[instanceUrl]:
            # New errorType for this instance.
            self.__log[instanceUrl].update({errorType: []})

        self.__log[instanceUrl][errorType].append((
            nowInMinutes(),
            searchResult.statusCode(),
            searchResult.error()
        ))