fork_demure_scripts/endlessh_scoreboard.py

#!/usr/bin/env python3

## Written by demure (demuredemeanor)
## This tool makes generates a static html page for showing an endlessh scoreboard.
## This program depends on ... to work


## TODO: Add flag to exclude IPs
## TODO: Add flag to disable GeoIp and sleep
## TODO: Add flag to change log location
## TODO: Add flag to change output file
## TODO: Use config file
## TODO: Sort default output, maybe by Org
    ## NOTE: best point to apply is the html table string


### Imports ### {{{
## Imports
import sys          ## Need for argv
import os           ## For file open
import datetime     ## For log line time and math
import re           ## For RegEx, <3
import time         ## For sleep between geoip lookups
import inspect      ## For cleaning up multi line strings
import hashlib      ## For hiding the IPs
import requests     ## For curl type requests
### End Imports ### }}}


### f_open_log_file ### {{{
def f_open_log_file():
    Log_Location = '/var/log/'      ## Declare log path
    Log_Name = 'endlessh.log'       ## Declare log name

    ## Open file
    for filename in os.listdir(Log_Location):
        if filename == Log_Name:
            fh  = open(os.path.join(Log_Location, Log_Name), "r")

    FilterArray = f_parse_log(fh)   ## Parse log and store
    # FilterArray.sort()              ## Sort array to make next part easier
    fh.close()                      ## Clean up and close file
    return(FilterArray)             ## Return FilterArray
### End f_open_log_file ### }}}


### f_parse_log ### {{{
def f_parse_log(fh):
    ## Declare variables
    CutOff = int(datetime.datetime.utcnow().strftime("%s")) - int(datetime.timedelta(days=7).total_seconds())
    LineParse = []
    LineHost = ""
    LineTime = ""
    FilterArray = []

    for x in fh:                                                            ## Iterate over file
        LineParse = x.split()                                               ## Split line into array
        if LineParse[1] == "CLOSE":                                         ## Only process CLOSE lines
            ## Convert line timestamp to epoch
            LineTime = int(datetime.datetime.strptime(LineParse[0], '%Y-%m-%dT%H:%M:%S.%fZ').strftime("%s"))
            if LineTime > CutOff:                                           ## Discard lines that are too old
                if re.match('host=::ffff:', LineParse[2]) is not None:
                    LineHost = re.sub('host=::ffff:', "", LineParse[2])     ## Clean up IPv4 host
                else:
                    LineHost = re.sub('host=', "", LineParse[2])            ## Clean up IPv6 host
                LineTime = re.sub('time=', "", LineParse[5])                ## Clean up time
                FilterArray.append((LineHost, LineTime))                    ## Add to Array
    return(FilterArray)                                                     ## Return FilterArray
### End f_parse_log ###}}}


### f_dict_computation ### {{{
def f_dict_computation(FilterArray):
    ## Declare variables
    HostDict = dict()
    Host = ""
    HostTime = ""
    GeoLookup = ""

    for x in FilterArray:
        Host = x[0]
        HostTime = float(x[1])

        if Host in HostDict:                        ## Test if Host already in HostDict
            HostDict[Host]["count"] += 1
            if HostTime < HostDict[Host]["min"]:    ## Replace min if new HostTime is less
                HostDict[Host]["min"] = HostTime
            if HostTime > HostDict[Host]["max"]:    ## Replace max if new HostTime is more
                HostDict[Host]["max"] = HostTime
            HostDict[Host]["sum"] += HostTime       ## Add current HostTime to sum
            HostDict[Host]["ave"] = HostDict[Host]["sum"] / HostDict[Host]["count"]
        else:                                       ## Initialize if not already in HostDict
            HostDict[Host] = dict()
            HostDict[Host]["count"] = 1
            HostDict[Host]["min"] = HostTime
            HostDict[Host]["max"] = HostTime
            HostDict[Host]["sum"] = HostTime
            HostDict[Host]["ave"] = HostTime / 1

            ### GeoIp lookup ### {{{
            ## Get newline output, splitline into array
            ## NOTE: Delay must be enabled if real lookup enabled!
            # time.sleep(0.5)     ## Add delay to not exceed lookup limit
            time.sleep(1)     ## Add delay to not exceed lookup limit
            GeoLookup = requests.get('http://ip-api.com/line/' + Host + '?fields=status,message,country,regionName,org').text.splitlines()
            # GeoLookup = ['success', 'DEBUG_Country', 'DEBUG_State', 'DEBUG_Org']      ## DEBUG pass
            # GeoLookup = ['fail', 'Reserved']                        ## DEBUG fail

            ## Check for lack of response
            if not bool(GeoLookup) == False:
                ## Check for successful lookup
                if GeoLookup[0] == 'success':
                    HostDict[Host]["country"] = GeoLookup[1]
                    HostDict[Host]["region"] = GeoLookup[2]
                    HostDict[Host]["org"] = GeoLookup[3]
                else:
                    ## If lookup had issues, set null
                    HostDict[Host]["country"] = "Error2"
                    HostDict[Host]["region"] = ""
                    HostDict[Host]["org"] = ""
            else:
                ## If lookup had issues, set null
                HostDict[Host]["country"] = "Error1"
                HostDict[Host]["region"] = ""
                HostDict[Host]["org"] = ""
            ### End GeoIp lookup ### }}}
    return(HostDict)
### End f_dict_computation ### }}}


### f_dict_total ### {{{
def f_dict_total(HostDict):
    ## Declare variables
    HighDict = dict()
    ServerDict = dict()
    Uniq = 0

    ## Process for HighDict
    for Host in HostDict:
        Time = HostDict[Host]["max"]
        Country = HostDict[Host]["country"]
        Count = HostDict[Host]["count"]
        Ave = HostDict[Host]["ave"]
        Sum = HostDict[Host]["sum"]

        if 'count' not in HighDict:                                             ## Initialize if not set
            HighDict["count"] = dict(Count=Count,Host=Host,Country=Country)
            HighDict["max"] = dict(Time=Time,Host=Host,Country=Country)
            HighDict["ave"] = dict(Time=0,Host="None Qualified",Country="")     ## First run not qualified to win
            HighDict["sum"] = dict(Time=0,Host="None Qualified",Country="")     ## First run not qualified to win
        else:
            if Count > HighDict["count"]["Count"]:                              ## Update count if greater
                HighDict["count"] = dict(Count=Count,Host=Host,Country=Country)
            if Time > HighDict["max"]["Time"]:                                  ## Update max time if greater
                HighDict["max"] = dict(Time=Time,Host=Host,Country=Country)
            if Ave > HighDict["ave"]["Time"] and Count >= 3:                    ## Update ave time if greater, and count three or more
                HighDict["ave"] = dict(Time=Ave,Host=Host,Country=Country)
            if Sum > HighDict["sum"]["Time"] and Count >= 2:                    ## Update total time if greater, and count two or more
                HighDict["sum"] = dict(Time=Sum,Host=Host,Country=Country)

    ## Process for ServerDict
    for Host in HostDict:
        Count = HostDict[Host]["count"]
        Sum = HostDict[Host]["sum"]

        if 'count' not in ServerDict:           ## Initialize if not set
            ServerDict["count"] = Count
            ServerDict["sum"] = Sum
            ServerDict["uniq"] = 1
        else:
            ServerDict["count"] += Count
            ServerDict["sum"] += Sum
            ServerDict["uniq"] += 1

    return(HighDict,ServerDict)
### End f_dict_total ### }}}


### f_gen_html ### {{{
def f_gen_html(HighDict,HostDict,ServerDict):
    ## Declare variables
    s_main_table = ""

    ### s_html_top ### {{{
    s_html_top = inspect.cleandoc("""
        <html>
            <center>
                <body>
                    <style> table {border-spacing: 0; border: 1px solid black; font-family: monospace;} th {cursor: pointer;} th, td {border: 1px solid black; border-collapse: collapse; padding: 2px;}</style>

                    <h2>Tar Pit Score Board</h2>

                    <h3>Info</h3>
                    <p>Here is a little Score Board of <strike>ssh attempts</strike> players from the last seven days.<BR>
                    To <i>"keep things fair"</i>, players who try too many times are given a time out.</p>
    """)
    ### End s_html_top ### }}}

    ### s_top_table ### {{{
    s_top_table = """
            <h3>Top Players</h3>
            <table id="topTable">
                <tr><th>Category</th><th>Value</th><th>Host</th><th>Country</th></tr>
                <tr><td>Most Attempts</td><td align=right>{Count}</td><td>{CHost}</td><td>{CCountry}</td></tr>
                <tr><td>Longest Conn</td><td align=right>{LTime}</td><td>{LHost}</td><td>{LCountry}</td></tr>
                <tr><td>Highest Ave*</td><td align=right>{ATime}</td><td>{AHost}</td><td>{ACountry}</td></tr>
                <tr><td>Highest Total*</td><td align=right>{TTime}</td><td>{THost}</td><td>{TCountry}</td></tr>
            </table>
            <p><small>* Top Average requires three connections, <BR>and Top Total two connections to qualify.</small></p>
            """.format(Count=HighDict["count"]["Count"],
                    CHost=HighDict["count"]["Host"],
                    CCountry=HighDict["count"]["Country"],
                    LTime=f_mins(HighDict["max"]["Time"]),
                    LHost=HighDict["max"]["Host"],
                    LCountry=HighDict["max"]["Country"],
                    ATime=f_mins(HighDict["ave"]["Time"]),
                    AHost=HighDict["ave"]["Host"],
                    ACountry=HighDict["ave"]["Country"],
                    TTime=f_mins(HighDict["sum"]["Time"]),
                    THost=HighDict["sum"]["Host"],
                    TCountry=HighDict["sum"]["Country"])
    ### End s_top_table ### }}}

    ### s_main_table_pre ### {{{
    s_main_table_pre = """
            <h3>Player List</h3>
                <p>In the past seven days there have been {TPlayers} players, and {TAttempts} connections.<BR>
                The average connection time is {TAve} minutes,<BR>
                with a total accumulated time of {TTime} minutes.<BR>
                <small>All <b>Conn</b>ection lengths are in minutes.<BR>
                Number columns may be sorted by clicking on their header.</small></p>
            <table id="mainTable">
                <thead>
                    <tr><th>Host</th><th>Country</th><th>Region</th><th>Org</th><th>Attempts</th><th>Shortest Conn</th><th>Longest Conn</th><th>Average Conn</th><th>Total Conns</th></tr>
                </thead>
                <tbody>""".format(TAttempts=ServerDict["count"],
                TPlayers=ServerDict["uniq"],
                TAve=f_mins(ServerDict["sum"] / ServerDict["count"]),
                TTime=f_mins(ServerDict["sum"]))
    ### End s_main_table_pre ### }}}

    ### s_main_table ### {{{
    for Host in HostDict:
        ## Make easy to read in code without added newlines
        s_main_table_temp = (
            "<tr>"
            "<td>{Host}</td>"
            "<td>{Country}</td>"
            "<td>{Region}</td>"
            "<td>{Org}</td>"
            "<td align=right>{Count}</td>"
            "<td align=right>{Min}</td>"
            "<td align=right>{Max}</td>"
            "<td align=right>{Ave:.3f}</td>"
            "<td align=right>{Sum:.3f}</td>"
            "</tr>"
            ).format(Host=Host,
                    Country=HostDict[Host]["country"],
                    Region=HostDict[Host]["region"],
                    Org=HostDict[Host]["org"],
                    Count=HostDict[Host]["count"],
                    Min=f_mins(HostDict[Host]["min"]),
                    Max=f_mins(HostDict[Host]["max"]),
                    Ave=f_mins(HostDict[Host]["ave"]),
                    Sum=f_mins(HostDict[Host]["sum"]))

        s_main_table = """{0}
                    {1}""".format(s_main_table, s_main_table_temp)
    ### End s_main_table ### }}}

    ### s_main_table_post ### {{{
    ## This was broken off of s_html_bottom so that the .format didn't error
    ## No inspect.cleandoc as it would collapse to beginning of line
    s_main_table_post = """
                </tbody>
            </table>

            <p>This page is brought to you by a python <a href="https://notabug.org/demure/scripts/src/master/endlessh_scoreboard.py">script</a>,<BR>
            which parses an <a href="https://github.com/skeeto/endlessh">endlessh</a> log file.</p>
            <p><b>Last updated {Date}</b></p>
    """.format(Date=datetime.datetime.utcnow().strftime("%a %d %b %Y %H:%M:%S UTC"))
    ### End s_main_table_post ### }}}

    ### s_html_bottom ### {{{
    s_html_bottom = inspect.cleandoc("""
                    <!-- from https://developer.mozilla.org/en-US/docs/Web/HTML/Element/table -->
                    <script>
                        for (let table of document.querySelectorAll('#mainTable')) {
                            for (let th of table.tHead.rows[0].cells) {
                                th.onclick = function(){
                                    const tBody = table.tBodies[0];
                                    const rows = tBody.rows;
                                    for (let tr of rows) {
                                        Array.prototype.slice.call(rows)
                                            .sort(function(tr1, tr2){
                                                const cellIndex = th.cellIndex;
                                                return tr1.cells[cellIndex].textContent.localeCompare(tr2.cells[cellIndex].textContent);
                                            })
                                            .forEach(function(tr){
                                                this.appendChild(this.removeChild(tr));
                                            }, tBody);
                                    }
                                }
                            }
                        }
                    </script>
                </body>
            </center>
        </html>
    """)
    ### End s_html_bottom ### }}}

    return(s_html_top,s_top_table,s_main_table_pre,s_main_table,s_main_table_post,s_html_bottom)
### End f_gen_html ### }}}


### f_write_out_file ### {{{
def f_write_out_file(s_output_html):
    Out_Path = '/var/www/pit/index.html'    ## Set path to output
    OutFile = open(Out_Path, "w")           ## Open file for writing
    for String in s_output_html:            ## Iterate over strings
        OutFile.write(String)               ## Write each string in order
    OutFile.close()                         ## Close file
### End f_write_out_file ### }}}


### f_hash ### {{{
## Make the Hashing a simple function call
def f_hash(Host):
        ## Has, and salt with repeatability
    return(hashlib.sha512(os.uname()[1].encode('utf-8') + Host.encode('utf-8')).hexdigest()[0:16])
### End f_hash ### }}}


### f_mins ### {{{
## Convert and round seconds to minutes
def f_mins(Time):
    return(round((Time / 60), 3))
### End f_mins ### }}}


## DEBUG
### f_debug_print ### {{{
def f_debug_print(Mode,HostDict,HighDict,ServerDict):
    if Mode == 0 or Mode == 1:
        for Host in HostDict:
            ## Print host matches
            print(("Host: {Host}, "
                "Country: {Country}, "
                "Region: {Region}, "
                "Org: {Org}, "
                "Count: {Count}, "
                "Min: {Min}, "
                "Max: {Max}, "
                "Ave: {Ave:.3f}, "
                "Sum: {Sum:.3f}"
                ).format(Host=Host,
                        Country=HostDict[Host]["country"],
                        Region=HostDict[Host]["region"],
                        Org=HostDict[Host]["org"],
                        Count=HostDict[Host]["count"],
                        Min=f_mins(HostDict[Host]["min"]),
                        Max=f_mins(HostDict[Host]["max"]),
                        Ave=f_mins(HostDict[Host]["ave"]),
                        Sum=f_mins(HostDict[Host]["sum"])))

    if Mode == 0 or Mode == 2:
        ## Print High Scores
        print(("Most Attempts: {Count}, "
                "Host: {Host}, "
                "Country: {Country}"
                ).format(Count=HighDict["count"]["Count"],
                        Host=HighDict["count"]["Host"],
                        Country=HighDict["count"]["Country"]))
        print(("Longest Conn: {Time}, "
                "Host: {Host}, "
                "Country: {Country}"
                ).format(Time=f_mins(HighDict["max"]["Time"]),
                        Host=HighDict["max"]["Host"],
                        Country=HighDict["max"]["Country"]))
        print(("Highest Average: {Time}, "
                "Host: {Host}, "
                "Country: {Country}"
                ).format(Time=f_mins(HighDict["ave"]["Time"]),
                        Host=HighDict["ave"]["Host"],
                        Country=HighDict["ave"]["Country"]))
        print(("Highest Total: {Time}, "
                "Host: {Host}, "
                "Country: {Country}"
                ).format(Time=f_mins(HighDict["sum"]["Time"]),
                        Host=HighDict["sum"]["Host"],
                        Country=HighDict["sum"]["Country"]))


    if Mode == 0 or Mode == 3:
        print(("Total Attempts: {TAttempts}, "
                "Total Players: {TPlayers}, "
                "Total Ave Conn: {TAve}, "
                "Total Conn Time: {TTime}"
                    ).format(TAttempts=ServerDict["count"],
                        TPlayers=ServerDict["uniq"],
                        TAve=f_mins(ServerDict["sum"] / ServerDict["count"]),
                        TTime=f_mins(ServerDict["sum"])))
### f_debug_print ### }}}


### Arguments ### {{{
## Parse the command line arguments
def parse_argv(argv):
   arg1=argv[1]

   ## Single augment
   if len(argv) == 2:
       ## Option help
       if arg1 == "-h" or arg1 == "--help" or arg1 == "help":
           msg_help()
           return

       ## Option version
       if arg1 == "-V" or arg1 == "--version" or arg1 == "version":
           msg_version()
           return

       ## If they get here, the didn't match; print help.
       print("["+arg1+"] invalid use, or invalid option!")
       msg_help()
       return

   ## Two Augments
   elif len(argv) > 2:
       arg2=argv[2]

       ## Option mute
       if arg1 == "-e" or arg1 == "--exclude" or arg1 == "exclude":
           if re.match("^(\d+)$", arg2):
               mute_toggle(arg2)
           else:
               print("["+arg2+"] is not a valid sink input!")
               msg_help()
           return
   return
### End Arguments ### }}}


## Print help message
def msg_help():
    help_msg = inspect.cleandoc("""
    Usage: endlessh_scoreboard.py [OPTION...]
        When run without OPTION, will DO A THING
        When run with BLAH, WILL DO OTHER THING

        -e, --exclude           Exclude IP [ip1] [ip2] [ip3]...
        -V, --version           Display version information
        -h, --help              Display this help message
    """)
    print(help_msg)
    quit()
    return


## Print version number message
def msg_version():
    version_msg="endlessh_scoreboard version: 0.3.2"
    print(version_msg)
    quit()
    return


## Main
def main(argv):
    ## Process input, if given
    if len(argv) > 1:
        parse_argv(argv)

    FilterArray = f_open_log_file()
    HostDict = f_dict_computation(FilterArray)
    HighDict,ServerDict = f_dict_total(HostDict)
    s_output_html=f_gen_html(HighDict,HostDict,ServerDict)
    f_write_out_file(s_output_html)

    # f_debug_print(3,HostDict,HighDict,ServerDict)


if __name__ == '__main__':
    main(sys.argv)