mitmengine/scripts/filename_to_metadata.py

#!/usr/bin/env python3
import re
import sys
import os.path as ospath
import subprocess
from collections import defaultdict
from lxml import etree
import json

class MitmFingerprint:
    def __init__(self, mitm_name, mitm_type):
        if mitm_name == "none":
            mitm_name = ""
            mitm_type = ""
        self.raw_ua = ""
        self.name = mitm_name
        self.type = mitm_type

class UserAgentFingerprint:
    def __init__(self, device, os, os_version, browser, browser_version, platform):
        # handle some parsing exceptions
        if browser == "ipad":
            device="Tablet"
            os = "iOS"
            platform = "iPad"
            browser = "Safari"

        if browser == "iphone":
            device="Phone"
            os="iOS"
            platform="iPhone"
            browser="Safari"
        
        # use os version for browser version if not known
        if browser_version == "":
            browser_version = os_version

        # normalize browser
        browser = browser.replace("chrome", "Chrome")
        browser = browser.replace("firefox", "Firefox")
        browser = browser.replace("safari", "Safari")
        browser = browser.replace("android", "Android")
        browser = browser.replace("opera", "Opera")
        browser = browser.replace("silk", "Silk")
        browser = browser.replace("ie", "IE")
        browser = browser.replace("edge", "IE")

        # normalize browser version
        if not (re.match("^([0-9]+)\.([0-9]+)\.([0-9]+)$", browser_version)
            or re.match("^([0-9]+)\.([0-9]+)$", browser_version)
            or re.match("^([0-9]+)$", browser_version)):
            browser_version = ""

        # normalize device
        if browser == "Android":
            device = "Phone" # some of these could be tablets, but w/e
            platform = "Linux"
            os = "Android"
        device = device.replace("computer", "Computer")

        # normalize platform
        platform = platform.replace("android", "Linux")
        platform = platform.replace("ipod", "iPod")
        platform = platform.replace("ipad", "iPad")
        platform = platform.replace("iphone", "iPhone")
        platform = platform.replace("OS_X", "Mac")
        platform = platform.replace("mac", "Mac")
        platform = platform.replace("windows", "Windows")

        # normalize os
        os = os.replace("OS_X", "MacOSX")
        os = os.replace("mac", "MacOSX")
        os = os.replace("ios", "iOS")
        os = os.replace("android", "Android")
        os = os.replace("windows", "Windows")

        # normalize os version
        if os == "Windows":
            os_version = os_version.replace("XP", "5.1.0")
            os_version = os_version.replace("7", "6.1.0")
            os_version = os_version.replace("8.1", "6.3.0")
            os_version = os_version.replace("8", "6.2.0")
            os_version = os_version.replace("10", "10.0.0")
        elif os == "MacOSX":
            os_version = os_version.replace("El_Capitan", "10.11.0")
            os_version = os_version.replace("Yosemite", "10.10.0")
            os_version = os_version.replace("Mavericks", "10.9.0")
            os_version = os_version.replace("Mountain_Lion", "10.8.0")
            os_version = os_version.replace("Lion", "10.7.0")
            os_version = os_version.replace("Snow_Leopard", "10.6.0")
        if not (re.match("^([0-9]+)\.([0-9]+)\.([0-9]+)$", browser_version)
            or re.match("^([0-9]+)\.([0-9]+)$", browser_version)
            or re.match("^([0-9]+)$", browser_version)):
            os_version = ""

        self.browser = browser
        self.browser_version = browser_version
        self.os = os
        self.os_version = os_version
        self.platform = platform
        self.device = device

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument("filename", type=str, help="pcap containing TLS client hello")
    args = parser.parse_args()

    record = {}

    # metadata fields
    desc = ""
    os = ""
    os_version = ""
    browser = ""
    browser_version = ""
    device = ""
    platform = ""
    mitm_name = ""
    mitm_version = ""
    mitm_type = ""

    # only process files named handshake.pcap
    dirname, basename = ospath.split(args.filename)
    if basename != "handshake.pcap":
        sys.exit(1)

    # the lowest level directory contains a description
    _, desc = ospath.split(dirname)

    metadata = {}
    metadata["desc"] = desc
    metadata["comment"] = "generated by {}".format(sys.argv[0])
    metadata["handshake_pcap"] = args.filename

    # check if accompanying header pcap is present
    header_pcap = ospath.join(dirname, "header.pcap")
    if ospath.exists(header_pcap):
        metadata["header_pcap"] = header_pcap

    if "browsers" in dirname:
        # browser filenames should conform to this format
        m = re.match('^([^-]+)-([^-]+)-([^-]+)-([^-]+)-([^-]+)$', desc)
        if not m:
            sys.exit(1)
        device = m.group(1)
        os = m.group(2)
        os_version = m.group(3)
        browser = m.group(4)
        browser_version = m.group(5)
        platform = os

        ua_fp = UserAgentFingerprint(device, os, os_version, browser, browser_version, platform)
        metadata["ua_fingerprint"] = ua_fp.__dict__

    elif "antivirus-run2" in dirname:
        # mitm description should conform to this format (middle field can contain '-')
        m = re.match('^([^-]+)-([^-]+)-(.+)-([^-]+)-([^-]+)$', desc)
        if not m:
            sys.exit(1) 
        os = m.group(1)
        os_version = m.group(2)
        browser = m.group(4)
        browser_version = m.group(5)
        device = "Computer"
        mitm_name = m.group(3)
        mitm_type = "Antivirus"

        mitm_fp = MitmFingerprint(mitm_name, mitm_type)
        ua_fp = UserAgentFingerprint(device, os, os_version, browser, browser_version, platform)
        metadata["ua_fingerprint"] = ua_fp.__dict__
        metadata["mitm_fingerprint"] = mitm_fp.__dict__

    elif "middleboxes" in dirname:
        mitm_type = "Proxy"
        mitm_name = desc
        mitm_fp = MitmFingerprint(mitm_name, mitm_type)
        ua_fp = UserAgentFingerprint(device, os, os_version, browser, browser_version, platform)
        metadata["ua_fingerprint"] = ua_fp.__dict__
        metadata["mitm_fingerprint"] = mitm_fp.__dict__

        pass

    print(json.dumps(metadata))