Blender-Pipeline/network/http_client.py

# (c) J.Y.Amihud 2023
# GPL-3 or any later version

import os
import json
import zlib
import time
import fnmatch
import threading
import hashlib
import urllib.request
import urllib.parse
import subprocess


# GTK module ( Graphical interface
import gi
gi.require_version('Gtk', '3.0')
from gi.repository import Gtk
from gi.repository import GLib
from gi.repository import Gdk
import cairo

# Own modules
from settings import settings
from settings import fileformats
from settings import talk
from project_manager import pm_project

#UI modules
from UI import UI_elements
from UI import UI_color
from UI import UI_math

from studio import story
from studio import analytics

# This is a http client. To work on projects remotely.

def csize(x):

    x = float(x)

    l = ["B","KB", "MB", "GB", "TB"]

    for i in range(5):
        if x > 1024:
            x = x / 1024
        else:
            return str(round(x, 2))+" "+l[i]
    return str(round(x, 2))+" "+l[i]

def go(win, website, path):

    url = website+path
    print("url", url)
    response = urllib.request.urlopen(url)
    data = response.read()
    text = data.decode("utf-8")
    data = json.loads(text)
    
    return data

def down(win, website, filename, fsize=100000):

    # If it is a blend file, we are in trouble.

    # if filename.endswith(".blend"):
    #     dp = go(win, website, "/blend"+filename)
    #     dplen = len(dp.get("files", {}))
    #     for f in dp.get("files", {}):
    #         try:
    #             hashis = hashlib.md5(open(win.project+f,'rb').read()).hexdigest()
    #         except:
    #             hashis = ""
    #         if hashis != dp.get("files", {}).get(f, {}).get("md5"):
    #             win.current["http-server"]["message"] = f[f.rfind("/")+1:]
    #             filesize = dp.get("files", {}).get(f, {}).get("filesize")
    #             print("trying to get:", f)
    #             pp = down(win, website, f, filesize)
    
    # Making folder for the files
    
    try:
        os.makedirs(win.project+filename[:filename.rfind("/")+1])
    except:
        pass

    # downloading the file
    
    url = website+"/download_z"+filename
    response = urllib.request.urlopen(url)
    for line in str(response.info()).split("\n"):
        if line.startswith("Content-length: "):
            try:
                fsize = int(line[line.find(" ")+1:])
            except:
                pass


        
    savef = open(win.project+filename, "wb")



    zd = b""
    sofar = 0
    chunk = response.read(8192)
    
    while chunk:
        zd = zd + chunk
        chunk = response.read(8192)
        sofar = sofar + 8192
        win.current["http-server"]["message"] = filename[filename.rfind("/")+1:]
        win.current["http-server"]["fileprog"] = sofar / fsize

    unz = zlib.decompress(zd)
        
    savef.write(unz)
    savef.close()

def download_missing_changed(win, cur, call):

    # This function downloads files from remote server using the
    # remote-folder-data data.

    website = win.analytics["remote-server-url"]

    for t in ["missing", "changed"]:
        while win.current["remote-folder-data"][cur][t]: # IDK WHAT I'M DOING!!!
            try:
                for f in win.current["remote-folder-data"][cur][t]:
                    fdata = win.current["remote-folder-data"][cur][t][f]
                    if fdata.get("to_download", True):
                        print("Downloading", f)
                        try:
                            down(win, website, f, fdata.get("filesize", 10000))
                        except Exception as e:
                            print(e)
                    del win.current["remote-folder-data"][cur][t][f]
            except:
                pass

    win.current["remote-folder-data"][cur]["downloading"] = False
    win.current["calls"][call]["var"] = False
                
def get_folder_info(win, folders, cur):

    # This function gets an information of what files should be changed. And outputs
    # the found data into the the global win thing, so it could accessed later.

    # Making sure we have the infrastructure.

    print("Getting Update Info About Folders")
    website = win.analytics["remote-server-url"]
    
    
    if "remote-folder-data" not in win.current:
        win.current["remote-folder-data"] = {}

    if cur not in win.current["remote-folder-data"]:
        win.current["remote-folder-data"][cur] = {"missing":{},
                                                  "changed":{}}

    win.current["remote-folder-data"][cur]["loaded"] = False

    def sort_file(filename, files, f, filedata=None):

        if not filedata:
            filedata = files.get(f, {})
        filedata["finished"] = False

        try:
            hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
        except:
            hashis = ""
        if not os.path.exists(win.project+filename):
            win.current["remote-folder-data"][cur]["missing"][filename] = filedata
        elif hashis != filedata.get("md5"):
            filedata["to_download"] = True
            win.current["remote-folder-data"][cur]["changed"][filename] = filedata
            
        
    # Part one: Getting all the data about the files.
        
    for foldername in folders:

        try:
            answer = go(win, website, "/list"+foldername)
            files = list(answer.get("files", {}).keys())
            for f in fnmatch.filter(files, folders.get(foldername, "*")):

                # Now we need to figure out whether we have the file
                # , whether we don't even have it ,
                # or whether we have a different version.

                fullfilename = foldername+"/"+f
                sort_file(fullfilename, answer.get("files", {}), f)

                


        except Exception as e:
            error = "Error: "+ str(e)
            print(error)


    # Part two: Unwrapping all the blend dependancies data.

    # Here we are going to store blend file that were checked already.
    win.current["remote-folder-data"][cur]["blend_checked"] = []

    # A function checking if we still have some unchecked .blends
    def not_checked():
        for t in ["missing", "changed"]:
            for i in win.current["remote-folder-data"][cur][t]:
                if i not in win.current["remote-folder-data"][cur]["blend_checked"] and i.endswith(".blend"):
                    return i
        return False

    nc = not_checked()
    
    while nc:
        try:
            dp = go(win, website, "/blend"+nc)

           
            for f in dp.get("files", {}):
                
                try:
                    sort_file(f,
                              dp.get("files", {}),
                              f[f.rfind("/")+1:],
                              filedata=dp.get("files", {})[f])
                except Exception as e:
                    print(e)
        except Exception as e:
            print(e)
                    
        win.current["remote-folder-data"][cur]["blend_checked"].append(nc)
            
        nc = not_checked()

    for t in ["missing", "changed"]:
        print(t+":")
        for i in win.current["remote-folder-data"][cur][t]:
            print("    "+i)

    win.current["remote-folder-data"][cur]["loaded"] = True

def get_folders(win):

    # This function will get specified subfolders
    win.current["http-server"]["message"] = "Getting Remote Folders"
    
    folder = win.current["http-server"]["args"]
    website = win.analytics["remote-server-url"]
    call = win.current["http-server"]["call"]

    try:
        answer = go(win, website, "/list"+folder)
        for n, i in enumerate(answer.get("folders", [])):
            
            win.current["http-server"]["progress"] = n/len(answer.get("folders", []))
            win.current["http-server"]["message"] = i
            try:

                os.makedirs(win.project+folder+"/"+i)
            except:
                pass
                
    except Exception as e:
        error = "Error: "+ str(e)
        print(error)
        win.current["http-server"]["message"] = error
        time.sleep(1)
        
    
    # Ending the process
    
    win.current["calls"][call]["var"] = False

def get_asset_files(win):

    # This function will get specified subfolders
    win.current["http-server"]["message"] = "Getting / Updating Files"
    
    folder = win.current["http-server"]["args"]
    website = win.analytics["remote-server-url"]
    call = win.current["http-server"]["call"]

    folder = folder.replace("//", "/")
    if folder.endswith("/"):
        folder = folder[:-1]
    
    try:
        answer = go(win, website, "/list"+folder)
        for n, i in enumerate(answer.get("files", [])):
            
            win.current["http-server"]["progress"] = n/len(answer.get("files", []))
            win.current["http-server"]["message"] = i

            filename = folder+"/"+i            
            
            try:
                hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
            except:
                hashis = ""

            if hashis != answer.get("files", {}).get(i, {}).get("md5"):
                filesize = answer.get("files", {}).get(i, {}).get("filesize")
                down(win, website, filename, filesize)


        win.current["http-server"]["message"] = "Getting Thumbnail"
        other_folder = folder+"/renders"
        print("otherfolder 1", other_folder)
        answer = go(win, website, "/list"+other_folder)
        for n, i in enumerate(answer.get("files", [])):
            
            win.current["http-server"]["progress"] = n/len(answer.get("files", []))
            win.current["http-server"]["message"] = i

            if i.startswith("Preview"):

                print("getting", i)
                
                filename = other_folder+"/"+i            

                try:
                    hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
                except:
                    hashis = ""

                if hashis != answer.get("files", {}).get(i, {}).get("md5"):
                    filesize = answer.get("files", {}).get(i, {}).get("filesize")
                    down(win, website, filename, filesize)

        win.current["http-server"]["message"] = "Getting Asset Blend"
        other_folder = folder[:folder.rfind("/")+1].replace("/dev/", "/ast/")
        print("otherfolder 2", other_folder, folder)
        answer = go(win, website, "/list"+other_folder)
        for n, i in enumerate(answer.get("files", [])):
            
            win.current["http-server"]["progress"] = n/len(answer.get("files", []))
            win.current["http-server"]["message"] = i

            if i.startswith(folder[folder.rfind("/")+1:]):

                print("getting", i)
                
                filename = other_folder+"/"+i            

                try:
                    hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
                except:
                    hashis = ""

                if hashis != answer.get("files", {}).get(i, {}).get("md5"):
                    filesize = answer.get("files", {}).get(i, {}).get("filesize")
                    down(win, website, filename, filesize)
            
                
    except Exception as e:
        error = "Error: "+ str(e)
        print(error)
        win.current["http-server"]["message"] = error
        time.sleep(1)

    
    # Ending the process
    
    win.current["calls"][call]["var"] = False
    
def get_files(win):

    # This function will get specified subfolders
    win.current["http-server"]["message"] = "Getting / Updating Files"
    
    folder = win.current["http-server"]["args"]
    website = win.analytics["remote-server-url"]
    call = win.current["http-server"]["call"]

    try:
        answer = go(win, website, "/list"+folder)
        for n, i in enumerate(answer.get("files", [])):
            
            win.current["http-server"]["progress"] = n/len(answer.get("files", []))
            win.current["http-server"]["message"] = i

            filename = folder+"/"+i            
            
            try:
                hashis = hashlib.md5(open(win.project+filename,'rb').read()).hexdigest()
            except:
                print("missing", filename)
                hashis = ""

            if hashis != answer.get("files", {}).get(i, {}).get("md5"):
                filesize = answer.get("files", {}).get(i, {}).get("filesize")
                down(win, website, filename, filesize)

            
                
    except Exception as e:
        error = "Error: "+ str(e)
        print(error)
        win.current["http-server"]["message"] = error
        time.sleep(1)
    
    # Ending the process
    
    win.current["calls"][call]["var"] = False
    
def get_essentials(win):

    # This function will get essential parts of the project.

    # Essentials are: Story, Analitycs, Project.progress  and Banner Image.

    essentials = [
        "/pln/story.vcss",
        "/set/analytics.json",
        "/set/project.progress",
        "/set/banner.png"]

    website = win.analytics["remote-server-url"]
    call = win.current["http-server"]["call"]
    
    for n, i in enumerate(essentials):
        win.current["http-server"]["progress"] = n/len(essentials)
        win.current["http-server"]["message"] = i

        try:
            down(win, website, i)
        except Exception as e:
            error = "Error: "+ str(e)
            print(error)
            win.current["http-server"]["message"] = error
            time.sleep(1)
            
            
    win.story        = story.load(win.project)
    win.analytics    = analytics.load(win.project)

    win.analytics["remote-server-url"] = website
    win.analytics["from-remote-server"] = True
    analytics.save(win.project, win.analytics)
            
    # Ending the process
    
    win.current["calls"][call]["var"] = False
    
    

def layer(win, call):
    
    # Making the layer
    surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, win.current['w'],
                                                      win.current['h'])
    layer = cairo.Context(surface)
    
    
    #text setting
    layer.select_font_face("Monospace", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL)
    
    UI_color.set(layer, win, "dark_overdrop")
    layer.rectangle(
        0,
        0,
        win.current["w"],
        win.current["h"],
        )
    layer.fill()
    
    # So it's going to be like a little window in the center of the VCStudio
    # with a simple UI. Probably like 2 things. Folder and a projectname.
    
    UI_color.set(layer, win, "node_background")
    UI_elements.roundrect(layer, win, 
        win.current["w"]/2-250,
        win.current["h"]/2-60,
        500,
        150,
        10)
    
    # Title of the operation. Incase the user forgot. 
    UI_elements.text(layer, win, "scan_project_title",
        win.current["w"]/2-250,
        win.current["h"]/2-35,
        500,
        30,
        10,
        fill=False,
        centered=True,
        editable=False)
    
    win.text["scan_project_title"]["text"] = win.current["http-server"]["message"]


    # File Progressbar
    UI_color.set(layer, win, "progress_background")
    UI_elements.roundrect(layer, win,
                          win.current["w"]/2-200,
                          win.current["h"]/2+10,
                          400,
                          20,
                          10,
                          tip="Hello")

    UI_color.set(layer, win, "progress_active")
    UI_elements.roundrect(layer, win,
                          win.current["w"]/2-200,
                          win.current["h"]/2+10,
                          400*max(0, min(1, win.current["http-server"]["fileprog"])),
                          20,
                          10,)

    
    # Progressbar
    UI_color.set(layer, win, "progress_background")
    UI_elements.roundrect(layer, win,
                          win.current["w"]/2-200,
                          win.current["h"]/2+40,
                          400,
                          20,
                          10,
                          tip="Hello")

    UI_color.set(layer, win, "progress_active")
    UI_elements.roundrect(layer, win,
                          win.current["w"]/2-200,
                          win.current["h"]/2+40,
                          400*max(0, min(1, win.current["http-server"]["progress"])),
                          20,
                          10,)
    
    return surface

def prompt_layer(win, call):
    
    # Making the layer
    surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, win.current['w'],
                                                      win.current['h'])
    layer = cairo.Context(surface)
    
    
    #text setting
    layer.select_font_face("Monospace", cairo.FONT_SLANT_NORMAL, cairo.FONT_WEIGHT_NORMAL)
    
    UI_color.set(layer, win, "dark_overdrop")
    layer.rectangle(
        0,
        0,
        win.current["w"],
        win.current["h"],
        )
    layer.fill()
    
    # So it's going to be like a little window in the center of the VCStudio
    # with a simple UI. Probably like 2 things. Folder and a projectname.
    
    UI_color.set(layer, win, "node_background")
    UI_elements.roundrect(layer, win, 
        win.current["w"]/2-350,
        100,
        700,
        win.current["h"]-200,
        10)
    
    

    cur = win.current["remote-folder-data"]["prompt"]


    if "downloading" not in win.current["remote-folder-data"][cur]:
        win.current["remote-folder-data"][cur]["downloading"] = False
    
    if not win.current["remote-folder-data"][cur]["downloading"]:


        # Exit button
        def do():
            win.current["calls"][call]["var"] = False


        UI_elements.roundrect(layer, win, 
            win.current["w"]/2+310,
            win.current["h"]-140,
            40,
            40,
            10,
            button=do,
            icon="cancel",
            tip=talk.text("cancel"))

        # Download button

        if win.current["remote-folder-data"][cur]["loaded"]:

            def do():
                win.current["remote-folder-data"][cur]["downloading"] = True

                start_downaloding = threading.Thread(target=download_missing_changed,
                                                 args=(win, cur, call, ))
                start_downaloding.setDaemon(True)
                start_downaloding.start()


            UI_elements.roundrect(layer, win, 
                win.current["w"]/2+270,
                win.current["h"]-140,
                40,
                40,
                10,
                button=do,
                icon="download",
                tip=talk.text("DownloadRemoteServer"))
        else:
            UI_color.set(layer, win, "text_normal")
            layer.set_font_size(15)
            layer.move_to(win.current["w"]/2,
                win.current["h"]-120,)
            layer.show_text(talk.text("StillLoadingRemoteData"))

    else:

        # Progress bar

        UI_color.set(layer, win, "progress_background")
        UI_elements.roundrect(layer, win,
                          win.current["w"]/2-200,
                              win.current["h"]-140,
                          400,
                          20,
                          10,
                          tip="Hello")

        UI_color.set(layer, win, "progress_active")
        UI_elements.roundrect(layer, win,
                          win.current["w"]/2-200,
                              win.current["h"]-140,
                          400*max(0, min(1, win.current["http-server"]["fileprog"])),
                          20,
                          10,)

        UI_color.set(layer, win, "text_normal")
        layer.set_font_size(15)
        layer.move_to(win.current["w"]/2-180,
                      win.current["h"]-145)
        layer.show_text(win.current["http-server"]["message"])


    # Top Tabs
        
    if "active_tab" not in win.current["remote-folder-data"][cur]:
        win.current["remote-folder-data"][cur]["active_tab"] = "changed"

    # Missing
    def do():
        win.current["remote-folder-data"][cur]["active_tab"] = "missing"

    if win.current["remote-folder-data"][cur]["active_tab"] == "missing":
        UI_color.set(layer, win, "progress_time")
        UI_elements.roundrect(layer, win,
                              win.current["w"]/2-340,
                              110,
                              330,
                              40,
                              10)
        
    UI_elements.roundrect(layer, win, 
                          win.current["w"]/2-340,
                          110,
                          330,
                          40,
                          10,
                          button=do,
                          icon="new_file",
                          tip=talk.text("MissingFiles"))

    
    
    
    UI_color.set(layer, win, "text_normal")
    layer.set_font_size(20)
    layer.move_to(win.current["w"]/2-290,
                   135)
    layer.show_text(str(len(win.current["remote-folder-data"][cur]["missing"]))+" "+talk.text("MissingFiles"))

    # Changed
    def do():
        win.current["remote-folder-data"][cur]["active_tab"] = "changed"

    if win.current["remote-folder-data"][cur]["active_tab"] == "changed":
        UI_color.set(layer, win, "progress_time")
        UI_elements.roundrect(layer, win,
                              win.current["w"]/2+10,
                              110,
                              330,
                              40,
                              10)
        
    UI_elements.roundrect(layer, win, 
                          win.current["w"]/2+10,
                          110,
                          330,
                          40,
                          10,
                          button=do,
                          icon="configure_file",
                          tip=talk.text("ChangedFiles"))
    
    UI_color.set(layer, win, "text_normal")
    layer.set_font_size(20)
    layer.move_to(win.current["w"]/2+60,
                   135)
    layer.show_text(str(len(win.current["remote-folder-data"][cur]["changed"]))+" "+talk.text("ChangedFiles"))
        


    # Search
    
    UI_elements.image(layer, win, "settings/themes/"\
        +win.settings["Theme"]+"/icons/search.png", 
        win.current["w"]/2,
        160, 
        40,
        40)
    
    UI_elements.text(layer, win, "http_client_search",
        win.current["w"]/2+50,
        160,
        250,
        40)


    # Toggle Visible

    if "toggle_visible" not in win.current["remote-folder-data"][cur]:
        win.current["remote-folder-data"][cur]["toggle_visible"] = True

    toggle = {"do":None}
        
    def do():
        win.current["remote-folder-data"][cur]["toggle_visible"] = not win.current["remote-folder-data"][cur]["toggle_visible"]
        toggle["do"] = win.current["remote-folder-data"][cur]["toggle_visible"]

    icon = "unchecked"
    if win.current["remote-folder-data"][cur]["toggle_visible"]:
        icon = "checked"
        
    UI_elements.roundrect(layer, win, 
                          win.current["w"]/2-340,
                          160,
                          330,
                          40,
                          10,
                          button=do,
                          icon=icon,
                          tip=talk.text("ToggleVisible"))

    UI_color.set(layer, win, "text_normal")
    layer.set_font_size(20)
    layer.move_to(win.current["w"]/2-290,
                   185)
    layer.show_text(talk.text("ToggleVisible"))
    
    # Clipping everything
    UI_elements.roundrect(layer, win, 
        win.current["w"]/2-350,
        200,
        700,
        win.current["h"]-360,
        10,
        fill=False)
    layer.clip()
    
    clip = [
        win.current["w"]/2-350,
        200,
        700,
        win.current["h"]-360]
    
    # Setting up the scroll
    if "http-prompt" not in win.scroll:
        win.scroll["http-prompt"] = 0

    
        
    current_Y = 100

    # Display list
    DisplayList = {}
    ActiveTab = win.current["remote-folder-data"][cur]["active_tab"]
    DisplayList = win.current["remote-folder-data"][cur][ActiveTab]

    
    try:
        for f in DisplayList:

            fdata = DisplayList[f]
            
            if fdata.get("finished"):
                continue

            if win.text["http_client_search"]["text"] and not UI_math.found(f, win.text["http_client_search"]["text"]):
                continue
            
            ticon = "unchecked"
            if fdata.get("to_download", True):
                ticon = "checked"

            if toggle["do"] != None:
                fdata["to_download"] = toggle["do"]
                
            def do():
                fdata["to_download"] = not fdata.get("to_download", True)

            # Filetype icon
            ext = f[f.rfind(".")+1:]
            if ext in fileformats.images:
                icon = "image"
            elif ext in fileformats.videos:
                icon = "video"
            elif ext in fileformats.sounds:
                icon = "mus"
            elif ext == "blend":
                icon = "blender"
            elif ext == "progress":
                icon = "checklist"
            else:
                icon = "file"

            
                
            UI_elements.roundrect(layer, win, 
                                  win.current["w"]/2-315,
                                  110 + current_Y + win.scroll["http-prompt"],
                                  660,
                                  40,
                                  10,
                                  button=do,
                                  icon=ticon,
                                  tip=f)

            UI_elements.image(layer, win, "settings/themes/"+win.settings["Theme"]+"/icons/"+icon+".png", 
                              win.current["w"]/2-275,
                              110 + current_Y + win.scroll["http-prompt"],
                              40,
                              40)

            UI_color.set(layer, win, "text_normal")
            layer.set_font_size(20)
            layer.move_to(win.current["w"]/2-220,
                          current_Y + win.scroll["http-prompt"] + 135)
            layer.show_text(f[f.rfind("/")+1:])

            current_Y += 50
    except Exception as e:
        print(e)
        pass
    
    UI_elements.scroll_area(layer, win, "http-prompt", 
        int(win.current["w"]/2-350),
        200,
        700,
        win.current["h"]-360,
        current_Y,
        bar=True,
        mmb=True,
        url="http-server-prompt"
        )
    
    return surface