Blender-Pipeline/network/network_multiuser.py

# THIS FILE IS A PART OF VCStudio
# PYTHON 3

###############################################################################

# In order for Multiuser to function. VCStudio should have a process on the 
# background can access to all the data, that will talk to the Multiuser
# server.

###############################################################################


##################### IMPLEMENTED FEATURES LIST ###############################

# [V] List of users     |   Important to know the usernames of all users
# [V] Assets            |   Important to have up to date assets everywhere
# [ ] Shots             |   Important to have up to date shots / and their assets
# [ ] Rendering         |   Ability to render on a separate machine.
# [ ] Story             |   Real time Sync of the story edito and the script writer
# [ ] Analytics         |   Real time Sync of History and Schedules.
# [ ] Messages          |   A little messaging system in the Multiuser window.

###############################################################################

import os
import sys
import time
import json
import socket
import random
import hashlib
import datetime
import threading
import subprocess

from UI import UI_elements
from settings import talk
from network import insure

time_format = "%Y/%m/%d-%H:%M:%S"

def client(win):
    
    ###########################################################################
    
    # This function is the thing that we need. It's going to run in it's own
    # thread. Separatelly from the rest of the program.
    
    ###########################################################################
   
    while True:
        try:
            # So the first thing that we want to do is to listen for a server
            # broadcasting himself. And when found connect to that server.

            connect(win)
            
            # Then it's going to ask us that project are we in and our username.
            
            server = win.multiuser["server"]
            
            # Then as soon as we are connected. We want to start the main loop. I know
            # that some stuff should be done early. But I guess it's better to do them
            # in the main loop. So basically the server will request a bunch of stuff. 
            # And when it sends no valid request. Or something like KEEP ALIVE then it's
            # a turn for as to send requests to server. Or nothing. The communication 
            # happens all the time.
            
            while win.multiuser["server"]:
            
                request = insure.recv(server)
                
                if request == "yours":
                    
                    ############################################################    
                    #                   WE REQUEST FROM SERVER                 #
                    ############################################################
                    
                    if win.multiuser["curs"]:
                        
                        # The get asset function
                        
                        get(win, list(win.multiuser["curs"].keys())[0], 
                            win.multiuser["curs"][list(win.multiuser["curs"].keys())[0]])
                        
                        try:
                            del win.multiuser["curs"][list(win.multiuser["curs"].keys())[0]]
                        except:
                            pass
                    
                    elif win.multiuser["request"]:
                        insure.send(server, win.multiuser["request"])
                        win.multiuser["request"] = []
                    
                    
                    elif not win.multiuser["users"]:
                        insure.send(server, "users")
                        win.multiuser["users"] = insure.recv(server)
                    
                    elif win.cur and win.cur != win.multiuser["last_request"]:
                        
                        win.multiuser["last_request"] = win.cur
                        
                        # If we are currently at some asset or shot. We want to
                        # send to the server the current state of the folder
                        # if such exists. So if we have the latest one. Every
                        # body else could send us a give request. But if somebody
                        # has the latest and it's not us. We could send them
                        # the get request. 
                        
                        # First step will be to check whitch cur are we in.
                        if win.url == "assets":
                            t = "/dev"
                        elif win.url == "script":
                            t = "/rnd"
                        else:
                            t = ""
                        
                        # Then we need to see if there is a folder to begin 
                        # with. Some scenes and some shots have no folder.
                        
                        if not os.path.exists(win.project+t+win.cur):
                            insure.send(server, "yours")
                        else:
                            
                            # If there is a folder let's get the timestamp.
                            timestamp = getfoldertime(win.project+t+win.cur)
                        
                            insure.send(server, ["at", t+win.cur, timestamp])
                    
                    else:
                        
                        insure.send(server, "yours")
                        
                        if win.url not in ["assets", "script", "analytics"]:
                            win.multiuser["last_request"] = ""
                            win.cur = ""
                else:   
                    
                    ############################################################    
                    #                  SERVER REQUESTS FROM US                 #
                    ############################################################
                    
                    
                    if request == "story":
                        
                        if not win.multiuser["story_check"]:
                            storytime = "1997/07/30-00:00:00"
                            win.multiuser["story_check"] = True
                        else:
                            storytime = gettime(win.project+"/pln/story.vcss")
                        
                        selectedtmp = win.story["selected"]
                        tmppointers = win.story["pointers"]
                        tmpcamera   = win.story["camera"]
                        
                        insure.send(server, [win.story, storytime])
                        
                        story = insure.recv(server)
                        
                        win.story = story[1].copy()
                        win.story["selected"] = selectedtmp
                        win.story["camera"]   = tmpcamera
                        win.story["pointers"] = tmppointers
                        
                        if story[0] in win.multiuser["users"]:
                            win.multiuser["users"][story[0]]["camera"] = story[1]["camera"]
                        

                    
                    elif request == "users":
                        win.multiuser["users"] = {}
                        
                        
                    elif request == "analytics":
                        if not win.multiuser["analytics_check"]:
                            analyticstime = "1997/07/30-00:00:00"
                            win.multiuser["analytics_check"] = True
                        else:
                            analyticstime = datetime.datetime.strftime(datetime.datetime.now(), time_format)
                        insure.send(server, [win.analytics, analyticstime])
                        
                        win.analytics = insure.recv(server)

                    elif request == "assets":
                        
                        assets = list_all_assets(win)
                        insure.send(server, assets)
                        new_assets = insure.recv(server)
                        
                        
                        for asset in new_assets:
                            
                            if asset not in assets or assets[asset][1] < new_assets[asset][1]:
                                
                                # If a given asset is not on our system or
                                # being updated on another system. We want to
                                # call for get() function. To get the new
                                # and up to date version of the asset. And if
                                # doesn't exist get the asset.
                                
                                # But it's better to do one by one.
                                try:
                                    win.multiuser["curs"]["/dev"+asset] = new_assets[asset][0]
                                except:
                                    pass
                        
                                
                    elif request[0] == "give":
                        give(win, request[1], server)            
                    
                    elif request[1] == "at":
                        print(request)
                        try:
                            # If we need an update let's update
                            if getfoldertime(win.project+request[2]) < request[3]:
                                win.multiuser["curs"][request[2]] = request[0]
                            # Else tell the world that we are the newest ones
                            elif getfoldertime(win.project+request[2]) > request[3]:
                                insure.send(server, ["at", request[2], getfoldertime(win.project+request[2])])
                                insure.recv(server)
                        except:
                            pass
                    
                    elif request[0] == "messages":
                        win.multiuser["messages"] = request[1]
                        win.multiuser["unread"] += 1 
                        win.scroll["multiuser_messages"] = 0-500*len(request[1])
                    
                    
                    insure.send(server, "yours")
                    
        except Exception as e:
            #raise()
            win.multiuser["server"] = False
            print("Connection Multiuser Error | "+str(e)+" | Line: "+str(sys.exc_info()[-1].tb_lineno))

def getfoldertime(path):
    
    # I noticed a problem with getting a last modification time for a directory.
    # It is not nearly the same time as the contents inside that directory. And
    # when for example I have /dev/chr/Moria folder. And I want to check is this
    # asset is newer here or somebody has a more up to date version. It checks
    # for only the time of the folder /dev/chr/Moria and not the contents of
    # that folder. Which is not cool. I want to get the newest time from all the
    # files and folders inside it. Including in the case of assets the 
    # /ast/chr/Moria.blend files. So this is why you see this function here.
    
    
    
    if os.path.isdir(path):
        
        # So basically we are doing it only if it's actually a directory. In
        # case there is an error. I didn't check. But who knows.
        
        # We need to get the full list of subdirectories and files. 
        
        allstuff = []
        
        if "/dev/" in path and os.path.exists(path[:path.find("/dev")]+"/ast"+path[path.find("/dev")+4:]+".blend"):
            allstuff.append(gettime(path[:path.find("/dev")]+"/ast"+path[path.find("/dev")+4:]+".blend"))
        
        for i in os.walk(path):
            if i[-1]:
                for b in i[-1]:
                    allstuff.append(gettime(i[0]+"/"+b))
            else:
                allstuff.append(gettime(i[0]))
        
        # Now let's find the biggest one
        biggest = allstuff[0]
        for i in allstuff:
            if i > biggest:
                biggest = i
        
        return biggest
        
    else:
        return gettime(path)

def gettime(path):
    
    # This function will get a pretty time for a given path. The last change
    # to the file or the folder recorded by the os.
    time_format = "%Y/%m/%d-%H:%M:%S"
    timestamp = os.path.getmtime(path)
    timestamp = datetime.datetime.fromtimestamp(timestamp)
    timestamp = datetime.datetime.strftime(timestamp, time_format)
    return timestamp
    
def connect(win):
    
    # This is going to be a function that connects to the server and makes a
    # handshake
    
    while not win.multiuser["server"]:
        
        # So the first step will be to listen for multiuser. 
        
        data = ""
        
        try:
            sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 
            sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            sock.bind(("255.255.255.255", 54545))
            
            
            data, addr = sock.recvfrom(1024)
            data = data.decode('utf8')

            sock.close()
            
        except:
            pass
        
        # If any data revieved. It's not nessesarily the server. So let's read it
        
        if data.startswith("VCStudio MULTIUSER SERVER"):
            try:
                data, ip, project = data.split(" | ")
            except:
                continue
            
            # So now we know the ip of the server. And the name of a project
            # that it's hosting. We need to check that our name is the same
            # and if yes. Connect to the server.
            
            if win.analytics["name"] == project:
                
                server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                server.connect((ip, 64646))
                win.multiuser["server"] = server
                win.multiuser["userid"] = str(server.getsockname()[0])+":"+str(server.getsockname()[1])
                
                insure.recv(server)
                insure.send(server, win.settings["Username"])
                insure.recv(server)
                insure.send(server, win.analytics["name"])
                
                print("Connected to Multiuser as: "+win.multiuser["userid"])
                
                # Adiing myself into the list of all users
                win.multiuser["users"][win.multiuser["userid"]] = {"username":win.settings["Username"], "camera":[0,0]}

def hash_file(f):
    try:
        BLOCKSIZE = 65536
        hasher = hashlib.md5()
        with open(f, 'rb') as afile:
            buf = afile.read(BLOCKSIZE)
            while len(buf) > 0:
                hasher.update(buf)
                buf = afile.read(BLOCKSIZE)
        return str(hasher.hexdigest())
    except:
        return "FOLDER"

        
def list_all_assets(win):
    
    # This function is listing all the asset CURs in the project. Not the shots
    # only the assets. Since we want to have what assets are being created. For
    # shots. It's done using the story editor file. So we don't really need it.
    
    allcurs = {}
    
    for c in ["chr", "veh", "loc", "obj"]:
        
        for i in os.listdir(win.project+"/dev/"+c):
            allcurs["/"+c+"/"+i] = [
                win.multiuser["userid"], 
                getfoldertime(win.project+"/dev/"+c+"/"+i)
                    ]
    
    return allcurs

def get_give_folder_list(project, folder):
    
    # This function will prepare our folder list for get and giv functions.
    
    path = folder
    astblend = path[:path.find("/dev")]+"/ast"+path[path.find("/dev")+4:]+".blend"
    
    fs = []
    
    if os.path.exists(project+astblend):
        fs.append([astblend, hash_file(project+astblend)])
    
    # There might not even be any folder as far as this function concerned
    # there is nothing in it if there is no folder. 
    
    try:
        for f in os.walk(project+folder):
            
            # If files in the folder
            if f[2]:
                for i in f[2]:  
                    fs.append([f[0].replace(project, "")+"/"+i, hash_file(f[0]+"/"+i)])
            # Else just put the folder in
            else:
                fs.append([f[0].replace(project, ""), "FOLDER"])
    
    except Exception as e:
        print("get_give_folder_list(): "+str(e))    
    
    return fs

def get(win, folder, userid):    
    
    # This function will get any folder from any other user using the connection
    # to the server.
    
    print("Trying to get: [", folder, "] From: [",  userid, "]")
    
    server = win.multiuser["server"]
    
    insure.send(server, ["get", userid, folder])
    
    # Next we will recieve the list of file / folders in the directory
    # we are looking for. With the MD5 hash of each. We do not want to 
    # download files that are identical between both machines.
    
    available = insure.recv(server)
    current   = get_give_folder_list(win.project, folder)
    getlist   = []
    
    # Now we need to compare between them to get a list of files we want.
    # Also at this stange we can already make the folders
    
    for f in available:
        if f not in current:
            if f[1] == "FOLDER":
                
                # If it's a folder there is nothing I need to download, we
                # already have the name. So let's just make it.
                try:
                    os.makedirs(win.project+f[0])
                except:
                    pass
                
            else:
                
                # If it's not a folder. And it does not exist. Let's actually
                # get it.
                
                getlist.append(f[0])
    
    # Now we want to send to that other user the "getlist" so he would know.
    # which files to send back to us. This is a bit harder communication then
    # just sending a big object contaning all of the files in it. Because I'm
    # using Json for the complex data structures. It's just not going to be cool
    # because at best it will convert the bytes of the files into strings of
    # text. Which are like 4 to 8 times larger in sizes. And at worst it will
    # fail complitelly. So what I will do is ask the files one by one. The
    # insure script knows how to deal with bytes objects so we are fine.
    
    insure.send(server, getlist)
    
    # Now let's just recieve the files and save them.
    for f in getlist:
        
        # We also want the folder to be make just in case.
        try:
            os.makedirs(win.project+f[:f.rfind("/")])
        except:
            pass
    
        data = open(win.project+f, "wb")
        data.write(insure.recv(server))
        data.close()
        
        insure.send(server, "saved")
        
    # Refrashing peviews for the images and stuff
    win.checklists = {}
    UI_elements.reload_images(win)
    
def give(win, folder, server):
    
    # This function will send to the server any folder that other users might
    # request.
    
    print("Someone wants: [", folder, "]")
    
    # We are going to send the list of files and folder and their hash values
    # to the other client. So it could choose what files does it wants. Not 
    # all files will be changed. So there is no need to copy the entire folder.
    
    insure.send(server, get_give_folder_list(win.project, folder))
    
    
    # The other user will select the files that he needs. Based on what's
    # changed. And will send us the short version of the same list.
    
    getlist = insure.recv(server)
    
    # Now let's send the actuall binaries of the files.
    
    for f in getlist:
        print("sending file:", f)
        data = open(win.project+f, "rb")
        data = data.read()
        insure.send(server, data)
        
        insure.recv(server)