VaM 1.x Python script to copy VAR files from one VAM install to another

[stateofmindx5]

Like many of you I started using VAM and downloading vars and dependencies, but within a few days I realized that this was resulting in two major problems for me:
1) My install folder was getting too big
2) VAM was taking very long to load/initialize

I thought about removing vars that I didn't need, but once I looked into the AddonPackages folders, I realized it would be almost impossible to do this manually, since my var files number in the thousands, and there's no knowing which vars depend on which.

I looked around for a solution, but it seems there are none that exist, at least none that I could find. So, I decided to try my own.

I used Pharanesch's dependencyScanner script as the basis and then built my own code on top of it. The way it now works is, I have a main/new/clean VAM install with no vars in it, and a backup VAM folder which contains all the vars. I then take one specific var (mostly a scene) - let's call this desiredVar - that I want to run in VAM, copy that into the AddonPackages folder of my clean VAM install, and then run the script like so:

python var_copy.py -p "[clean_vam_path]" -c "[backup_vam_path]"

If running directly from within the clean VAM folder, you can run the script like this:

python var_copy.py -c "[backup_vam_path]"

This takes all the vars that are listed as dependencies in desiredVar, and it copies the correct versions of the dependency vars from the backup VAM folder and iteratively keeps doing this until there are no more dependencies to copy.

I've tested this on multiple vars in multiple situations, and it appears to work as expected. Unlike Pharanesch's script, my code doesn't currently do anything with the Custom folder or with vap files, though I haven't found this to be an issue so far. Please check it out and let me know if there are any bugs you encounter, and I'll try to fix them. Also, if someone would like to create a GUI based version of this tool, feel free to do so.

I've copied the script below, and also attached a file containing the script, the file will need its extension to be changed from .txt to .py before it works.

#!/usr/bin/python3
from zipfile import ZipFile
from glob import glob
from os.path import join, split, splitext, exists, basename, isdir, dirname
import sys
import argparse
from pathlib import Path
from json import load
import re
import shutil

# class representing list of all vars in a folder
class varList:
    def __init__(self) -> None:
        self.varDict = {}
        pass

    # add a var to the varList. If it's a new var, create a new varGroup, if it's an existing var, add it to the corresponding varGroup
    def add(self, varPath: str):
        varNameAndVersion = splitext(basename(varPath))[0]
        varNameAndVersionParts = varNameAndVersion.split('.')

        if varNameAndVersionParts[-1].isdigit():
            varName = '.'.join(varNameAndVersionParts[:-1])
        else:
            varName = varNameAndVersion

        if varName in self.varDict:
            self.varDict[varName].add(varPath)
        else:
            self.varDict[varName] = varGroup(varPath)
        return
   
    # get the given var's location from the varList. If asking for a specific version, get that version, otherwise get the latest version. If var does not exist in varList, return empty string
    def getVarPath(self, dependency: str) -> str:
        dependencyVersion = ""
        dependencyName = ""

        dependencyParts = dependency.split('.')
        if dependencyParts[-1].isdigit() or dependencyParts[-1] == 'latest':
            dependencyName = '.'.join(dependencyParts[:-1])
            dependencyVersion = dependencyParts[-1]
        else:
            dependencyName = dependency

        if dependencyName not in self.varDict:
            return ""
       
        dependencyVarGroup = self.varDict[dependencyName]

        if not dependencyVersion:
            return dependencyVarGroup.varVersions[dependencyName]
        elif dependencyVersion == "latest":
            return dependencyVarGroup.varVersions[dependencyVarGroup.latest]
        else:
            return dependencyVarGroup.varVersions[dependency]
           
# class representing a var and all its different versions
class varGroup:
    # create a new varGroup for a particular var and its versions
    def __init__(self, varPath: str):
        varNameAndVersion = splitext(basename(varPath))[0]
        varNameAndVersionParts = varNameAndVersion.split('.')

        if varNameAndVersionParts[-1].isdigit():
            self.varName = '.'.join(varNameAndVersionParts[:-1])
        else:
            self.varName = varNameAndVersion

        self.latest = varNameAndVersion
        self.varVersions = {varNameAndVersion: varPath}
        return
   
    # add a var to an existing varGroup, making sure to set the latest version of the var based on all the versions currently available in the varGroup
    def add(self, varPath: str):
        newVarNameAndVersion = splitext(basename(varPath))[0]
        newVarNameAndVersionParts = newVarNameAndVersion.split('.')

        if not newVarNameAndVersionParts[-1].isdigit():
            dpPrint(f"ERROR, varGrouping.add: new var does not have a version. newVar: {newVarNameAndVersion}, varColl: {self.varName}", args.output)
            return

        newVarName = '.'.join(newVarNameAndVersionParts[:-1])
        if self.varName != newVarName:
            dpPrint(f"ERROR, varGrouping.add: new var is not the same as the var collection. newVar: {newVarName}, varColl: {self.varName}", args.output)
            return          
       
        latestVarNameAndVersionParts = self.latest.split('.')
        if not latestVarNameAndVersionParts[-1].isdigit():
            dpPrint(f"ERROR, varGrouping.add: varColl does not have a version. varColl: {self.varName}, latest: {self.latest}", args.output)
            return
       
        if int(newVarNameAndVersionParts[-1]) > int(latestVarNameAndVersionParts[-1]):
            self.latest = newVarNameAndVersion

        self.varVersions[newVarNameAndVersion] = varPath
        return
 

def dpPrint(text, toFile: str = ''):
    print(text, flush=True)
    if toFile:
        try:
            with open(toFile, 'a', encoding='utf-8') as f:
                f.write(text + '\n')
        except Exception as error:
            print(f"ERROR, failed to write to {toFile}: {error}", file=sys.stderr, flush=True)
    return

def check_name_variation(varName: str, dependenciesList: list) -> bool:
    parts = varName.split('.')
    if parts[-1].isdigit() or parts[-1] == 'latest':
        baseName = '.'.join(parts[:-1])
    else:
        baseName = varName
    return varName in dependenciesList or f'{baseName}.latest' in dependenciesList


def check_name_variation_2(varName: str, dependenciesList: list) -> bool:
    parts = varName.split('.')
    if parts[-1].isdigit() or parts[-1] == 'latest':
        baseName = '.'.join(parts[:-1])
    else:
        baseName = varName
    for dep in dependenciesList:
        depParts = dep.split('.')
        depBaseName = '.'.join(depParts[:-1])
        if baseName == depBaseName:
            return True
    return False

def getDependencies(varFile: str) -> tuple[Exception|None, set]:
    try:
        with ZipFile(varFile) as myzip:
            with myzip.open('meta.json') as metaJson:
                data = load(metaJson)
            myzip.close()
     
        if 'dependencies' not in data:
            return None, set()
        return None, set(data['dependencies'].keys())
    except Exception as error:
        return error, set()

# Get all dependencies listed in presets
def getPresetDependencies(customPath: str) -> dict:
    allDependencies = {}
    presets = sorted(glob(join(customPath, 'Custom', '**/*.vap'), recursive=True))
 
    for vap in presets:
        try:
            with open(vap, 'r', encoding='utf-8') as f:
                pattern = r'"([^"]+):/[^"]*"'
                occurences = set(re.findall(pattern, f.read()))
        except Exception as error:
            dpPrint(f"ERROR, failed to read: {vap}: {error}", args.output)
            continue
 
        parts = Path(vap).parts
        presetName = '/'.join(parts[parts.index("Custom")+1:])
        dependenciesDict = {key: {presetName} for key in occurences}
        for key, val in dependenciesDict.items():
            if key in allDependencies:
                allDependencies[key].update(val)
            else:
                allDependencies[key] = val
    return allDependencies

# Get all dependencies listed in meta.json stored directly in folders
def getMetaDependencies(folder: str) -> tuple[Exception|None, set]:
    filename = join(folder, 'meta.json')
    try:
        with open(filename, 'r', encoding='utf-8') as metaJson:
            data = load(metaJson)
         
        if 'dependencies' not in data:
            return None, set()
        return None, set(data['dependencies'].keys())
    except Exception as error:
        return error, set()
   
# Find all dependencies in dependenciesList that are missing from varsList
def getMissingDependencies(dependenciesList, varsList) -> list:
    missingVars = []
    for var in dependenciesList:
        if not check_name_variation_2(var, varsList):
            missingVars.append(var)
    missingVars.sort()
    return missingVars

# copy missing dependencies from sourcePath to destinationPath
def copyMissingDependencies(destinationPath: str, sourcePath: str):
    # get all vars in the sourcePath's AddonPackages folder and create a list of these vars, along with what versions are available
    varsInCopyLocation = glob(join(sourcePath, 'AddonPackages', '**/*.var'), recursive=True)
    existingVarsList = varList()
    for var in varsInCopyLocation:
        existingVarsList.add(var)
   
    finished = False

    # keep running loop until no new dependencies are found
    while not finished:
        # get all vars in destinationPath and create a list of missing dependencies for these vars
        allVarList, allVarDependencies = getAllVars(destinationPath)
        allDepVars = list(allVarDependencies.keys())

        missingVars = getMissingDependencies(allDepVars, allVarList)

        # take the list of missing vars in the destination, and use the list of sourcePath vars to find the locations of the missing vars
        missingVarsPaths = {}
        for var in missingVars:
            missingVarsPaths[var] = existingVarsList.getVarPath(var)
       
        # now copy all the vars from source to destination, keeping a count of all vars copied
        copyDestinationPath = join(destinationPath, 'AddonPackages')
        count = 0

        for key in missingVarsPaths.keys():
            if missingVarsPaths[key]:
                shutil.copy(missingVarsPaths[key], copyDestinationPath)
                count += 1
            else:
                # dpPrint(f"ERROR, Could not find file for '{key}'", args.output)
                pass
       
        dpPrint(f"Copied {count} vars from '{sourcePath}' to '{destinationPath}'", args.output)

        # if no vars were copied in this loop, end loop
        if count == 0:
            finished = True

# Collect all vars, a complete list of all dependencies, and which vars uses which dependency
def getAllVars(directoryPath: str) -> tuple[set, dict]:
    allDependencies = {}
 
    vars = glob(join(directoryPath, 'AddonPackages', '**/*.var'), recursive=True)
    allVarList = {splitext(basename(var))[0] for var in vars}
 
    for varFilename in vars:
        if isdir(varFilename):
            error, dependencies = getMetaDependencies(varFilename)
        else:
            error, dependencies = getDependencies(varFilename)
 
        if (isinstance(error, Exception)):
            dpPrint(f"ERROR: {varFilename}: {error}", args.output)
            continue
 
        if (len(dependencies) == 0):
            continue

        dependenciesDict = {key: {split(varFilename)[1]} for key in dependencies}
        for key, val in dependenciesDict.items():
            if key in allDependencies:
                allDependencies[key].update(val)
            else:
                allDependencies[key] = val
 
    return allVarList, allDependencies
 

argParser = argparse.ArgumentParser(prog='DependencyScanner.py', description='Searches for dependencies inside all vars in a folder and if a name is given, lists whether there a vars that depend on it. If no name is given, it will instead list all vars in the folder which isn\'t a dependency of any other var. \nIf the copy command is given alongwith a copypath, it will copy all the dependencies missing in the ')
argParser.add_argument("-p", "--path",
                        type=Path,
                        default='.',
                        help="Path your VaM folder")
argParser.add_argument("-n", "--name",
                        type=str,
                        default='',
                        help="Name of a var file. Can be a partial name")
argParser.add_argument("-c", "--copy",
                        type=Path,
                        nargs='?',
                        default=None,
                        help="Copy dependencies from the path given next")
argParser.add_argument("-o", "--output",
                        type=Path,
                        nargs='?',
                        default=None,
                        const='.\\SearchVarOutput.txt',
                        help="File to save the results in (default: '.\\SearchVarOutput.txt')")

args = argParser.parse_args()

# Check the given arguments are valid
if not exists(args.path):
    print(f"The given path doesn't exist: {args.path}")
    exit(1)

if args.output:
    if isdir(args.output):
        print(f"The given output is a folder, not a file: {args.output}")
        exit(1)

    if not isdir(dirname(args.output)) and not basename(args.output) != '':
        print(f"The given output file folder path doesn't exist: {args.output}")
        exit(1)

    open(args.output, 'w', encoding='utf-8').close()

parts = args.name.split('.')
if parts[-1] == 'var':
    args.name = '.'.join(parts[:-1])

# Get a list of all vars and a dict of all dependencies from vars
allVarList, allVarDependencies = getAllVars(args.path)
allDepVars = list(allVarDependencies.keys())

# Get a dict of all dependencies from presets
allPresetDependencies = getPresetDependencies(args.path)
allDepVarsPresets = list(allPresetDependencies.keys())

# Combine all dependencies
allDependencies = allVarDependencies | allPresetDependencies

# Get a list of which vars in the folder that are not used as a dependency
noDependencyVar = [var for var in allVarList if not check_name_variation(var, allDepVars)]
noDependencyPreset = [var for var in allVarList if not check_name_variation(var, allDepVarsPresets)]
noDependency = sorted(list(set(noDependencyVar) & set(noDependencyPreset)), key=str.lower)

# If copy command is given, run the copy command from copypath to path
if args.copy:
    if not exists(args.copy):
        print(f"The given copypath doesn't exist: {args.copy}")
    dpPrint("Copy:", args.output)
    copyMissingDependencies(args.path, args.copy)
    exit(0)

# If no name is given, list all vars that aren't used as a dependency
if args.name == '':
    dpPrint(f"{len(noDependency)} vars are not used as a dependency:", args.output)
    for var in noDependency:
        dpPrint("\t" + var, args.output)
    if args.output:
        print(f"\nResults saved to: {args.output}")
    exit(0)

# List all vars with the given name that aren't used as a dependency
foundVars = [var for var in noDependency if args.name.lower() in var.lower()]
if len(foundVars) > 0:
    dpPrint(f"The following {str(len(foundVars)) + ' vars are' if len(foundVars) > 1 else 'var is'} not used as a dependency in other vars:", args.output)
    for var in foundVars:
        dpPrint("\t" + var, args.output)

# List all vars with the given name that depend on other vars
foundDepVars = sorted([var for var in allDependencies.keys() if args.name.lower() in var.lower()], key=str.lower)
if len(foundDepVars) > 0:
    dpPrint(f"\nThe following {len(foundDepVars)} iteration{'s' if len(foundDepVars) > 1 else ''} of '" + args.name + "' has other vars that depend on it:", args.output)

    # print all dependencies where the name is a key, or the name is part of the key
    for key in foundDepVars:
        if args.name.lower() in key.lower():
            if args.name != key:
                dpPrint(f"{key} ->", args.output)
            for var in sorted(allDependencies[key], key=str.lower):
                dpPrint("\t" + var, args.output)

if args.output:
    print(f"\nResults saved to: {args.output}")