o3de/scripts/build/Jenkins/Jenkinsfile

#!/usr/bin/env groovy
/*
 * Copyright (c) Contributors to the Open 3D Engine Project.
 * For complete copyright and license terms please see the LICENSE at the root of this distribution.
 *
 * SPDX-License-Identifier: Apache-2.0 OR MIT
 *
 */

import groovy.json.JsonOutput
import java.util.regex.Pattern

SCRIPTS_PATH = 'scripts/build' // Root path for all CI scripts
PIPELINE_CONFIG_FILE = "${SCRIPTS_PATH}/Jenkins/o3de.json"
INCREMENTAL_BUILD_SCRIPT_PATH = "${SCRIPTS_PATH}/bootstrap/incremental_build_util.py"
EBS_SNAPSHOT_SCRIPT_PATH = "${SCRIPTS_PATH}/tools/ebs_snapshot.py"
PIPELINE_RETRY_ATTEMPTS = 3

// Number of minutes of inactivity in all stages of the pipeline to reach the timeout
PIPELINE_TIMEOUT = 60

EMPTY_JSON = readJSON text: '{}'

ENGINE_REPOSITORY_NAME = 'o3de'
ENGINE_DEVELOPMENT_BRANCH = 'development'
ENGINE_URL = "https://github.com/o3de/o3de.git"

COMMIT_DOMAIN_NAME = ''
COMMIT_ORGANIZATION_NAME = ''
COMMIT_REPOSITORY_NAME = ''

@NonCPS
def doit()
{
    def httpsPattern = ~'https://(.+)/(.+)/(.+).git'
    def commitUrl = scm.userRemoteConfigs.url[0]
    def matcher = commitUrl =~ httpsPattern
    echo "matcher ${commitUrl} =~ ${httpsPattern}"
    matcher.find()
    COMMIT_DOMAIN_NAME = matcher.group(1)
    echo "matcher.group(1) = COMMIT_DOMAIN_NAME = ${COMMIT_DOMAIN_NAME}"
    COMMIT_ORGANIZATION_NAME = matcher.group(2)
    echo "matcher.group(2) = COMMIT_ORGANIZATION_NAME = ${COMMIT_ORGANIZATION_NAME}"
    COMMIT_REPOSITORY_NAME = matcher.group(3)
    echo "matcher.group(3) = COMMIT_REPOSITORY_NAME = ${COMMIT_REPOSITORY_NAME}"
}
doit()

// Branches with build snapshots
BUILD_SNAPSHOTS = ['development', 'stabilization/2305']

// Build snapshots with empty snapshot (for use with 'SNAPSHOT' pipeline parameter)
BUILD_SNAPSHOTS_WITH_EMPTY = BUILD_SNAPSHOTS + ''

// The default build snapshot to be selected in the 'SNAPSHOT' pipeline parameter
DEFAULT_BUILD_SNAPSHOT = BUILD_SNAPSHOTS_WITH_EMPTY.get(0)

// Branches with build snapshots as comma separated value string
env.BUILD_SNAPSHOTS = BUILD_SNAPSHOTS.join(",")

def pipelineProperties = []

def pipelineParameters = [
    // Build/clean Parameters
    // The CLEAN_OUTPUT_DIRECTORY is used by ci_build scripts. Creating the parameter here passes it as an environment variable to jobs and is consumed that way
    booleanParam(defaultValue: false, description: 'Deletes the contents of the output directory before building. This will cause a \"clean\" build. NOTE: does not imply CLEAN_ASSETS', name: 'CLEAN_OUTPUT_DIRECTORY'),
    booleanParam(defaultValue: false, description: 'Deletes the contents of the output directories of the AssetProcessor before building.', name: 'CLEAN_ASSETS'),
    booleanParam(defaultValue: false, description: 'Deletes the contents of the workspace and forces a complete pull.', name: 'CLEAN_WORKSPACE'),
    booleanParam(defaultValue: false, description: 'Recreates the volume used for the workspace. The volume will be created out of a snapshot taken from main.', name: 'RECREATE_VOLUME'),
    booleanParam(defaultValue: false, description: 'Cancels AR immediately on any failure in the pipeline and marks it as failed', name: 'FAIL_FAST'),
    booleanParam(defaultValue: false, description: 'Deletes the volume used for the workspace after the pipeline steps are completed.', name: 'DISCARD_VOLUME')
]

def PlatformSh(cmd, lbl = '', winSlashReplacement = true, winCharReplacement = true) {
    if (env.IS_UNIX) {
        sh label: lbl,
           script: cmd
    } else {
        if (winSlashReplacement) {
            cmd = cmd.replace('/','\\')
        }
        if (winCharReplacement) {
            cmd = cmd.replace('%', '%%')
        }
        bat label: lbl,
            script: cmd
    }
}

def PlatformMkdir(path) {
    if (env.IS_UNIX) {
        sh label: "Making directories ${path}",
           script: "mkdir -p ${path}"
    } else {
        def win_path = path.replace('/','\\')
        bat label: "Making directories ${win_path}",
            script: "mkdir ${win_path}."
    }
}

def PlatformRm(path, sudo=false) {
    if (env.IS_UNIX) {
        if (sudo) {
           sh label: "Removing ${path}",
              script: "sudo rm ${path}"
        } else {
            sh label: "Removing ${path}",
              script: "rm ${path}"
        }
    } else {
        def win_path = path.replace('/','\\')
        bat label: "Removing ${win_path}",
            script: "del /Q ${win_path}"
    }
}

def PlatformRmDir(path, sudo=false) {
    if (env.IS_UNIX) {
        if (sudo) {
           sh label: "Removing ${path}",
              script: "if [ -d ${path} ]; then sudo rm -rf ${path}; fi"
        } else {
           sh label: "Removing ${path}",
              script: "if [ -d ${path} ]; then rm -rf ${path}; fi"
        }
    } else {
        def win_path = path.replace('/','\\')
        bat label: "Removing ${win_path}",
            script: "IF exist ${win_path} rd /s /q ${win_path}"
    }
}

def IsPullRequest(branchName) {
    // temporarily using the name to detect if we are in a PR
    // In the future we will check with github
    return branchName.startsWith('PR-')
}

def IsPeriodicPipeline(pipelineName) {
    return pipelineName.startsWith('periodic-')
}

def IsJobEnabled(branchName, buildTypeMap, pipelineName, platformName) {
    if (IsPullRequest(branchName) && !IsPeriodicPipeline(pipelineName)) {
        return buildTypeMap.value.TAGS && buildTypeMap.value.TAGS.contains(pipelineName)
    }
    def job_list_override = params.JOB_LIST_OVERRIDE ? params.JOB_LIST_OVERRIDE.tokenize(',') : ''
    if (!job_list_override.isEmpty()) {
        return params[platformName] && job_list_override.contains(buildTypeMap.key);
    } else {
        return params[platformName] && buildTypeMap.value.TAGS && buildTypeMap.value.TAGS.contains(pipelineName)
    }
}

def GetRunningPipelineName(JENKINS_JOB_NAME) {
    // If the job name has an underscore
    def job_parts = JENKINS_JOB_NAME.tokenize('/')[0].tokenize('_')
    if (job_parts.size() > 1) {
        return [job_parts.take(job_parts.size() - 1).join('_'), job_parts[job_parts.size()-1]]
    }
    return [job_parts[0], 'default']
}

@NonCPS
def RegexMatcher(str, regex) {
    def matcher = (str =~ regex)
    return matcher ? matcher.group(1) : null
}

def LoadPipelineConfig(String pipelineName, String branchName) {
    echo 'Loading pipeline config'
    def pipelineConfig = {}
    pipelineConfig = readJSON file: PIPELINE_CONFIG_FILE
    PlatformRm(PIPELINE_CONFIG_FILE)
    pipelineConfig.platforms = EMPTY_JSON

    // Load the pipeline configs per platform
    pipelineConfig.PIPELINE_CONFIGS.each { pipeline_config ->
        def platform_regex = pipeline_config.replace('.','\\.').replace('*', '(.*)')
        if (!env.IS_UNIX) {
            platform_regex = platform_regex.replace('/','\\\\')
        }
        echo "Searching platform pipeline configs in ${pipeline_config} using ${platform_regex}"
        for (pipeline_config_path in findFiles(glob: pipeline_config)) {
            echo "\tFound platform pipeline config ${pipeline_config_path}"
            def platform = RegexMatcher(pipeline_config_path, platform_regex)
            if(platform) {
                pipelineConfig.platforms[platform] = EMPTY_JSON
                pipelineConfig.platforms[platform].PIPELINE_ENV = readJSON file: pipeline_config_path.toString()
            }
            PlatformRm(pipeline_config_path.toString())
        }
    }

    // Load the build configs
    pipelineConfig.BUILD_CONFIGS.each { build_config ->
        def platform_regex = build_config.replace('.','\\.').replace('*', '(.*)')
        if (!env.IS_UNIX) {
            platform_regex = platform_regex.replace('/','\\\\')
        }
        echo "Searching configs in ${build_config} using ${platform_regex}"
        for (build_config_path in findFiles(glob: build_config)) {
            echo "\tFound config ${build_config_path}"
            def platform = RegexMatcher(build_config_path, platform_regex)
            if(platform) {
                pipelineConfig.platforms[platform].build_types = readJSON file: build_config_path.toString()
            }
        }
    }
    return pipelineConfig
}

def GetBuildEnvVars(Map platformEnv, Map buildTypeEnv, String pipelineName) {
    def envVarMap = [:]
    platformPipelineEnv = platformEnv['ENV'] ?: [:]
    platformPipelineEnv.each { var ->
        envVarMap[var.key] = var.value
    }
    platformEnvOverride = platformEnv['PIPELINE_ENV_OVERRIDE'] ?: [:]
    platformPipelineEnvOverride = platformEnvOverride[pipelineName] ?: [:]
    platformPipelineEnvOverride.each { var ->
        envVarMap[var.key] = var.value
    }
    buildTypeEnv.each { var ->
        // This may override the above one if there is an entry defined by the job
        envVarMap[var.key] = var.value
    }

    // Environment that only applies to to Jenkins tweaks.
    // For 3rdParty downloads, we store them in the EBS volume so we can reuse them across node
    // instances. This allow us to scale up and down without having to re-download 3rdParty
    envVarMap['LY_PACKAGE_DOWNLOAD_CACHE_LOCATION'] = "${envVarMap['WORKSPACE']}/3rdParty/downloaded_packages"
    envVarMap['LY_PACKAGE_UNPACK_LOCATION'] = "${envVarMap['WORKSPACE']}/3rdParty/packages"

    return envVarMap
}

def GetEnvStringList(Map envVarMap) {
    def strList = []
    envVarMap.each { var ->
        strList.add("${var.key}=${var.value}")
    }
    return strList
}

def GetCrashArtifactDir() {
    if (env.IS_UNIX) {
        def uname = sh(script: 'uname', returnStdout: true).trim()
        if (uname.startsWith('Darwin')) {
            return '~/Library/Logs/DiagnosticReports/' // MacOS
        }
        else {
            return '/var/lib/apport/coredump/'
        }
    } else {
        return "$LOCALAPPDATA\\CrashDumps"
    }
}

def GetRemoteConfig(repositoryName, repositoryUrl) {
    def theRemoteConfig = [
        name: repositoryName,
        url: repositoryUrl,
        credentialsId: scm.userRemoteConfigs.credentialsId[0]
    ]
    return theRemoteConfig
}

def GetEngineRemoteConfig() {
    return GetRemoteConfig(ENGINE_REPOSITORY_NAME, ENGINE_URL)
}

def CheckoutEngineBootstrapScripts(branchName, userRemoteConfig = [GetEngineRemoteConfig()]) {
    checkout([$class: 'GitSCM',
        branches: [[name: "*/${branchName}"]],
        doGenerateSubmoduleConfigurations: false,
        extensions: [
            [$class: 'PruneStaleBranch'],
            [$class: 'AuthorInChangelog'],
            [$class: 'SparseCheckoutPaths', sparseCheckoutPaths: [
                [ $class: 'SparseCheckoutPath', path: "${SCRIPTS_PATH}/Jenkins/" ],
                [ $class: 'SparseCheckoutPath', path: "${SCRIPTS_PATH}/bootstrap/" ],
                [ $class: 'SparseCheckoutPath', path: "${SCRIPTS_PATH}/Platform" ],
                [ $class: 'SparseCheckoutPath', path: "${SCRIPTS_PATH}/tools/" ]
            ]],
            // Shallow checkouts break changelog computation. Do not enable.
            [$class: 'CloneOption', noTags: false, reference: '', shallow: false]
        ],
        submoduleCfg: [],
        userRemoteConfigs: userRemoteConfig
    ])
}

def CheckoutRepo(String branchName, userRemoteConfigs, boolean disableSubmodules = false, boolean setEnvCommit = false) {
    echo "Checkout Repo:"
    if (fileExists('.git')) {
        // If the repository after checkout is locked, likely we took a snapshot while git was running,
        // to leave the repo in a usable state, garbage collect.
        def indexLockFile = '.git/index.lock'
        if (fileExists(indexLockFile)) {
            PlatformSh('git gc', 'Git GarbageCollect')
        }
        if (fileExists(indexLockFile)) { // if it is still there, remove it
            PlatformRm(indexLockFile)
        }
    }

    def random = new Random()
    def retryAttempt = 0
    retry(5) {
        if (retryAttempt > 0) {
            sleep random.nextInt(60 * retryAttempt)  // Stagger checkouts to prevent HTTP 429 (Too Many Requests) response from CodeCommit
        }
        retryAttempt = retryAttempt + 1
        checkout scm: [
            $class: 'GitSCM',
            branches: [[name: branchName]],
            extensions: [
                [$class: 'PruneStaleBranch'],
                [$class: 'AuthorInChangelog'],
                [$class: 'SubmoduleOption', disableSubmodules: disableSubmodules, recursiveSubmodules: true],
                [$class: 'GitLFSPull'],
                [$class: 'CheckoutOption', timeout: 60]
            ],
            userRemoteConfigs: userRemoteConfigs
        ]
    }

    if (setEnvCommit) {
        // CHANGE_ID is used by some scripts to identify uniquely the current change (usually metric jobs)
        PlatformSh('git rev-parse HEAD > commitid', 'Getting commit id')
        env.CHANGE_ID = readFile file: 'commitid'
        env.CHANGE_ID = env.CHANGE_ID.trim()
        PlatformRm('commitid')

        // CHANGE_DATE is used by the installer to provide some ability to sort tagged builds in addition to BRANCH_NAME and CHANGE_ID
        commitDateFmt = '%%cI'
        if (env.IS_UNIX) commitDateFmt = '%cI'

        PlatformSh("git show -s --format=${commitDateFmt} ${env.CHANGE_ID} > commitdate", 'Getting commit date', winSlashReplacement=true, winCharReplacement=false)
        env.CHANGE_DATE = readFile file: 'commitdate'
        env.CHANGE_DATE = env.CHANGE_DATE.trim()
        PlatformRm('commitdate')
    }
}

def HandleDriveMount(String snapshot, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean recreateVolume = false) {
    unstash name: 'incremental_build_script'

    def pythonCmd = ''
    if(env.IS_UNIX) pythonCmd = 'sudo -E python3 -u '
    else pythonCmd = 'python3 -u '

    if(recreateVolume) {
        PlatformSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume', winSlashReplacement=false)
    }
    PlatformSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action mount --snapshot-hint ${snapshot} --repository_name ${repositoryName} --project ${projectName} --pipeline ${pipeline} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Mounting volume', winSlashReplacement=false)

    if(env.IS_UNIX) {
        sh label: 'Setting volume\'s ownership',
            script: """
            if sudo test ! -d "${workspace}"; then
                sudo mkdir -p ${workspace}
                cd ${workspace}/..
                sudo chown -R lybuilder:root .
            fi
            """
    }
}

//this should assume it will always be executed from the workspace root
def NoControlFile(repoName, processed_repos, o3de_project, defer_enable_gems) {
    echo "NoControlFile(reponame:${repoName}, processed_repos:${processed_repos})"

    processed_repos.add(repoName)

    def ext = ''
    if(env.IS_UNIX) {
        ext = '.sh'
    }

    //There is no control file so we must determine if this is a composite repo or a singular repo
    dir(repoName) {

        //a singular object repo will have one of these o3de json files in the root
        if(fileExists("engine.json")) {
            PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --engine-path ${workspace}/${repoName} || echo>nul", "Registering engine in ${workspace}/${repoName}")
        }
        else if(fileExists("gem.json")) {
            PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --gem-path ${workspace}/${repoName} || echo>nul", "Registering gem in ${workspace}/${repoName}")
            if(o3de_project) {
                defer_enable_gems.add([COMMAND: "${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} enable-gem --gem-path ${workspace}/${repoName} --project-name ${o3de_project}", ECHO: "Enabling ${gemName} in ${o3de_project}"])
            }
        }
        else if(fileExists("project.json")) {
            PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --project-path ${workspace}/${repoName} || echo>nul", "Registering project in ${workspace}/${repoName}")
        }
        else if(fileExists("template.json")) {
            PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --template-path ${workspace}/${repoName} || echo>nul", "Registering template in ${workspace}/${repoName}")
        }
        else if(fileExists("restricted.json")) {
            PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --restricted-path ${workspace}/${repoName} || echo>nul", "Registering restricted in ${workspace}/${repoName}")
        }
        else {
            //A standard composite repo has the form
            //root
            //  Engines
            //  Gems
            //  Projects
            //  Templates
            //  Restricted
            def found = false
            if(fileExists("Engines")) {
                PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --all-engines-path ${workspace}/${repoName}/Engines || echo>nul", "Registering all engines in ${workspace}/${repoName}/Engines")
                found = true
            }
            if(fileExists("Gems")) {
                PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --all-gems-path ${workspace}/${repoName}/Gems || echo>nul", "Registering all gems in ${workspace}/${repoName}/Gems")
                if(o3de_project) {
                    defer_enable_gems.add([COMMAND: "${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} enable-gem --all-gem-paths ${workspace}/${repoName}/Gems --project-name ${o3de_project}", ECHO: "Enabling all gems in ${o3de_project}"])
                }
                found = true
            }
            if(fileExists("Projects")) {
                PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --all-projects-path ${workspace}/${repoName}/Projects || echo>nul", "Registering all projects in ${workspace}/${repoName}/Projects")
                found = true
            }
            if(fileExists("Templates")) {
                PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --all-templates-path ${workspace}/${repoName}/Templates || echo>nul", "Registering all templates in ${workspace}/${repoName}/Templates")
                found = true
            }
            if(fileExists("Restricted")) {
                PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --all-restricted-path ${workspace}/${repoName}/Restricted || echo>nul", "Registering all restricted in ${workspace}/${repoName}/Restricted")
                found = true
            }
            if(!found) {
                echo "WARNING: No objects found in repo ${repoName}!"
            }
        }
    }

    return processed_repos
}

//this should assume it will always be executed from the workspace root
def ExecuteControlFile(controlFile, repoName, processed_repos, o3de_project, defer_enable_gems) {
    echo "ExecuteControlFile(controlFile:${controlFile}, reponame:${repoName}, processed_repos:${processed_repos})"

    processed_repos.add(repoName)

    def ext = ''
    if(env.IS_UNIX) {
        ext = '.sh'
    }

    //register all the objects specified in this controlFile
    if(controlFile.REGISTER_ENGINES) {
        echo 'REGISTER_ENGINES'
        controlFile.REGISTER_ENGINES.each { engineName ->
            PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --engine-path ${workspace}/${repoName}/${engineName} || echo>nul", "Registering engine in ${workspace}/${repoName}/${engineName}")
        }
    }
    if(controlFile.REGISTER_GEMS) {
        echo 'REGISTER_GEMS'
        controlFile.REGISTER_GEMS.each { gemName ->
            PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --gem-path ${workspace}/${repoName}/${gemName} || echo>nul", "Registering gem in ${workspace}/${repoName}/${gemName}")
        }
    }
    if(controlFile.REGISTER_PROJECTS) {
        echo 'REGISTER_PROJECTS'
        controlFile.REGISTER_PROJECTS.each { projectName ->
            PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --project-path ${workspace}/${repoName}/${projectName} || echo>nul", "Registering project in ${workspace}/${repoName}/${projectName}")
        }
    }
    if(controlFile.REGISTER_TEMPLATES) {
        echo 'REGISTER_TEMPLATES'
        controlFile.REGISTER_TEMPLATES.each { templateName ->
            PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --template-path ${workspace}/${repoName}/${templateName} || echo>nul", "Registering template in ${workspace}/${repoName}/${templateName}")
        }
    }
    if(controlFile.REGISTER_RESTRICTED) {
        echo 'REGISTER_RESTRICTED'
        controlFile.REGISTER_RESTRICTED.each { restrictedName ->
            PlatformSh("${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} register --restricted-path ${workspace}/${repoName}/${restrictedName} || echo>nul", "Registering restricted in ${workspace}/${repoName}/${restrictedName}")
        }
    }
    //queue enable all gems specified in the control file
    if(controlFile.ENABLE_GEMS && o3de_project) {
        controlFile.ENABLE_GEMS.each { gemName ->
            if(o3de_project) {
                defer_enable_gems.add([COMMAND: "${workspace}/${ENGINE_REPOSITORY_NAME}/scripts/o3de${ext} enable-gem --gem-path ${workspace}/${repoName}/${gemName} --project-name ${o3de_project}", ECHO: "Enabling ${workspace}/${repoName}/${gemName} for ${o3de_project}"])
            }
        }
    }

    //check out all repos
    if(controlFile.REPOS) {
        echo 'REPOS'
        controlFile.REPOS.each { repo ->
            //stop cyclical repos
            if(processed_repos.contains(repo.NAME)) {
                echo "Already processed ${repo.NAME}"
            }
            else {
                def repoControlFile = {}
                def foundRepoControlFile = false
                dir(repo.NAME) {
                    CheckoutRepo(repo.BRANCH, [GetRemoteConfig(repo.NAME, repo.URL)])

                    if(fileExists('.automatedtesting.json')) {
                        echo "Loading ${repo.URL} .automatedtesting.json"
                        repoControlFile = readJSON file: '.automatedtesting.json'
                        foundRepoControlFile = true
                    }
                }

                if(foundRepoControlFile) {
                    //There is a control file, execute it
                    processed_repos = ExecuteControlFile(repoControlFile, repo.NAME, processed_repos, o3de_project, defer_enable_gems)
                } else {
                    processed_repos = NoControlFile(repo.NAME, processed_repos, o3de_project, defer_enable_gems)
                }
            }
        }
    }

    return processed_repos
}

def PreBuildCommonSteps(Map pipelineConfig, Map params, String snapshot, String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType, String workspace, boolean mount = true, boolean disableSubmodules = false) {
    echo 'Starting pre-build common steps...'

    if (mount) {
        if(env.RECREATE_VOLUME?.toBoolean()){
            echo 'Starting to recreating drive...'
            HandleDriveMount(snapshot, repositoryName, projectName, pipeline, branchName, platform, buildType, workspace, true)
        } else {
            echo 'Starting to mounting drive...'
            HandleDriveMount(snapshot, repositoryName, projectName, pipeline, branchName, platform, buildType, workspace, false)
        }
    }

    // Cleanup previous repo location, we are currently at the root of the workspace, if we have a .git folder
    // we need to cleanup. Once all branches take this relocation, we can remove this
    if(env.CLEAN_WORKSPACE?.toBoolean() || fileExists("${workspace}/.git")) {
        if(fileExists(workspace)) {
            echo 'Clean workspace...'
            PlatformRmDir(workspace, true)
        }
    }
    
    def engine_project = ''
    
    if (pipelineConfig.DEFAULT_PROJECT) {
        engine_project = pipelineConfig.DEFAULT_PROJECT
    }
    else {
        engine_project = 'AutomatedTesting'
    }

    dir(workspace) {

        // Add folder where we will store the 3rdParty downloads and packages
        dir('3rdParty') {
        }

        def processed_repos = []
        def defer_enable_gems = []

        //we always get the engine repo first and get python, setEnvCommit only if the repo is the engine repo
        def engineAutomatedTestingControlFile = {}
        def engineAutomatedTestingControlFileFound = false
        dir(ENGINE_REPOSITORY_NAME) {
            echo "Checkout Engine: ${ENGINE_URL}"
            def setEnvCommit = false
            def the_branch = branchName
            def remoteConfig = scm.userRemoteConfigs
            if(COMMIT_REPOSITORY_NAME == ENGINE_REPOSITORY_NAME) {
                //the pr is for the engine repo, so we want to use the passed in branch name and record the change id and change date
                setEnvCommit = true
            } else {
                //this pr is for another repo so we want to use the development branch for the engine and do not set the commit id or change date
                the_branch = ENGINE_DEVELOPMENT_BRANCH
                remoteConfig = [GetEngineRemoteConfig()]
            }

            //checkout the engine branch
            CheckoutRepo(the_branch, remoteConfig, disableSubmodules, setEnvCommit)

            // Get python and clean.
            // Always run the clean step, the scripts detect what variables were set, but it also cleans if
            // the NODE_LABEL has changed
            if(env.IS_UNIX) {
                sh label: 'Getting python',
                   script: "${pipelineConfig.PYTHON_DIR}/get_python.sh"
                sh label: "Running ${platform} clean",
                   script: "${pipelineConfig.PYTHON_DIR}/python.sh -u ${pipelineConfig.BUILD_ENTRY_POINT} --platform ${platform} --type clean"
            } else {
                bat label: 'Getting python',
                    script: "${pipelineConfig.PYTHON_DIR}/get_python.bat"
                bat label: "Running ${platform} clean",
                    script: "${pipelineConfig.PYTHON_DIR}/python.cmd -u ${pipelineConfig.BUILD_ENTRY_POINT} --platform ${platform} --type clean"
            }

            //register the engine
            def ext = ''
            if(env.IS_UNIX) {
                ext = '.sh'
            }
            PlatformSh("scripts/o3de${ext} register --this-engine", "Registering this engine, will generate the o3de_manifest")

            //if the engine has an .automatedtesting.json execute it
            if(fileExists('.automatedtesting.json')) {
                echo 'Loading Engine .automatedtesting.json'
                engineAutomatedTestingControlFile = readJSON file: '.automatedtesting.json'
                engineAutomatedTestingControlFileFound = true
            }
        }
        if(engineAutomatedTestingControlFileFound) {
            processed_repos = ExecuteControlFile(engineAutomatedTestingControlFile, ENGINE_REPOSITORY_NAME, processed_repos, engine_project, defer_enable_gems)
        }

        //if we are not committing to the engine repo then pull that repo in here
        if(COMMIT_REPOSITORY_NAME != ENGINE_REPOSITORY_NAME) {
            echo "Commit repo: ${scm.userRemoteConfigs.url[0]}"
            def automatedTestingControlFile = {}
            def automatedTestingControlFileFound = false
            dir(COMMIT_REPOSITORY_NAME) {
                CheckoutRepo(branchName, scm.userRemoteConfigs, disableSubmodules, true)

                //if the repo has an .automatedtesting.json execute it
                if(fileExists('.automatedtesting.json')) {
                    echo "Loading ${COMMIT_REPOSITORY_NAME} .automatedtesting.json"
                    automatedTestingControlFile = readJSON file: '.automatedtesting.json'
                    automatedTestingControlFileFound = true
                }
            }
            
            def external_project = ''
            if(env.PROJECT_OVERRIDE) {
                external_project = env.PROJECT_OVERRIDE
            }
            else if(params && params.CMAKE_LY_PROJECTS) {
                external_project = params.CMAKE_LY_PROJECTS
            }
            else {
                echo "No PROJECT_OVERRIDE or CMAKE_LY_PROJECTS defined"
            }
            
            echo "external_project = ${external_project}"
            
            if(automatedTestingControlFileFound) {
                processed_repos = ExecuteControlFile(automatedTestingControlFile, COMMIT_REPOSITORY_NAME, processed_repos, external_project, defer_enable_gems)
            }
            else {
                processed_repos = NoControlFile(COMMIT_REPOSITORY_NAME, processed_repos, external_project, defer_enable_gems)
            }
            
            // For only external projects, set CMAKE_LY_PROJECTS to the relative project_path so it will work on all platforms.
            dir(ENGINE_REPOSITORY_NAME) {
                // Try/catch in the event that this function is called before the scripts are stashed
                try {
                    unstash "${COMMIT_REPOSITORY_NAME}-scripts"
                } catch (e) {
                    print "Unstash failed, ignoring"
                }
                def ext = ''
                if(env.IS_UNIX) {
                    ext = '.sh'
                }
                if (external_project) {
                    PlatformSh("scripts/o3de${ext} get-registered --project-name ${external_project} > projectpath", "Get the project path")
                    def project_path = readFile file: 'projectpath'
                    PlatformRm('projectpath')
                    project_path = project_path.trim()
                    project_path = project_path.replace("\\", "/")
                    workspace_path = "${workspace}"
                    workspace_path = workspace_path.replace("\\", "/")
                    project_path = project_path.replace("${workspace}", "..") // Set the relative path to the project from the engine path
                    project_path = (project_path =~ /\.\.\/[\/a-zA-Z0-9-_]+/)[0] // Ensure only the path is assigned to the variable
                    env.CMAKE_LY_PROJECTS = project_path
                    echo "env.CMAKE_LY_PROJECTS = ${env.CMAKE_LY_PROJECTS}"
                }
            }
        }

        //defer enabling gems until after the repos are done registering all objects
        if(defer_enable_gems) {
            echo 'ENABLE_GEMS'
            defer_enable_gems.each { enableGemCommands ->
                PlatformSh(enableGemCommands.COMMAND, enableGemCommands.ECHO)
            }
        }
    }
}

def Build(Map pipelineConfig, String platform, String type, String workspace) {
    timeout(time: env.TIMEOUT, unit: 'MINUTES', activity: true) {
        def command = "${pipelineConfig.PYTHON_DIR}/python"
        def ext = ''
        if(env.IS_UNIX) {
            command += '.sh'
            ext = '.sh'
        }
        else command += '.cmd'
        command += " -u ${pipelineConfig.BUILD_ENTRY_POINT} --platform ${platform} --type ${type}"

        dir(workspace) {
            dir(ENGINE_REPOSITORY_NAME) {
                PlatformSh(command, "Running ${platform} ${type}")
            }
        }
    }
}

def TestMetrics(Map pipelineConfig, String workspace, String branchName, String repoName, String buildJobName, String outputDirectory, String configuration) {
    catchError(buildResult: null, stageResult: null) {
        def cmakeBuildDir = [workspace, ENGINE_REPOSITORY_NAME, outputDirectory].join('/')
        def command = "${pipelineConfig.PYTHON_DIR}/python"
        if(env.IS_UNIX) command += '.sh'
        else command += '.cmd'
        dir(workspace){
            dir(ENGINE_REPOSITORY_NAME) {
                checkout scm: [
                    $class: 'GitSCM',
                    branches: [[name: '*/main']],
                    extensions: [
                        [$class: 'AuthorInChangelog'],
                        [$class: 'RelativeTargetDirectory', relativeTargetDir: 'mars']
                    ],
                    userRemoteConfigs: [[url: "${env.MARS_REPO}", name: 'mars', credentialsId: "${env.GITHUB_USER}"]]
                ]
                withCredentials([usernamePassword(credentialsId: "${env.SERVICE_USER}", passwordVariable: 'apitoken', usernameVariable: 'username')]) {
                    command += " -u mars/scripts/python/ctest_test_metric_scraper.py " +
                                  '-e jenkins.creds.user %username% -e jenkins.creds.pass %apitoken% ' +
                                  "-e jenkins.base_url ${env.JENKINS_URL} " +
                                  "${cmakeBuildDir} ${branchName} %BUILD_NUMBER% AR ${configuration} ${repoName} --url ${env.BUILD_URL.replace('%','%%')}"
                    bat label: "Publishing ${buildJobName} Test Metrics",
                        script: command
                }
            }
        }
    }
}

def BenchmarkMetrics(Map pipelineConfig, String workspace, String branchName, String outputDirectory) {
    catchError(buildResult: null, stageResult: null) {
        def cmakeBuildDir = [workspace, ENGINE_REPOSITORY_NAME, outputDirectory].join('/')
        def command = "${pipelineConfig.PYTHON_DIR}/python"
        if(env.IS_UNIX) command += '.sh'
        else command += '.cmd'
        dir(workspace) {
            dir(ENGINE_REPOSITORY_NAME) {
                checkout scm: [
                    $class: 'GitSCM',
                    branches: [[name: '*/main']],
                    extensions: [
                        [$class: 'AuthorInChangelog'],
                        [$class: 'RelativeTargetDirectory', relativeTargetDir: 'mars']
                    ],
                    userRemoteConfigs: [[url: "${env.MARS_REPO}", name: 'mars', credentialsId: "${env.GITHUB_USER}"]]
                ]
                command += " -u mars/scripts/python/benchmark_scraper.py ${cmakeBuildDir} ${branchName}"
                PlatformSh(command, "Publishing Benchmark Metrics")
            }
        }
    }
}

def ExportTestResults(Map options, String platform, String type, String workspace, Map params) {
    catchError(message: "Error exporting tests results (this won't fail the build)", buildResult: 'SUCCESS', stageResult: 'FAILURE') {
        dir(workspace) {
            dir(ENGINE_REPOSITORY_NAME) {
                dir(params.OUTPUT_DIRECTORY) {
                    junit testResults: "Testing/**/*.xml"
                }
            }
        }
    }
}

def ExportTestScreenshots(Map options, String branchName, String platformName, String jobName, String workspace, Map params) {
    catchError(message: "Error exporting test screenshots (this won't fail the build)", buildResult: 'SUCCESS', stageResult: 'FAILURE') {
        dir(workspace){
            dir(ENGINE_REPOSITORY_NAME) {
                def screenshotsFolder = "AutomatedTesting/user/PythonTests/Automated/Screenshots"
                def s3Uploader = "${SCRIPTS_PATH}/tools/upload_to_s3.py"
                def command = "${options.PYTHON_DIR}/python.cmd -u ${s3Uploader} --base_dir ${screenshotsFolder} " +
                              '--file_regex \\"(.*zip\$)\\" ' +
                              "--bucket ${env.TEST_SCREENSHOT_BUCKET} " +
                              "--search_subdirectories True --key_prefix ${branchName}_${env.BUILD_NUMBER} " +
                              '--extra_args {\\"ACL\\":\\"bucket-owner-full-control\\"}'
                PlatformSh(command, "Uploading test screenshots for ${jobName}")
            }
        }
    }
}

// All files are included by default.
// --include will only re-include files that have been excluded from an --exclude filter.
//See more details at https://docs.aws.amazon.com/cli/latest/reference/s3/#use-of-exclude-and-include-filters
def ArchiveArtifactsOnS3(String artifactsSource, String s3Prefix="", boolean recursive=false, List<String> includes=[], List<String> excludes=[]) {
    if (!fileExists(s3Prefix)) {
        PlatformMkdir(s3Prefix)
    }
    PlatformSh("echo ${env.BUILD_URL} > ${s3Prefix}/build_url.txt")
    // archiveArtifacts is very slow, so we only archive one file and upload the rest artifacts to the same bucket using S3 CLI.
    archiveArtifacts artifacts: "${s3Prefix}/build_url.txt"
    def command = "aws s3 cp ${artifactsSource} s3://${env.JENKINS_ARTIFACTS_S3_BUCKET}/${env.JENKINS_JOB_NAME}/${env.BUILD_NUMBER}/artifacts/${s3Prefix} --only-show-errors "
    excludes.each{ exclude ->
        command += "--exclude \"${exclude}\" "
    }
    includes.each{ include ->
        command += "--include \"${include}\" "
    }
    if (recursive) command += "--recursive "
    PlatformSh(command, "Archiving artifacts to ${env.JENKINS_JOB_NAME}/${env.BUILD_NUMBER}/artifacts/${s3Prefix}", false)
}

def UploadAPLogs(String platformName, String jobName, String workspace, Map params) {
    catchError(message: "Error archiving AssetProcessor logs (this won't fail the build)", buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
        dir(workspace) {
            dir(ENGINE_REPOSITORY_NAME) {
                projects = params.CMAKE_LY_PROJECTS.split(",")
                projects.each{ project ->
                    ArchiveArtifactsOnS3("${project}/user/log", "ap_logs/${platformName}/${jobName}/${project}", true)
                }
            }
        }
    }
}

def UploadTestArtifacts(String workspace, String outputDirectory) {
    catchError(message: "Error archiving test artifacts (this won't fail the build)", buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
        def cmakeBuildDir = [workspace, ENGINE_REPOSITORY_NAME, outputDirectory].join('/')
        echo "Uploading Test Artifacts: ${cmakeBuildDir}/Testing"
        ArchiveArtifactsOnS3("${cmakeBuildDir}/Testing", "test_artifacts", true)
    }
}
def UploadCrashArtifacts(String platform) {
    catchError(message: "Error archiving crash artifacts (this won't fail the build)", buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
        def crashArtifactDir = GetCrashArtifactDir()
        echo "Uploading Crash Artifacts: ${crashArtifactDir}"
        ArchiveArtifactsOnS3("${crashArtifactDir}", "${platform}_crash_artifacts", true)
    }
}

def UploadBuildArtifacts(String workspace, String platform, String jobName, String s3sisUploadParams = "") {
    catchError(message: "Error uploading build artifacts (this won't fail the build)", buildResult: 'UNSTABLE', stageResult: 'FAILURE') {
        dir("${workspace}/s3sis") {
            PlatformSh("aws s3 cp ${env.S3SIS_SOURCE} . --recursive", "Downloading s3sis", winSlashReplacement=false)
            if (env.IS_UNIX) {
                PlatformSh("sudo python3 setup.py install", "Installing S3SIS")
            } else {
                PlatformSh("python3 setup.py install", "Installing S3SIS")
            }
            PlatformSh("s3siscli configure", "Configuring S3SIS")
        }
        dir("${workspace}/${ENGINE_REPOSITORY_NAME}") {
            try {
                def cmd = "s3siscli upload"
                if (s3sisUploadParams?.trim()) {
                    cmd += " ${s3sisUploadParams}"
                }
                current_cmd = cmd + " --label ${env.JENKINS_JOB_NAME}/${platform}/${jobName}/${env.BUILD_NUMBER}"
                PlatformSh(current_cmd, "Uploading build artifacts", winSlashReplacement=false)
                latest_cmd = cmd + " --label ${env.JENKINS_JOB_NAME}/${platform}/${jobName}/latest"
                PlatformSh(latest_cmd, "Uploading build artifacts", winSlashReplacement=false)
            } catch (Exception e) {
                echo "WARN: Failed to upload build artifacts. \n${e}"
            }
        }
    }
}

def PostBuildCommonSteps(String workspace, Map params, String projectName, String pipelineName, String branchName, String platform, String buildType, boolean mount = true) {
    echo 'Starting post-build common steps...'
    if (params && params.containsKey('OUTPUT_DIRECTORY')){
        dir(workspace) {
            dir(ENGINE_REPOSITORY_NAME) {
                dir(params.OUTPUT_DIRECTORY) {
                    // Clean up Testing directory
                    PlatformRmDir("Testing")
                    // Recreate test runner xml directories that need to be pre generated to prevent race condition on incremental runs
                    PlatformMkdir("Testing/Pytest")
                    PlatformMkdir("Testing/Gtest")
                }
            }
        }
    }
    
    if (mount) {
        def pythonCmd = ''
        if(env.IS_UNIX) pythonCmd = 'sudo -E python3 -u '
        else pythonCmd = 'python3 -u '

        try {
            timeout(5) {
                if (env.DISCARD_VOLUME?.toBoolean()) {
                    PlatformSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action delete --repository_name ${env.REPOSITORY_NAME} --project ${projectName} --pipeline ${pipelineName} --branch ${branchName} --platform ${platform} --build_type ${buildType}", 'Deleting volume', winSlashReplacement=false)
                } else {
                    PlatformSh("${pythonCmd} ${INCREMENTAL_BUILD_SCRIPT_PATH} --action unmount", 'Unmounting volume')
                }
            }
        } catch (Exception e) {
            echo "Unmount script error ${e}"
        }
    }
}

def HandleDriveSnapshots(String repositoryName, String projectName, String pipeline, String branchName, String platform, String buildType) {  
    unstash name: 'ebs_snapshot_script'

    catchError(message: "Error snapshotting volume (this won't fail the build)", buildResult: 'UNSTABLE', stageResult: 'FAILURE') {        
        def pythonCmd = 'python3 -u '
        
        mountName = "Name:${repositoryName}_${projectName}_${pipeline}_${branchName}_${platform}_${buildType}"
        mountName = mountName.replace('/', '_').replace('\\', '_')
        PlatformSh("${pythonCmd} ${EBS_SNAPSHOT_SCRIPT_PATH} --action create --tags ${mountName} --execute", "Starting volume snapshots", true)
        PlatformSh("${pythonCmd} ${EBS_SNAPSHOT_SCRIPT_PATH} --action delete --tags ${mountName} --retention ${env.SNAP_RETENTION} --execute", "Cleaning up old snapshots", true)
    }
}

def CreateSetupStage(Map pipelineConfig, Map params, String snapshot, String repositoryName, String projectName, String pipelineName, String branchName, String platformName, String jobName, Map environmentVars, boolean onlyMountEBSVolume = false) {
    return {
        stage('Setup') {
            if(onlyMountEBSVolume) {
                HandleDriveMount(snapshot, repositoryName, projectName, pipelineName, branchName, platformName, jobName, environmentVars['WORKSPACE'], false)
            } else {
                PreBuildCommonSteps(pipelineConfig, params, snapshot, repositoryName, projectName, pipelineName, branchName, platformName, jobName,  environmentVars['WORKSPACE'], environmentVars['MOUNT_VOLUME'], false)
            }
        }
    }
}

def CreateBuildStage(Map pipelineConfig, String platformName, String jobName, Map environmentVars) {
    return {
        stage("${jobName}") {
            Build(pipelineConfig, platformName, jobName, environmentVars['WORKSPACE'])
        }
    }
}

def CreateTestMetricsStage(Map pipelineConfig, String branchName, Map environmentVars, String buildJobName, String outputDirectory, String configuration) {
    return {
        stage("${buildJobName}_metrics") {
            TestMetrics(pipelineConfig, environmentVars['WORKSPACE'], branchName, env.DEFAULT_REPOSITORY_NAME, buildJobName, outputDirectory, configuration)
            BenchmarkMetrics(pipelineConfig, environmentVars['WORKSPACE'], branchName, outputDirectory)
        }
    }
}

def CreateExportTestResultsStage(Map pipelineConfig, String platformName, String jobName, Map environmentVars, Map params) {
    return {
        stage("${jobName}_results") {
            ExportTestResults(pipelineConfig, platformName, jobName, environmentVars['WORKSPACE'], params)
        }
    }
}

def CreateExportTestScreenshotsStage(Map pipelineConfig, String branchName, String platformName, String jobName, Map environmentVars, Map params) {
    return {
        stage("${jobName}_screenshots") {
            ExportTestScreenshots(pipelineConfig, branchName, platformName, jobName, environmentVars['WORKSPACE'], params)
        }
    }
}

def CreateUploadAPLogsStage(String platformName, String jobName, String workspace, Map params) {
    return {
        stage("${jobName}_upload_ap_logs") {
            UploadAPLogs(platformName, jobName, workspace, params)
        }
    }
}

def CreateUploadTestArtifactStage(String jobName, String workspace, String outputDirectory) {
    return {
        stage("${jobName}_upload_test_artifacts") {
            UploadTestArtifacts(workspace, outputDirectory)
        }
    }
}

def CreateUploadCrashArtifactStage(String jobName, String platform) {
    return {
        stage("${jobName}_upload_crash_artifacts") {
            UploadCrashArtifacts(platform)
        }
    }
}

def CreateUploadBuildArtifactStage(String workspace, String platform, String jobName, String s3sisUploadParams="") {
    if(env.S3SIS_UPLOAD_PARAMS?.trim()) {
        s3sisUploadParams = env.S3SIS_UPLOAD_PARAMS
    }
    return {
        stage("${jobName}_upload_build_artifacts") {
            UploadBuildArtifacts(workspace, platform, jobName, s3sisUploadParams)
        }
    }
}

def CreateTeardownStage(Map environmentVars, Map params, String projectName, String pipelineName, String branchName, String platformName, String buildType) {
    return {
        stage('Teardown') {
            PostBuildCommonSteps(environmentVars['WORKSPACE'], params, projectName, pipelineName, branchName, platformName, buildType, environmentVars['MOUNT_VOLUME'])
        }
    }
}

def CreateSnapshotStage(String repositoryName, String projectName, String pipelineName, String branchName, String platformName, String buildType, String jobName) {
    return{
        stage("${jobName}_snapshot_ebs_volume") {
            HandleDriveSnapshots(repositoryName, projectName, pipelineName, branchName, platformName, buildType)
        }
    }
}

def CreateSingleNode(Map pipelineConfig, def platform, def build_job, Map envVars, String branchName, String pipelineName, String repositoryName, String projectName, boolean onlyMountEBSVolume = false) {
    def nodeLabel = envVars['NODE_LABEL']
    return {
        def currentResult = ''
        def currentException = ''
        retry(PIPELINE_RETRY_ATTEMPTS) {
            node("${nodeLabel}") {
                if(isUnix()) { // Has to happen inside a node
                    envVars['IS_UNIX'] = 1
                }
                withEnv(GetEnvStringList(envVars)) {
                    def build_job_name = build_job.key
                    def params = platform.value.build_types[build_job_name].PARAMETERS
                    try {
                        CreateSetupStage(pipelineConfig, params, snapshot, repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, envVars, onlyMountEBSVolume).call()

                        if(build_job.value.steps) { // This is a pipe with many steps so create all the build stages
                            pipelineEnvVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, build_job.value.PIPELINE_ENV ?: EMPTY_JSON, pipelineName)
                            build_job.value.steps.each { build_step ->
                                build_job_name = build_step
                                params = platform.value.build_types[build_job_name].PARAMETERS
                                // This addition of maps makes it that the right operand will override entries if they overlap with the left operand
                                envVars = pipelineEnvVars + GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, platform.value.build_types[build_step].PIPELINE_ENV ?: EMPTY_JSON, pipelineName)
                                try {
                                    CreateBuildStage(pipelineConfig,  platform.key, build_step, envVars).call()
                                }
                                catch (Exception e) {
                                    if (envVars['NONBLOCKING_STEP']?.toBoolean()) {
                                        unstable(message: "Build step ${build_step} failed but it's a non-blocking step in build job ${build_job.key}")
                                    } else {
                                        throw e
                                    }
                                }
                            }
                        } else {
                            CreateBuildStage(pipelineConfig,  platform.key, build_job.key, envVars).call()
                        }
                    }
                    catch(Exception e) {
                        if (e instanceof org.jenkinsci.plugins.workflow.steps.FlowInterruptedException) {
                            def causes = e.getCauses().toString()
                            if (causes.contains('RemovedNodeCause')) {
                                error "Node disconnected during build: ${e}"  // Error raised to retry stage on a new node
                            }
                        }
                        if (build_job_name.toLowerCase().contains('asset') && env.IS_UPLOAD_AP_LOGS?.toBoolean()) {
                            CreateUploadAPLogsStage(platform.key, build_job_name, envVars['WORKSPACE'], platform.value.build_types[build_job_name].PARAMETERS).call()
                        }
                        // Upload test artifacts only on builds that failed and ran test suites
                        if (env.IS_UPLOAD_TEST_ARTIFACTS?.toBoolean() && params.containsKey('CTEST_OPTIONS')) {
                            CreateUploadTestArtifactStage(build_job_name, envVars['WORKSPACE'], params.OUTPUT_DIRECTORY).call()
                        }
                        // All other errors will be raised outside the retry block
                        currentResult = envVars['ON_FAILURE_MARK'] ?: 'FAILURE'
                        currentException = e.toString()
                    }
                    finally {
                        if (env.MARS_REPO && params && params.containsKey('TEST_METRICS') && params.TEST_METRICS == 'True') {
                            CreateTestMetricsStage(pipelineConfig, branchName, envVars, build_job_name, params.OUTPUT_DIRECTORY, params.CONFIGURATION).call()
                        }
                        if (params && params.containsKey('TEST_RESULTS') && params.TEST_RESULTS == 'True') {
                            CreateExportTestResultsStage(pipelineConfig, platform.key, build_job_name, envVars, params).call()
                        }
                        if (fileExists(GetCrashArtifactDir())) {
                            CreateUploadCrashArtifactStage(build_job_name, platform.key).call()
                        }
                        if (params && params.containsKey('TEST_SCREENSHOTS') && params.TEST_SCREENSHOTS == 'True' && currentResult == 'FAILURE') {
                            CreateExportTestScreenshotsStage(pipelineConfig, branchName, platform.key, build_job_name, envVars, params).call()
                        }
                        if (env.UPLOAD_BUILD_ARTIFACTS?.toBoolean()) {
                            CreateUploadBuildArtifactStage(envVars['WORKSPACE'], platform.key, build_job_name, envVars['S3SIS_UPLOAD_PARAMS']).call()
                        }
                        CreateTeardownStage(envVars, params, projectName, pipelineName, branchName, platform.key, build_job.key).call()
                        if (envVars['CREATE_SNAPSHOT']?.toBoolean()) {
                            CreateSnapshotStage(repositoryName, projectName, pipelineName, branchName, platform.key, build_job.key, build_job_name).call()
                        }
                    }
                }
            }
        }
        // https://github.com/jenkinsci/jenkins/blob/master/core/src/main/java/hudson/model/Result.java
        // {SUCCESS,UNSTABLE,FAILURE,NOT_BUILT,ABORTED}
        if (currentResult == 'FAILURE') {
            currentBuild.result = 'FAILURE'
            error "FAILURE: ${currentException}"
        } else if (currentResult == 'UNSTABLE') {
            currentBuild.result = 'UNSTABLE'
            unstable(message: "UNSTABLE: ${currentException}")
        }
    }
}

// Used in CreateBuildJobs() to preprocess the build_job steps to programmatically create
// Node sections with a set of steps that can run on that node.
class PipeStepJobData {
    String m_nodeLabel = ""
    def m_steps = []

    PipeStepJobData(String label) {
        this.m_nodeLabel = label
    }

    def addStep(def step) {
        this.m_steps.add(step)
    }
}

def CreateBuildJobs(Map pipelineConfig, def platform, def build_job, Map envVars, String branchName, String pipelineName, String repositoryName, String projectName) {
    
    // if this is a pipeline, split jobs based on the NODE_LABEL 
    if(build_job.value.steps) {
        def defaultLabel = envVars['NODE_LABEL']
        def lastNodeLabel = ""
        def jobList = []
        def currentIdx = -1;

        // iterate the steps to build the order of node label + steps sets.
        // Order matters, as it is executed from first to last.
        // example layout.
        // node A
        //  step 1
        //  step 2
        // node B
        //  step 3
        // node C
        //  step 4
        build_job.value.steps.each { build_step ->
            //if node label defined
            if(platform.value.build_types[build_step] && platform.value.build_types[build_step].PIPELINE_ENV &&
               platform.value.build_types[build_step].PIPELINE_ENV['NODE_LABEL']) {
               
               //if the last node label doesn't match the new one, append it.
               if(platform.value.build_types[build_step].PIPELINE_ENV['NODE_LABEL'] != lastNodeLabel) {
                    lastNodeLabel = platform.value.build_types[build_step].PIPELINE_ENV['NODE_LABEL']
                    jobList.add(new PipeStepJobData(lastNodeLabel))
                    currentIdx++
                } 
            }
            //no label define, so it needs to run on the default node label
            else if(lastNodeLabel != defaultLabel) { //if the last node is not the default, append default
                lastNodeLabel = defaultLabel
                jobList.add(new PipeStepJobData(lastNodeLabel))
                currentIdx++
            }
            //add the build_step to the current node
            jobList[currentIdx].addStep(build_step)
        }
        
        return {
            jobList.eachWithIndex{ element, idx ->
                //update the node label + steps to the discovered data
                envVars['NODE_LABEL'] = element.m_nodeLabel
                build_job.value.steps = element.m_steps
                //no any additional nodes just mount the drive, do not handle clean parameters as that will be done by the first node.
                boolean onlyMountEBSVolume = idx != 0;
                //add this node
                CreateSingleNode(pipelineConfig, platform, build_job, envVars, branchName, pipelineName, repositoryName, projectName, onlyMountEBSVolume).call()
            }
        }
    } else {
        return CreateSingleNode(pipelineConfig, platform, build_job, envVars, branchName, pipelineName, repositoryName, projectName)
    }
}

def projectName = ''
def pipelineName = ''
def branchName = ''
def pipelineConfig = {}

// Start Pipeline
try {
    timeout(time: PIPELINE_TIMEOUT, unit: 'MINUTES', activity: true) {
        stage('Setup Pipeline') {
            node('controller') {
                def envVarList = []
                if(isUnix()) {
                    envVarList.add('IS_UNIX=1')
                }
                withEnv(envVarList) {
                    timestamps {
                        repositoryUrl = scm.getUserRemoteConfigs()[0].getUrl()
                        // repositoryName is the full repository name
                        repositoryName = (repositoryUrl =~ /https:\/\/(.+)\/(.+)\.git/)[0][1]
                        env.REPOSITORY_NAME = repositoryName
                        (projectName, pipelineName) = GetRunningPipelineName(env.JOB_NAME) // env.JOB_NAME is the name of the job given by Jenkins

                        env.PIPELINE_NAME = pipelineName
                        if(env.BRANCH_NAME) {
                            branchName = env.BRANCH_NAME
                        } else {
                            branchName = scm.branches[0].name // for non-multibranch pipelines
                            env.BRANCH_NAME = branchName // so scripts that read this environment have it (e.g. incremental_build_util.py)                        
                        }
                        if(env.CHANGE_TARGET) {
                            // PR builds
                            if(BUILD_SNAPSHOTS.contains(env.CHANGE_TARGET)) {
                                snapshot = env.CHANGE_TARGET
                                echo "Snapshot for destination branch \"${env.CHANGE_TARGET}\" found."
                            } else { 
                                snapshot = DEFAULT_BUILD_SNAPSHOT
                                echo "Snapshot for destination branch \"${env.CHANGE_TARGET}\" does not exist, defaulting to snapshot \"${snapshot}\""
                            }
                        } else {
                            // Non-PR builds
                            pipelineParameters.add(choice(name: 'SNAPSHOT', choices: BUILD_SNAPSHOTS_WITH_EMPTY, description: 'Selects the build snapshot to use. A more diverted snapshot will cause longer build times, but will not cause build failures.'))
                            snapshot = env.SNAPSHOT
                            echo "Snapshot \"${snapshot}\" selected."
                        }
                        pipelineProperties.add(disableConcurrentBuilds())

                        echo "Running repository: \"${repositoryName}\", pipeline: \"${pipelineName}\", branch: \"${branchName}\", CHANGE_ID: \"${env.CHANGE_ID}\", GIT_COMMMIT: \"${scm.GIT_COMMIT}\"..."

                        CheckoutEngineBootstrapScripts(branchName, scm.userRemoteConfigs)
                        // Stash any project based build configs
                        stash name: "${COMMIT_REPOSITORY_NAME}-scripts", includes: "scripts/build/**", allowEmpty: true
                        // Project or external repos may not have the bootstrapping scripts. For these cases, pull the bootstrap scripts from the engine repo at the default branch
                        if (!fileExists(SCRIPTS_PATH) || !fileExists(PIPELINE_CONFIG_FILE) || !(COMMIT_REPOSITORY_NAME == ENGINE_REPOSITORY_NAME)) {
                            echo "No bootstrap scripts found in ${SCRIPTS_PATH} or working repo is not the engine repo, downloading it from the engine repo at ${ENGINE_DEVELOPMENT_BRANCH}"
                            CheckoutEngineBootstrapScripts(ENGINE_DEVELOPMENT_BRANCH)
                        }
                        unstash "${COMMIT_REPOSITORY_NAME}-scripts"
                        
                        // Load configs
                        pipelineConfig = LoadPipelineConfig(pipelineName, branchName)

                        // Add each platform as a parameter that the user can disable if needed
                        if (!IsPullRequest(branchName) || IsPeriodicPipeline(pipelineName)) {
                            pipelineParameters.add(stringParam(defaultValue: '', description: 'Filters and overrides the list of jobs to run for each of the below platforms (comma-separated). Can\'t be used during a pull request.', name: 'JOB_LIST_OVERRIDE'))
                            pipelineParameters.add(stringParam(defaultValue: '', description: 'Name of the project to build. Overrides default project', name: 'PROJECT_OVERRIDE'))
                            pipelineParameters.add(booleanParam(defaultValue: false, description: 'Upload build artifacts to S3.', name: 'UPLOAD_BUILD_ARTIFACTS'),)
                            pipelineParameters.add(stringParam(defaultValue: '', description: 'Additional S3SIS upload parameters, for example, use --include or --exclude to specify the files to be uploaded.', name: 'S3SIS_UPLOAD_PARAMS'))
                            pipelineConfig.platforms.each { platform ->
                                pipelineParameters.add(booleanParam(defaultValue: true, description: '', name: platform.key))
                            }
                        }
                        // Add additional Jenkins parameters
                        pipelineConfig.platforms.each { platform ->
                            platformEnv = platform.value.PIPELINE_ENV
                            pipelineJenkinsParameters = platformEnv['PIPELINE_JENKINS_PARAMETERS'] ?: [:]
                            jenkinsParametersToAdd = pipelineJenkinsParameters[pipelineName] ?: [:]
                            jenkinsParametersToAdd.each{ jenkinsParameter ->
                                defaultValue = jenkinsParameter['default_value']
                                // Use last run's value as default value so we can save values in different Jenkins environment
                                if (jenkinsParameter['use_last_run_value']?.toBoolean()) {
                                    defaultValue = params."${jenkinsParameter['parameter_name']}" ?: jenkinsParameter['default_value']
                                }
                                switch (jenkinsParameter['parameter_type']) {
                                    case 'string':
                                        pipelineParameters.add(stringParam(defaultValue: defaultValue,
                                                                    description: jenkinsParameter['description'],
                                                                    name: jenkinsParameter['parameter_name']
                                                                    ))
                                        break
                                    case 'boolean':
                                        pipelineParameters.add(booleanParam(defaultValue: defaultValue,
                                                                    description: jenkinsParameter['description'],
                                                                    name: jenkinsParameter['parameter_name']
                                                                    ))
                                        break
                                    case 'password':
                                        pipelineParameters.add(password(defaultValue: defaultValue,
                                                                    description: jenkinsParameter['description'],
                                                                    name: jenkinsParameter['parameter_name']
                                                                    ))
                                        break
                                }
                            }
                        }

                        pipelineProperties.add(parameters(pipelineParameters.unique()))
                        properties(pipelineProperties)

                        // Stash the INCREMENTAL_BUILD_SCRIPT_PATH and EBS_SNAPSHOT_SCRIPT_PATH since all nodes will use it
                        stash name: 'incremental_build_script',
                            includes: INCREMENTAL_BUILD_SCRIPT_PATH
                        if (fileExists(EBS_SNAPSHOT_SCRIPT_PATH)) {
                            stash name: 'ebs_snapshot_script',
                            includes: EBS_SNAPSHOT_SCRIPT_PATH
                        }
                    }
                }
            }
        }

        if(env.BUILD_NUMBER == '1') {
            // Exit pipeline early on the initial build. This allows Jenkins to load the pipeline for the branch and enables users
            // to select build parameters on their first actual build. See https://issues.jenkins.io/browse/JENKINS-41929
            if (!IsPullRequest(branchName)) {
                currentBuild.result = 'SUCCESS'
                return
            }
            else if (IsPeriodicPipeline(pipelineName)) {
                currentBuild.result = 'NOT_BUILT'
                return
            }
        }

        def someBuildHappened = false

        // Build and Post-Build Testing Stage
        def buildConfigs = [:]

        // Platform Builds run on EC2
        pipelineConfig.platforms.each { platform ->
            platform.value.build_types.each { build_job ->
                if (IsJobEnabled(branchName, build_job, pipelineName, platform.key)) {   // User can filter jobs, jobs are tagged by pipeline
                    def envVars = GetBuildEnvVars(platform.value.PIPELINE_ENV ?: EMPTY_JSON, build_job.value.PIPELINE_ENV ?: EMPTY_JSON, pipelineName)
                    envVars['JENKINS_JOB_NAME'] = env.JOB_NAME // Save original Jenkins job name to JENKINS_JOB_NAME
                    envVars['JOB_NAME'] = "${branchName}_${platform.key}_${build_job.key}" // backwards compatibility, some scripts rely on this
                    someBuildHappened = true
                    buildConfigs["${platform.key} [${build_job.key}]"] = CreateBuildJobs(pipelineConfig, platform, build_job, envVars, branchName, pipelineName, repositoryName, projectName)
                }
            }
        }

        timestamps {
            stage('Build') {
                if (params.FAIL_FAST) {
                    echo "Fail fast option enabled"
                    buildConfigs.failFast = true
                }
                parallel buildConfigs // Run parallel builds
            }

            echo 'All builds successful'
        }
        if (!someBuildHappened) {
            currentBuild.result = 'NOT_BUILT'
        }
        if("${currentBuild.currentResult}" == "SUCCESS" && IsPeriodicPipeline(pipelineName) && IsPullRequest(branchName)) {
            currentBuild.result = 'UNSTABLE'
            currentBuild.description = 'Builds succeeded but regular PR run is required to merge the PR.'
        }
    }
}
catch(Exception e) {
    error "Exception: ${e}"
}
finally {
    try {
        node('controller') {
            if("${currentBuild.currentResult}" == "SUCCESS") {
                buildFailure = ""
                emailBody = "${BUILD_URL}\nSuccess!"
            } else {
                buildFailure = tm('${BUILD_FAILURE_ANALYZER}')
                emailBody = "${BUILD_URL}\n${buildFailure}!"
            }
            if(env.POST_AR_BUILD_SNS_TOPIC) {
                message_json = [
                    "build_url": env.BUILD_URL,
                    "build_number": env.BUILD_NUMBER,
                    "repository_name": env.REPOSITORY_NAME,
                    "branch_name": env.BRANCH_NAME,
                    "pipeline_name": GetRunningPipelineName(env.JOB_NAME)[1],
                    "full_pipeline_name": env.JOB_NAME,
                    "build_result": "${currentBuild.currentResult}",
                    "build_failure": buildFailure,
                    "recreate_volume": env.RECREATE_VOLUME,
                    "clean_output_directory": env.CLEAN_OUTPUT_DIRECTORY,
                    "clean_assets": env.CLEAN_ASSETS,
                    "fail_fast": env.FAIL_FAST,
                    "project_name": env.CMAKE_LY_PROJECTS
                ]
                snsPublish(
                    topicArn: env.POST_AR_BUILD_SNS_TOPIC,
                    subject:'Build Result',
                    message:JsonOutput.toJson(message_json)
                )
            }
            emailext (
                body: "${emailBody}",
                subject: "${currentBuild.currentResult}: ${JOB_NAME} - Build # ${BUILD_NUMBER}",
                recipientProviders: [
                    [$class: 'RequesterRecipientProvider']
                ]
            )
        }
    } catch(Exception e) {
    }
}