NuGet.Client/scripts/perftests/RunPerformanceTests.ps1

<#
.SYNOPSIS
Run a set of performance tests with a given client and a solution.

Both packages.config and PackageReference styles are supported, but mixed projects are not handled very well.

The scenarios are sequential as follows:

1. Clean restore - no http cache & other local caches, no files in the global package folder, absolutely everything gets downloaded and extracted.
1. Cold restore - There is only an http cache. This tells us more about the installation/extraction time. Potentially we might see some extra http calls depending on the project graph.
1. Force restore - The http cache & global packages folder are full. This usually means that there are no package downloads or installations happening. Since most tests are running 
1. NoOp restore

.PARAMETER nugetClientFilePath
The NuGet Client file path. Supported are dotnet.exe and NuGet.exe

.PARAMETER solutionFilePath
The solution file path. The entry point on which restore is called. It could be a project file as well.

.PARAMETER resultsFilePath
The results file path. This is an exact path to a file.

.PARAMETER logsFolderPath
The logs folder path. The default is a temp directory that gets cleaned up after the script has completed.

.PARAMETER nugetFoldersPath
The temp folder for all the nuget assets. This includes the location of the global packages folder, http, plugins cache & temp location 

.PARAMETER iterationCount
How many times to run each test. The default is 3

.PARAMETER isPackagesConfig
Specifies whether the solution is packages-config.

.PARAMETER skipWarmup
When running the tests, a warmup run is performed. Use this parameter to skip it.

.PARAMETER skipCleanRestores
Skips clean restores. 

.PARAMETER skipColdRestores
Skips cold restores

.PARAMETER skipOfflineRestores
Skips offline restores

.PARAMETER skipForceRestores
Skips force restores

.PARAMETER skipNoOpRestores
Skips no-op restore.

.PARAMETER staticGraphRestore
Uses static graph restore if applicable for the client.

.PARAMETER useLocallyBuiltNuGet
Whether to use locally built NuGet. Only works with msbuild.exe. It simply bootstraps the release configuration pre-built version of NuGet

.PARAMETER forceLegacyResolverFallback

.EXAMPLE
.\RunPerformanceTests.ps1 -nugetClientFilePath "C:\Program Files\dotnet\dotnet.exe" -solutionFilePath F:\NuGet.Client\NuGet.sln -resultsFilePath results.csv
#>
Param(
    [Parameter(Mandatory = $True)]
    [string] $nugetClientFilePath,
    [Parameter(Mandatory = $True)]
    [string] $solutionFilePath,
    [Parameter(Mandatory = $True)]
    [string] $resultsFilePath,
    [string] $logsFolderPath,
    [string] $nugetFoldersPath,
    [int] $iterationCount = 3,
    [switch] $isPackagesConfig,
    [switch] $skipWarmup,
    [switch] $skipCleanRestores,
    [switch] $skipColdRestores,
    [switch] $skipOfflineRestores,
    [switch] $skipForceRestores,
    [switch] $skipNoOpRestores,
    [switch] $staticGraphRestore,
    [switch] $useLocallyBuiltNuGet,
    [switch] $forceLegacyResolverFallback
)

. "$PSScriptRoot\PerformanceTestUtilities.ps1"

Function CreateNugetClientArguments(
    [string] $solutionFilePath,
    [string] $nugetClientFilePath,
    [string] $resultsFilePath,
    [string] $logsFolderPath,
    [string] $solutionName,
    [string] $testRunId,
    [string] $scenarioName,
    [string[]] $enabledSwitches)
{
    $arguments = @{
        solutionFilePath = $solutionFilePath
        nugetClientFilePath = $nugetClientFilePath
        resultsFilePath = $resultsFilePath
        logsFolderPath = $logsFolderPath
        scenarioName = $scenarioName
        solutionName = $solutionName
        testRunId = $testRunId
    }

    If ($enabledSwitches -ne $Null)
    {
        ForEach ($enabledSwitch In $enabledSwitches.GetEnumerator())
        {
            $arguments[$enabledSwitch] = $True
        }
    }

    Return $arguments
}

Try
{
    ##### Script logic #####

    If (!(Test-Path $solutionFilePath))
    {
        Log "$solutionFilePath does not exist!" "Red"
        Exit 1
    }

    If (!(Test-Path $nugetClientFilePath))
    {
        Log "$nugetClientFilePath does not exist!" "Red"
        Exit 1
    }

    $nugetClientFilePath = GetAbsolutePath $nugetClientFilePath
    $solutionFilePath = GetAbsolutePath $solutionFilePath
    $resultsFilePath = GetAbsolutePath $resultsFilePath
    $isClientDotnetExe = IsClientDotnetExe $nugetClientFilePath

    
    If ($isPackagesConfig)
    {
        If ($isClientDotnetExe)
        {
            Log "dotnet.exe does not support packages.config restore." "Red"

            Exit 1
        }

        Log "Restores are expected to be packages.config restores."

        If (!$skipForceRestores)
        {
            Log "Force restore is not supported with packages.config.  Skipping force restores." "Yellow"
            $skipForceRestores = $True
        }
    }

    If (![string]::IsNullOrEmpty($logsFolderPath))
    {
        $logsFolderPath = GetAbsolutePath $logsFolderPath

        If ([System.IO.Path]::GetDirectoryName($resultsFilePath).StartsWith($logsFolderPath))
        {
            Log "$resultsFilePath cannot be under $logsFolderPath" "red"
            Exit 1
        }
    }

    If ([string]::IsNullOrEmpty($nugetFoldersPath))
    {
        $default = GetDefaultNuGetTestFolder
        $nugetFoldersPath = GetNuGetFoldersPath $default
    }

    LogDotNetSdkInfo

    if(Test-Path $resultsFilePath)
    {
        Log "The results file $resultsFilePath already exists. The test results of this run will be appended to the same file." "yellow"
    }

    # Setup the NuGet folders - This includes global packages folder/http/plugin caches
    SetupNuGetFolders $nugetClientFilePath $nugetFoldersPath

    $processorInfo = GetProcessorInfo

    Log "Measuring restore for $solutionFilePath by $nugetClientFilePath" "Green"

    $solutionName = [System.IO.Path]::GetFileNameWithoutExtension($solutionFilePath)
    $testRunId = [System.DateTime]::UtcNow.ToString("O")

    If (!$skipWarmup)
    {
        Log "Running 1x warmup restore"
        $enabledSwitches = @("cleanGlobalPackagesFolder", "cleanHttpCache", "cleanPluginsCache", "killMSBuildAndDotnetExeProcess", "cleanRepository")
        If ($isPackagesConfig)
        {
            $enabledSwitches += "isPackagesConfig"
        }
        If ($staticGraphRestore)
        {
            $enabledSwitches += "staticGraphRestore"
        }
        If ($useLocallyBuiltNuGet)
        {
            $enabledSwitches += "useLocallyBuiltNuGet"
        }
        If ($forceLegacyResolverFallback)
        {
            $enabledSwitches += "forceLegacyResolverFallback"
        }
        $arguments = CreateNugetClientArguments $solutionFilePath $nugetClientFilePath $resultsFilePath $logsFolderPath $solutionName $testRunId "warmup" -enabledSwitches $enabledSwitches
        RunRestore @arguments
    }

    If (!$skipCleanRestores)
    {
        Log "Running $($iterationCount)x clean restores"
        $enabledSwitches = @("cleanGlobalPackagesFolder", "cleanHttpCache", "cleanPluginsCache", "killMSBuildAndDotnetExeProcess", "cleanRepository")
        If ($isPackagesConfig)
        {
            $enabledSwitches += "isPackagesConfig"
        }
        If ($staticGraphRestore)
        {
            $enabledSwitches += "staticGraphRestore"
        }
        If ($useLocallyBuiltNuGet)
        {
            $enabledSwitches += "useLocallyBuiltNuGet"
        }
        If ($forceLegacyResolverFallback)
        {
            $enabledSwitches += "forceLegacyResolverFallback"
        }
        $arguments = CreateNugetClientArguments $solutionFilePath $nugetClientFilePath $resultsFilePath $logsFolderPath $solutionName $testRunId "arctic" -enabledSwitches $enabledSwitches
        1..$iterationCount | % { RunRestore @arguments }
    }

    If (!$skipColdRestores)
    {
        Log "Running $($iterationCount)x without a global packages folder"
        $enabledSwitches = @("cleanGlobalPackagesFolder", "killMSBuildAndDotnetExeProcess", "cleanRepository")
        If ($isPackagesConfig)
        {
            $enabledSwitches += "isPackagesConfig"
        }
        If ($staticGraphRestore)
        {
            $enabledSwitches += "staticGraphRestore"
        }
        If ($useLocallyBuiltNuGet)
        {
            $enabledSwitches += "useLocallyBuiltNuGet"
        }
        If ($forceLegacyResolverFallback)
        {
            $enabledSwitches += "forceLegacyResolverFallback"
        }
        $arguments = CreateNugetClientArguments $solutionFilePath $nugetClientFilePath $resultsFilePath $logsFolderPath $solutionName $testRunId "cold" -enabledSwitches $enabledSwitches
        1..$iterationCount | % { RunRestore @arguments }
    }

    If (!$skipOfflineRestores)
    {
        Log "Running $($iterationCount)x offline restores without http cache"
        $enabledSwitches = @("cleanHttpCache", "cleanPluginsCache", "killMSBuildAndDotnetExeProcess", "cleanRepository")
        If ($isPackagesConfig)
        {
            $enabledSwitches += "isPackagesConfig"
        }
        If ($staticGraphRestore)
        {
            $enabledSwitches += "staticGraphRestore"
        }
        If ($useLocallyBuiltNuGet)
        {
            $enabledSwitches += "useLocallyBuiltNuGet"
        }
        If ($forceLegacyResolverFallback)
        {
            $enabledSwitches += "forceLegacyResolverFallback"
        }
        $arguments = CreateNugetClientArguments $solutionFilePath $nugetClientFilePath $resultsFilePath $logsFolderPath $solutionName $testRunId "offline" -enabledSwitches $enabledSwitches
        1..$iterationCount | % { RunRestore @arguments }
    }

    If (!$skipForceRestores)
    {
        Log "Running $($iterationCount)x force restores"
        $enabledSwitches = @("force")
        If ($staticGraphRestore)
        {
            $enabledSwitches += "staticGraphRestore"
        }
        If ($useLocallyBuiltNuGet)
        {
            $enabledSwitches += "useLocallyBuiltNuGet"
        }
        If ($forceLegacyResolverFallback)
        {
            $enabledSwitches += "forceLegacyResolverFallback"
        }
        $arguments = CreateNugetClientArguments $solutionFilePath $nugetClientFilePath $resultsFilePath $logsFolderPath $solutionName $testRunId "force" -enabledSwitches $enabledSwitches
        1..$iterationCount | % { RunRestore @arguments }
    }

    If (!$skipNoOpRestores)
    {
        Log "Running $($iterationCount)x no-op restores"

        [string[]]$enabledSwitches = @()
        If ($staticGraphRestore)
        {
            $enabledSwitches += "staticGraphRestore"
        }
        If ($useLocallyBuiltNuGet)
        {
            $enabledSwitches += "useLocallyBuiltNuGet"
        }
        If ($forceLegacyResolverFallback)
        {
            $enabledSwitches += "forceLegacyResolverFallback"
        }

        $arguments = CreateNugetClientArguments $solutionFilePath $nugetClientFilePath $resultsFilePath $logsFolderPath $solutionName $testRunId "noop" -enabledSwitches $enabledSwitches
        
        1..$iterationCount | % { RunRestore @arguments }
    }

    Log "Completed the performance measurements for $solutionFilePath.  Results are in $resultsFilePath." "green"
}
Finally
{
    CleanNuGetFolders $nugetClientFilePath $nugetFoldersPath
}