From cab8f2638617eddcc8885ee7ac11b6ba77a89e47 Mon Sep 17 00:00:00 2001
From: Dongbo Wang <dongbow@microsoft.com>
Date: Wed, 23 Jan 2019 11:42:32 -0800
Subject: [PATCH] Generate and deploy reference assembly for
 Microsoft.PowerShell.Commands.Utility.dll (#8716)

Update `New-ReferenceAssembly` and `New-UnifiedNugetPackage` to generate reference assembly for `Microsoft.PowerShell.Commands.Utility.dll` and properly deploy it for `Microsoft.PowerShell.Commands.Utility` NuGet package and `Microsoft.PowerShell.SDK` NuGet package.

An incremental step to fix, eventually, #8121
---
 tools/packaging/packaging.psm1         | 243 +++++++++++++++++--------
 tools/packaging/packaging.strings.psd1 |  44 ++++-
 2 files changed, 207 insertions(+), 80 deletions(-)

diff --git a/tools/packaging/packaging.psm1 b/tools/packaging/packaging.psm1
index e9c4b8467..f2495a3cc 100644
--- a/tools/packaging/packaging.psm1
+++ b/tools/packaging/packaging.psm1
@@ -1553,7 +1553,6 @@ function New-UnifiedNugetPackage
         $SnkFilePath = "$RepoRoot\src\signing\visualstudiopublic.snk"
 
         New-ReferenceAssembly -linux64BinPath $linuxBinPath -RefAssemblyDestinationPath $refBinPath -RefAssemblyVersion $PackageVersion -SnkFilePath $SnkFilePath -GenAPIToolPath $GenAPIToolPath
-        $refBinFullName = Join-Path $refBinPath 'System.Management.Automation.dll'
 
         foreach ($file in $fileList)
         {
@@ -1565,8 +1564,7 @@ function New-UnifiedNugetPackage
 
             #region ref
             $refFolder = New-Item (Join-Path $filePackageFolder.FullName 'ref/netstandard2.0') -ItemType Directory -Force
-            Copy-Item $refBinFullName -Destination $refFolder -Force
-            Write-Log "Copied file $refBinFullName to $refFolder"
+            CopyReferenceAssemblies -assemblyName $fileBaseName -refBinPath $refBinPath -refNugetPath $refFolder -assemblyFileList $fileList
             #endregion ref
 
             $packageRuntimesFolderPath = $packageRuntimesFolder.FullName
@@ -1695,6 +1693,44 @@ function New-UnifiedNugetPackage
     }
 }
 
+<#
+  Copy the generated reference assemblies to the 'ref/netstandard2.0' folder properly.
+  This is a helper function used by 'New-UnifiedNugetPackage'
+#>
+function CopyReferenceAssemblies
+{
+    param(
+        [string] $assemblyName,
+        [string] $refBinPath,
+        [string] $refNugetPath,
+        [string[]] $assemblyFileList
+    )
+
+    switch ($assemblyName) {
+        "Microsoft.PowerShell.Commands.Utility" {
+            $ref_Utility = Join-Path -Path $refBinPath -ChildPath Microsoft.PowerShell.Commands.Utility.dll
+            Copy-Item $ref_Utility -Destination $refNugetPath -Force
+            Write-Log "Copied file $ref_Utility to $refNugetPath"
+        }
+
+        "Microsoft.PowerShell.SDK" {
+            foreach ($asmFileName in $assemblyFileList) {
+                $refFile = Join-Path -Path $refBinPath -ChildPath $asmFileName
+                if (Test-Path -Path $refFile) {
+                    Copy-Item $refFile -Destination $refNugetPath -Force
+                    Write-Log "Copied file $refFile to $refNugetPath"
+                }
+            }
+        }
+
+        default {
+            $ref_SMA = Join-Path -Path $refBinPath -ChildPath System.Management.Automation.dll
+            Copy-Item $ref_SMA -Destination $refNugetPath -Force
+            Write-Log "Copied file $ref_SMA to $refNugetPath"
+        }
+    }
+}
+
 <#
 .SYNOPSIS
 Return the list of packages and versions used by a project
@@ -1840,15 +1876,6 @@ function New-ReferenceAssembly
         throw "New-ReferenceAssembly can be only executed on Windows platform."
     }
 
-    $genAPIFolder = New-TempFolder
-    $smaProjectFolder = New-Item -Path "$genAPIFolder/System.Management.Automation" -ItemType Directory -Force
-    $smaCs = Join-Path $smaProjectFolder 'System.Management.Automation.cs'
-    $smaCsFiltered = Join-Path $smaProjectFolder 'System.Management.Automation_Filtered.cs'
-
-    Write-Log "Working directory: $genAPIFolder."
-
-    #region GenAPI
-
     $genAPIExe = Get-ChildItem -Path "$GenAPIToolPath/*GenAPI.exe" -Recurse
 
     if (-not (Test-Path $genAPIExe))
@@ -1858,23 +1885,78 @@ function New-ReferenceAssembly
 
     Write-Log "GenAPI nuget package saved and expanded."
 
-    $linuxSMAPath = Join-Path $Linux64BinPath "System.Management.Automation.dll"
+    $genAPIFolder = New-TempFolder
+    $packagingStrings.NugetConfigFile | Out-File -FilePath "$genAPIFolder/Nuget.config" -Force
+    Write-Log "Working directory: $genAPIFolder."
 
-    if (-not (Test-Path $linuxSMAPath))
-    {
-        throw "System.Management.Automation.dll was not found at: $Linux64BinPath"
+    $SMAReferenceAssembly = $null
+    $assemblyNames = @(
+        "System.Management.Automation",
+        "Microsoft.PowerShell.Commands.Utility"
+    )
+
+    foreach ($assemblyName in $assemblyNames) {
+
+        $projectFolder = New-Item -Path "$genAPIFolder/$assemblyName" -ItemType Directory -Force
+        $generatedSource = Join-Path $projectFolder "$assemblyName.cs"
+        $filteredSource = Join-Path $projectFolder "${assemblyName}_Filtered.cs"
+
+        $linuxDllPath = Join-Path $Linux64BinPath "$assemblyName.dll"
+        if (-not (Test-Path $linuxDllPath)) {
+            throw "$assemblyName.dll was not found at: $Linux64BinPath"
+        }
+
+        $genAPIArgs = "$linuxDllPath","-libPath:$Linux64BinPath"
+        Write-Log "GenAPI cmd: $genAPIExe $genAPIArgsString"
+
+        Start-NativeExecution { & $genAPIExe $genAPIArgs } | Out-File $generatedSource -Force
+        Write-Log "Reference assembly file generated at: $generatedSource"
+
+        CleanupGeneratedSourceCode -assemblyName $assemblyName -generatedSource $generatedSource -filteredSource $filteredSource
+
+        try
+        {
+            Push-Location $projectFolder
+
+            $csProj = GenerateCSProjectContent -AssemblyName $assemblyName -RefAssemblyVersion $RefAssemblyVersion -SnkFilePath $SnkFilePath -SMAReferencePath $SMAReferenceAssembly
+            $csProj | Out-File -FilePath "$projectFolder/$assemblyName.csproj" -Force
+
+            Write-Host "##vso[artifact.upload containerfolder=artifact;artifactname=artifact]$projectFolder/$assemblyName.csproj"
+            Write-Host "##vso[artifact.upload containerfolder=artifact;artifactname=artifact]$generatedSource"
+
+            Start-NativeExecution { dotnet build -c Release }
+
+            $refBinPath = Join-Path $projectFolder "bin/Release/netstandard2.0/$assemblyName.dll"
+            if ($null -eq $refBinPath) {
+                throw "Reference assembly was not built."
+            }
+
+            Copy-Item $refBinPath $RefAssemblyDestinationPath -Force
+            Write-Log "Reference assembly '$assemblyName.dll' built and copied to $RefAssemblyDestinationPath"
+
+            if ($assemblyName -eq "System.Management.Automation") {
+                $SMAReferenceAssembly = $refBinPath
+            }
+        }
+        finally
+        {
+            Pop-Location
+        }
     }
 
-    $genAPIArgs = "$linuxSMAPath","-libPath:$Linux64BinPath"
-    Write-Log "GenAPI cmd: $genAPIExe $genAPIArgsString"
+    if (Test-Path $genAPIFolder)
+    {
+        Remove-Item $genAPIFolder -Recurse -Force -ErrorAction SilentlyContinue
+    }
+}
 
-    Start-NativeExecution { & $genAPIExe $genAPIArgs } | Out-File $smaCs -Force
-
-    Write-Log "Reference assembly file generated at: $smaCs"
-
-    #endregion GenAPI
-
-    #region Cleanup SMA.cs
+function CleanupGeneratedSourceCode
+{
+    param(
+        [string] $assemblyName,
+        [string] $generatedSource,
+        [string] $filteredSource
+    )
 
     $patternsToRemove = @(
         '[System.Management.Automation.ArgumentToEncodingTransformationAttribute]'
@@ -1888,78 +1970,93 @@ function New-ReferenceAssembly
         'typeof(System.Management.Automation.LanguagePrimitives.EnumMultipleTypeConverter)'
         '[System.Management.Automation.Internal.CommonParameters.ValidateVariableName]'
         '[System.Management.Automation.ArgumentEncodingCompletionsAttribute]'
+        '[Microsoft.PowerShell.Commands.AddMemberCommand'
+        '[System.Management.Automation.ArgumentCompleterAttribute(typeof(Microsoft.PowerShell.Commands.Utility.JoinItemCompleter))]'
+        '[System.Management.Automation.ArgumentCompleterAttribute(typeof(System.Management.Automation.PropertyNameCompleter))]'
+        '[System.Management.Automation.OutputTypeAttribute(typeof(Microsoft.PowerShell.MarkdownRender'
+        '[Microsoft.PowerShell.Commands.ArgumentToTypeNameTransformationAttribute]'
+        '[System.Management.Automation.Internal.ArchitectureSensitiveAttribute]'
+        '[Microsoft.PowerShell.Commands.SelectStringCommand.FileinfoToStringAttribute]'
+        '[System.Runtime.CompilerServices.IsReadOnlyAttribute]'
         )
 
-    $reader = [System.IO.File]::OpenText($smaCs)
-    $writer = [System.IO.File]::CreateText($smaCsFiltered)
+    $patternsToReplace = @(
+        @{
+            ApplyTo = "Microsoft.PowerShell.Commands.Utility"
+            Pattern = "[System.Runtime.CompilerServices.IsReadOnlyAttribute]ref Microsoft.PowerShell.Commands.JsonObject.ConvertToJsonContext"
+            Replacement = "in Microsoft.PowerShell.Commands.JsonObject.ConvertToJsonContext"
+        },
+        @{
+            ApplyTo = "Microsoft.PowerShell.Commands.Utility"
+            Pattern = "public partial struct ConvertToJsonContext"
+            Replacement = "public readonly struct ConvertToJsonContext"
+        }
+    )
+
+    $reader = [System.IO.File]::OpenText($generatedSource)
+    $writer = [System.IO.File]::CreateText($filteredSource)
 
     while($null -ne ($line = $reader.ReadLine()))
     {
-        $match = $line | Select-String -Pattern $patternsToRemove -SimpleMatch
+        $lineWasProcessed = $false
+        foreach ($patternToReplace in $patternsToReplace)
+        {
+            if ($assemblyName -eq $patternToReplace.ApplyTo -and $line.Contains($patternToReplace.Pattern)) {
+                $line = $line.Replace($patternToReplace.Pattern, $patternToReplace.Replacement)
+                $lineWasProcessed = $true
+                break
+            }
+        }
 
-        if ($null -ne $match)
-        {
-            $writer.WriteLine("//$line")
-        }
-        else
-        {
-            $writer.WriteLine($line)
+        if (!$lineWasProcessed) {
+            $match = Select-String -InputObject $line -Pattern $patternsToRemove -SimpleMatch
+            if ($null -ne $match)
+            {
+                $line = "//$line"
+            }
         }
+
+        $writer.WriteLine($line)
     }
+
     if ($null -ne $reader)
     {
         $reader.Close()
     }
+
     if ($null -ne $writer)
     {
         $writer.Close()
     }
 
-    Move-Item $smaCsFiltered $smaCs -Force
+    Move-Item $filteredSource $generatedSource -Force
+    Write-Log "Code cleanup complete for reference assembly '$assemblyName'."
+}
 
-    Write-Log "Reference assembly code cleanup complete."
+function GenerateCSProjectContent
+{
+    param(
+        [string] $AssemblyName,
+        [string] $RefAssemblyVersion,
+        [string] $SnkFilePath,
+        [string] $SMAReferencePath
+    )
 
-    #endregion Cleanup SMA.cs
-
-    #region Build SMA ref assembly
-
-    try
-    {
-        Push-Location $smaProjectFolder
-
-        $csProj = $packagingStrings.RefAssemblyCsProj -f $RefAssemblyVersion,$SnkFilePath
-
-        $csProj | Out-File -FilePath "$smaProjectFolder/System.Management.Automation.csproj" -Force
-
-        Write-Host "##vso[artifact.upload containerfolder=artifact;artifactname=artifact]$smaProjectFolder/System.Management.Automation.csproj"
-        Write-Host "##vso[artifact.upload containerfolder=artifact;artifactname=artifact]$smaCs"
-
-        $packagingStrings.NugetConfigFile | Out-File -FilePath "$genAPIFolder/Nuget.config" -Force
-
-        Start-NativeExecution { dotnet build -c Release }
-
-        $refBinPath = Join-Path $smaProjectFolder 'bin/Release/netstandard2.0/System.Management.Automation.dll'
-
-        if ($null -eq $refBinPath)
-        {
-            throw "Reference assembly was not built."
+    switch ($assemblyName) {
+        "System.Management.Automation" {
+            $csProj = $packagingStrings.'System.Management.Automation' -f $RefAssemblyVersion, $SnkFilePath
         }
 
-        Copy-Item $refBinPath $RefAssemblyDestinationPath -Force
+        "Microsoft.PowerShell.Commands.Utility" {
+            $csProj = $packagingStrings.'Microsoft.PowerShell.Commands.Utility' -f $RefAssemblyVersion, $SnkFilePath, $SMAReferencePath
+        }
 
-        Write-Log "Reference assembly built and copied to $RefAssemblyDestinationPath"
-    }
-    finally
-    {
-        Pop-Location
+        default {
+            throw "It's not yet supported to generate reference assembly for '$assemblyName'"
+        }
     }
 
-    if (Test-Path $genAPIFolder)
-    {
-        Remove-Item $genAPIFolder -Recurse -Force -ErrorAction SilentlyContinue
-    }
-
-    #endregion Build SMA ref assembly
+    return $csProj
 }
 
 <#
diff --git a/tools/packaging/packaging.strings.psd1 b/tools/packaging/packaging.strings.psd1
index 1e862b588..0b9209ec4 100644
--- a/tools/packaging/packaging.strings.psd1
+++ b/tools/packaging/packaging.strings.psd1
@@ -39,6 +39,7 @@ case "$1" in
     ;;
 esac
 '@
+
     UbuntuAfterRemoveScript = @'
 #!/bin/sh
 set -e
@@ -48,11 +49,13 @@ case "$1" in
         ;;
 esac
 '@
+
     MacOSLauncherScript = @'
 #!/usr/bin/env bash
 open {0}
 '@
-MacOSLauncherPlistTemplate = @'
+
+    MacOSLauncherPlistTemplate = @'
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
@@ -82,8 +85,9 @@ MacOSLauncherPlistTemplate = @'
 </dict>
 </plist>
 '@
-# see https://developer.apple.com/library/content/documentation/DeveloperTools/Reference/DistributionDefinitionRef/Chapters/Distribution_XML_Ref.html
-OsxDistributionTemplate = @'
+
+    # see https://developer.apple.com/library/content/documentation/DeveloperTools/Reference/DistributionDefinitionRef/Chapters/Distribution_XML_Ref.html
+    OsxDistributionTemplate = @'
 <?xml version="1.0" encoding="utf-8" standalone="yes"?>
 <installer-gui-script minSpecVersion="1">
     <title>{0}</title>
@@ -107,7 +111,8 @@ OsxDistributionTemplate = @'
     <pkg-ref id="{4}" version="{1}" onConclusion="none">{2}</pkg-ref>
 </installer-gui-script>
 '@
-NuspecTemplate = @'
+
+    NuspecTemplate = @'
 <?xml version="1.0" encoding="utf-8"?>
 <package xmlns="http://schemas.microsoft.com/packaging/2011/10/nuspec.xsd">
     <metadata>
@@ -132,7 +137,8 @@ NuspecTemplate = @'
     </metadata>
 </package>
 '@
-RefAssemblyCsProj = @'
+
+    'System.Management.Automation' = @'
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <TargetFramework>netstandard2.0</TargetFramework>
@@ -140,15 +146,39 @@ RefAssemblyCsProj = @'
     <DelaySign>true</DelaySign>
     <AssemblyOriginatorKeyFile>{1}</AssemblyOriginatorKeyFile>
     <SignAssembly>true</SignAssembly>
+    <LangVersion>Latest</LangVersion>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Management.Infrastructure" Version="1.0.0-alpha08" />
+    <PackageReference Include="Microsoft.Management.Infrastructure" Version="1.0.0" />
     <PackageReference Include="System.Security.AccessControl" Version="4.4.1" />
     <PackageReference Include="System.Security.Principal.Windows" Version="4.4.1" />
   </ItemGroup>
 </Project>
 '@
-NuGetConfigFile = @'
+
+    'Microsoft.PowerShell.Commands.Utility' = @'
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>netstandard2.0</TargetFramework>
+    <Version>{0}</Version>
+    <DelaySign>true</DelaySign>
+    <AssemblyOriginatorKeyFile>{1}</AssemblyOriginatorKeyFile>
+    <SignAssembly>true</SignAssembly>
+    <LangVersion>Latest</LangVersion>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System.Management.Automation">
+    <HintPath>{2}</HintPath>
+    </Reference>
+  </ItemGroup>
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.1" />
+    <PackageReference Include="System.Security.AccessControl" Version="4.5.0" />
+  </ItemGroup>
+</Project>
+'@
+
+    NuGetConfigFile = @'
 <configuration>
   <packageSources>
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />