diff --git a/src/System.Management.Automation/commands/utility/FormatAndOutput/common/DefaultFormatters/PowerShellCore_format_ps1xml.cs b/src/System.Management.Automation/commands/utility/FormatAndOutput/common/DefaultFormatters/PowerShellCore_format_ps1xml.cs
index 84c2d371a..47dd76fab 100644
--- a/src/System.Management.Automation/commands/utility/FormatAndOutput/common/DefaultFormatters/PowerShellCore_format_ps1xml.cs
+++ b/src/System.Management.Automation/commands/utility/FormatAndOutput/common/DefaultFormatters/PowerShellCore_format_ps1xml.cs
@@ -720,7 +720,7 @@ namespace System.Management.Automation.Runspaces
                 CustomControl.Create(outOfBand: true)
                     .StartEntry()
                         .AddScriptBlockExpressionBinding(@"
-                                    if ($_.FullyQualifiedErrorId -ne ""NativeCommandErrorMessage"" -and $ErrorView -ne ""CategoryView"")
+                                    if (($_.FullyQualifiedErrorId -ne ""NativeCommandErrorMessage"" -and $_.FullyQualifiedErrorId -ne ""NativeCommandError"") -and $ErrorView -ne ""CategoryView"")
                                     {
                                         $myinv = $_.InvocationInfo
                                         if ($myinv -and $myinv.MyCommand)
@@ -767,7 +767,7 @@ namespace System.Management.Automation.Runspaces
                                     }
                                 ")
                         .AddScriptBlockExpressionBinding(@"
-                                   if ($_.FullyQualifiedErrorId -eq ""NativeCommandErrorMessage"") {
+                                   if ($_.FullyQualifiedErrorId -eq ""NativeCommandErrorMessage"" -or $_.FullyQualifiedErrorId -eq ""NativeCommandError"") {
                                         $_.Exception.Message   
                                    }
                                    else
diff --git a/src/System.Management.Automation/engine/MshCommandRuntime.cs b/src/System.Management.Automation/engine/MshCommandRuntime.cs
index 489389319..361b1b55f 100644
--- a/src/System.Management.Automation/engine/MshCommandRuntime.cs
+++ b/src/System.Management.Automation/engine/MshCommandRuntime.cs
@@ -2841,7 +2841,7 @@ namespace System.Management.Automation
         /// but the command failure will ultimately be
         /// <see cref="System.Management.Automation.ActionPreferenceStopException"/>,
         /// </remarks>
-        internal void _WriteErrorSkipAllowCheck(ErrorRecord errorRecord, ActionPreference? actionPreference = null)
+        internal void _WriteErrorSkipAllowCheck(ErrorRecord errorRecord, ActionPreference? actionPreference = null, bool isNativeError = false)
         {
             ThrowIfStopping();
 
@@ -2923,7 +2923,9 @@ namespace System.Management.Automation
             PSObject errorWrap = PSObject.AsPSObject(errorRecord);
             // It's possible we've already added the member (this method is recursive sometimes
             // when tracing), so don't add the member again.
-            if (errorWrap.Members["writeErrorStream"] == null)
+
+            // We don't add a note property on messages that comes from stderr stream.
+            if (!isNativeError && errorWrap.Members["writeErrorStream"] == null)
             {
                 PSNoteProperty note = new PSNoteProperty("writeErrorStream", true);
                 errorWrap.Properties.Add(note);
diff --git a/src/System.Management.Automation/engine/NativeCommandProcessor.cs b/src/System.Management.Automation/engine/NativeCommandProcessor.cs
index 507a5bb7b..3c927cc1e 100644
--- a/src/System.Management.Automation/engine/NativeCommandProcessor.cs
+++ b/src/System.Management.Automation/engine/NativeCommandProcessor.cs
@@ -947,7 +947,7 @@ namespace System.Management.Automation
                     ErrorRecord record = outputValue.Data as ErrorRecord;
                     Dbg.Assert(record != null, "ProcessReader should ensure that data is ErrorRecord");
                     record.SetInvocationInfo(this.Command.MyInvocation);
-                    this.commandRuntime._WriteErrorSkipAllowCheck(record);
+                    this.commandRuntime._WriteErrorSkipAllowCheck(record, isNativeError: true);
                 }
                 else if (outputValue.Stream == MinishellStream.Output)
                 {
@@ -1880,17 +1880,11 @@ namespace System.Management.Automation
                 //
                 // Wrap the rest of the output in ErrorRecords with the "NativeCommandErrorMessage" error ID
                 //
-                char[] buffer = new char[4096];
-
-                int read = 0;
-
-                while ((read = _streamReader.Read(buffer, 0, buffer.Length)) != 0)
+                while ((line = _streamReader.ReadLine()) != null)
                 {
-                    StringBuilder errorMessage = new StringBuilder().Append(buffer, 0, read);
-
                     AddObjectToWriter(
                         new ErrorRecord(
-                            new RemoteException(errorMessage.ToString()),
+                            new RemoteException(line),
                             "NativeCommandErrorMessage",
                             ErrorCategory.NotSpecified,
                             null),
diff --git a/test/powershell/Scripting/NativeExecution/NativeStreams.Tests.ps1 b/test/powershell/Scripting/NativeExecution/NativeStreams.Tests.ps1
new file mode 100644
index 000000000..7c191802e
--- /dev/null
+++ b/test/powershell/Scripting/NativeExecution/NativeStreams.Tests.ps1
@@ -0,0 +1,53 @@
+Describe "Native streams behavior with PowerShell" -Tags 'CI' {
+    $powershell = Join-Path -Path $PsHome -ChildPath "powershell"
+
+    Context "Error stream" {
+        # we are using powershell itself as an example of a native program.
+        # we can create a behavior we want on the fly and test complex scenarios.
+
+        $error.Clear()
+
+        $command = [string]::Join('', @(
+            '[Console]::Error.Write(\"foo`n`nbar`n`nbaz\"); ',
+            '[Console]::Error.Write(\"middle\"); ',
+            '[Console]::Error.Write(\"foo`n`nbar`n`nbaz\")'
+        ))
+
+        $out = & $powershell -noprofile -command $command 2>&1
+
+        # this check should be the first one, because $error is a global shared variable
+        It 'should not add records to $error variable' {
+            # we are keeping existing Windows PS v5.1 behavior for $error variable
+            $error.Count | Should Be 9
+        }
+
+        It 'uses ErrorRecord object to return stderr output' {
+            ($out | measure).Count | Should BeGreaterThan 1
+
+            $out[0] | Should BeOfType 'System.Management.Automation.ErrorRecord'
+            $out[0].FullyQualifiedErrorId | Should Be 'NativeCommandError'
+
+            $out | Select-Object -Skip 1 | % {
+                $_ | Should BeOfType 'System.Management.Automation.ErrorRecord'
+                $_.FullyQualifiedErrorId | Should Be 'NativeCommandErrorMessage'
+            }
+        }
+
+        It 'uses correct exception messages for error stream' {
+            ($out | measure).Count | Should Be 9
+            $out[0].Exception.Message | Should Be 'foo'
+            $out[1].Exception.Message | Should Be ''
+            $out[2].Exception.Message | Should Be 'bar'
+            $out[3].Exception.Message | Should Be ''
+            $out[4].Exception.Message | Should Be 'bazmiddlefoo'
+            $out[5].Exception.Message | Should Be ''
+            $out[6].Exception.Message | Should Be 'bar'
+            $out[7].Exception.Message | Should Be ''
+            $out[8].Exception.Message | Should Be 'baz'
+        }
+
+        It 'preserves error stream as is with Out-String' {
+            ($out | Out-String).Replace("`r", '') | Should Be "foo`n`nbar`n`nbazmiddlefoo`n`nbar`n`nbaz`n"
+        }
+    }
+}