HAX: Make it crash less - bridge WPF DispatcherQueue into C++!

src/cascadia/TerminalControl/FlatC.cpp

@@ -11,6 +11,7 @@
 #include "ControlCore.h"
 #include "ControlInteractivity.h"
 
+#include <windows.system.h>
 #include <windowsx.h>
 
 #pragma warning(disable : 4100)
@@ -394,7 +395,7 @@ struct HwndTerminal
 
     wil::unique_hwnd _hwnd;
 
-    HwndTerminal(HWND parentHwnd)
+    HwndTerminal(HWND parentHwnd, winrt::Windows::System::DispatcherQueue dispatcher)
     {
         HINSTANCE hInstance = wil::GetModuleInstanceHandle();
 
@@ -417,7 +418,7 @@ struct HwndTerminal
 
         _settingsBridge = winrt::make_self<CsBridgeTerminalSettings>();
         _connection = winrt::make_self<CsBridgeConnection>();
-        _interactivity = winrt::make_self<implementation::ControlInteractivity>(*_settingsBridge, nullptr, *_connection);
+        _interactivity = winrt::make_self<implementation::ControlInteractivity>(*_settingsBridge, nullptr, *_connection, dispatcher);
         _core = _interactivity->Core();
 
         _core->ScrollPositionChanged({ this, &HwndTerminal::_scrollPositionChanged });
@@ -580,9 +581,87 @@ extern "C" void _stdcall AvoidBuggyTSFConsoleFlags()
     Microsoft::Console::TSF::Handle::AvoidBuggyTSFConsoleFlags();
 }
 
-__declspec(dllexport) HRESULT _stdcall CreateTerminal(HWND parentHwnd, _Out_ void** hwnd, _Out_ PTERM* terminal)
+#pragma region Implementation of IDispatcherQueue
+struct CsDispatcherQueue : public winrt::implements<CsDispatcherQueue, ::ABI::Windows::System::IDispatcherQueue, ::ABI::Windows::System::IDispatcherQueue2>
 {
-    auto inner = new HwndTerminal{ parentHwnd };
+    CsDispatcherQueue(TerminalDispatcherCallouts callouts) : _callouts{ std::move(callouts) } {}
+
+    IFACEMETHODIMP CreateTimer(::ABI::Windows::System::IDispatcherQueueTimer** result)
+    {
+        return E_FAIL;
+    }
+
+    IFACEMETHODIMP TryEnqueue(::ABI::Windows::System::IDispatcherQueueHandler* callback, boolean* result)
+    {
+        return TryEnqueueWithPriority(::ABI::Windows::System::DispatcherQueuePriority::DispatcherQueuePriority_Normal, callback, result);
+    }
+
+    IFACEMETHODIMP TryEnqueueWithPriority(
+        ::ABI::Windows::System::DispatcherQueuePriority priority,
+        ::ABI::Windows::System::IDispatcherQueueHandler* callback,
+        boolean* result)
+    {
+        RETURN_HR_IF(E_INVALIDARG, !result || !callback);
+        callback->AddRef(); // Retain on transition to managed
+        *result = _callouts.pDispatcherTryEnqueueWork(_callouts.context, static_cast<int>(priority), callback);
+        return S_OK;
+    }
+
+    IFACEMETHODIMP add_ShutdownStarting(
+        __FITypedEventHandler_2_Windows__CSystem__CDispatcherQueue_Windows__CSystem__CDispatcherQueueShutdownStartingEventArgs*,
+        EventRegistrationToken*)
+    {
+        return E_FAIL;
+    }
+
+    IFACEMETHODIMP remove_ShutdownStarting(
+        EventRegistrationToken)
+    {
+        return E_FAIL;
+    }
+
+    IFACEMETHODIMP add_ShutdownCompleted(
+        __FITypedEventHandler_2_Windows__CSystem__CDispatcherQueue_IInspectable*,
+        EventRegistrationToken*)
+    {
+        return E_FAIL;
+    }
+
+    IFACEMETHODIMP remove_ShutdownCompleted(
+        EventRegistrationToken)
+    {
+        return E_FAIL;
+    }
+
+    IFACEMETHODIMP get_HasThreadAccess(boolean* result)
+    {
+        RETURN_HR_IF(E_INVALIDARG, !result);
+        *result = _callouts.pDispatcherHasThreadAccess(_callouts.context);
+        return S_OK;
+    }
+
+private:
+    TerminalDispatcherCallouts _callouts;
+};
+
+__declspec(dllexport) HRESULT _stdcall InteropQueueHandlerInvoke(void* ptr)
+{
+    auto handler = static_cast<::ABI::Windows::System::IDispatcherQueueHandler*>(ptr);
+    const auto hr = handler->Invoke();
+    handler->Release();
+    return hr;
+}
+#pragma endregion
+
+__declspec(dllexport) HRESULT _stdcall CreateTerminal(HWND parentHwnd, _In_ TerminalDispatcherCallouts dispatcherCallouts, _Out_ void** hwnd, _Out_ PTERM* terminal)
+{
+    winrt::Windows::System::DispatcherQueue localDispatcherQueue{ nullptr };
+    if (dispatcherCallouts.pDispatcherTryEnqueueWork != nullptr)
+    {
+        auto dispatcher = winrt::make<CsDispatcherQueue>(dispatcherCallouts);
+        winrt::copy_from_abi(localDispatcherQueue, winrt::get_abi(dispatcher));
+    }
+    auto inner = new HwndTerminal{ parentHwnd, localDispatcherQueue };
     *terminal = inner;
     *hwnd = inner->_hwnd.get();
     inner->Initialize();

src/cascadia/TerminalControl/FlatC.h

@@ -17,6 +17,14 @@ using PTERM = void*;
 using PSCROLLCB = void(_stdcall*)(int, int, int);
 using PWRITECB = void(_stdcall*)(const wchar_t*);
 
+typedef struct _TerminalDispatcherCallouts
+{
+    bool(__stdcall* pDispatcherTryEnqueueWork)(void*, int, void*);
+    bool(__stdcall* pDispatcherHasThreadAccess)(void*);
+    // Context which will be passed back to the above function pointers.
+    void* context;
+} TerminalDispatcherCallouts;
+
 #define TERMINAL_API_TABLE(XX)                                                                                 \
     XX(SendOutput, LPCWSTR, data)                                                                              \
     XX(RegisterScrollCallback, PSCROLLCB, callback)                                                            \
@@ -58,6 +66,6 @@ TERMINAL_API_TABLE(GENERATOR)
 #undef GENERATOR_N
 #undef API_NAME
 
-__declspec(dllexport) HRESULT _stdcall CreateTerminal(HWND parentHwnd, _Out_ void** hwnd, _Out_ PTERM* terminal);
+__declspec(dllexport) HRESULT _stdcall CreateTerminal(HWND parentHwnd, TerminalDispatcherCallouts dispatcherCallouts, _Out_ void** hwnd, _Out_ PTERM* terminal);
 __declspec(dllexport) void _stdcall DestroyTerminal(PTERM terminal);
 };

src/cascadia/TerminalControl/dll/Microsoft.Terminal.Control.def

@@ -20,3 +20,5 @@ EXPORTS
   TerminalRegisterWriteCallback
   TerminalSendKeyEvent
   TerminalSendCharEvent
+
+  InteropQueueHandlerInvoke

src/cascadia/WpfTerminalControl/NativeMethods.cs

@@ -19,6 +19,22 @@ namespace Microsoft.Terminal.Wpf
         [UnmanagedFunctionPointer(CallingConvention.StdCall)]
         public delegate void WriteCallback([In, MarshalAs(UnmanagedType.LPWStr)] string data);
 
+        public struct DispatcherCallouts
+        {
+            [MarshalAs(UnmanagedType.FunctionPtr)]
+            public DispatcherTryEnqueue tryEnqueue;
+
+            [MarshalAs(UnmanagedType.FunctionPtr)]
+            public DispatcherHasThreadAccess hasThreadAccess;
+            public IntPtr context;
+
+            [UnmanagedFunctionPointer(CallingConvention.StdCall)]
+            public delegate bool DispatcherTryEnqueue(IntPtr context, int priority, IntPtr obj);
+
+            [UnmanagedFunctionPointer(CallingConvention.StdCall)]
+            public delegate bool DispatcherHasThreadAccess(IntPtr context);
+        }
+
         public enum WindowMessage : int
         {
             /// <summary>
@@ -175,7 +191,7 @@ namespace Microsoft.Terminal.Wpf
         public static extern void AvoidBuggyTSFConsoleFlags();
 
         [DllImport("Microsoft.Terminal.Control.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.StdCall, PreserveSig = false)]
-        public static extern void CreateTerminal(IntPtr parent, out IntPtr hwnd, out IntPtr terminal);
+        public static extern void CreateTerminal(IntPtr parent, [MarshalAs(UnmanagedType.Struct)] DispatcherCallouts dispatcherCallouts, out IntPtr hwnd, out IntPtr terminal);
 
         [DllImport("Microsoft.Terminal.Control.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.StdCall)]
         public static extern void DestroyTerminal(IntPtr terminal);
@@ -221,6 +237,9 @@ namespace Microsoft.Terminal.Wpf
         [DllImport("Microsoft.Terminal.Control.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.StdCall, PreserveSig = false)]
         public static extern void TerminalSetTheme(IntPtr terminal, [MarshalAs(UnmanagedType.Struct)] TerminalTheme theme, string fontFamily, short fontSize, int newDpi);
 
+        [DllImport("Microsoft.Terminal.Control.dll", CharSet = CharSet.Unicode, CallingConvention = CallingConvention.StdCall, PreserveSig = false)]
+        public static extern void InteropQueueHandlerInvoke(IntPtr handler);
+
         [DllImport("user32.dll", SetLastError = true)]
         public static extern IntPtr SetFocus(IntPtr hWnd);
 

src/cascadia/WpfTerminalControl/TerminalContainer.cs

@@ -11,6 +11,7 @@ namespace Microsoft.Terminal.Wpf
     using System.Windows.Automation.Peers;
     using System.Windows.Interop;
     using System.Windows.Media;
+    using System.Windows.Threading;
 
     /// <summary>
     /// The container class that hosts the native hwnd terminal.
@@ -26,6 +27,40 @@ namespace Microsoft.Terminal.Wpf
         private NativeMethods.ScrollCallback scrollCallback;
         private NativeMethods.WriteCallback writeCallback;
 
+        private DispatcherWrapper dispatcherWrapper;
+
+        // These callbacks are provided to CreateTerminal so that the terminal
+        // core can schedule workitems on the same dispatcher queue as WPF.
+        // They must be pinned with GC handles so that they do not get garbage collected.
+        private NativeMethods.DispatcherCallouts dispatcherCallouts;
+        private GCHandle dispatcherTryEnqueueCalloutGCHandle;
+        private GCHandle dispatcherHasThreadAccessCalloutGCHandle;
+
+        private class DispatcherWrapper
+        {
+            private readonly Dispatcher dispatcher;
+
+            public DispatcherWrapper(Dispatcher dispatcher)
+            {
+                this.dispatcher = dispatcher;
+            }
+
+#pragma warning disable IDE0060
+            public bool TryEnqueue(IntPtr context, int priority, IntPtr obj)
+            {
+                var actualPrio = priority switch {
+                    < 0 => DispatcherPriority.Inactive,
+                    0 => DispatcherPriority.Normal,
+                    > 0 => DispatcherPriority.Send,
+                };
+                var task = this.dispatcher.InvokeAsync(() => NativeMethods.InteropQueueHandlerInvoke(obj), actualPrio);
+                return task != null;
+            }
+
+#pragma warning disable IDE0060
+            public bool HasThreadAccess(IntPtr context) => this.dispatcher.CheckAccess();
+        }
+
         /// <summary>
         /// Initializes a new instance of the <see cref="TerminalContainer"/> class.
         /// </summary>
@@ -36,6 +71,11 @@ namespace Microsoft.Terminal.Wpf
             // (On later versions of Windows it just doesn't work.)
             NativeMethods.AvoidBuggyTSFConsoleFlags();
 
+            this.dispatcherWrapper = new DispatcherWrapper(this.Dispatcher);
+            this.dispatcherCallouts = new NativeMethods.DispatcherCallouts { tryEnqueue = this.dispatcherWrapper.TryEnqueue, hasThreadAccess = this.dispatcherWrapper.HasThreadAccess, context = (IntPtr)0 /* unused */ };
+            this.dispatcherTryEnqueueCalloutGCHandle = GCHandle.Alloc(this.dispatcherCallouts.tryEnqueue);
+            this.dispatcherHasThreadAccessCalloutGCHandle = GCHandle.Alloc(this.dispatcherCallouts.hasThreadAccess);
+
             this.MessageHook += this.TerminalContainer_MessageHook;
             this.Focusable = true;
         }
@@ -285,7 +325,7 @@ namespace Microsoft.Terminal.Wpf
         protected override HandleRef BuildWindowCore(HandleRef hwndParent)
         {
             var dpiScale = VisualTreeHelper.GetDpi(this);
-            NativeMethods.CreateTerminal(hwndParent.Handle, out this.hwnd, out this.terminal);
+            NativeMethods.CreateTerminal(hwndParent.Handle, this.dispatcherCallouts, out this.hwnd, out this.terminal);
 
             this.scrollCallback = this.OnScroll;
             this.writeCallback = this.OnWrite;
@@ -306,6 +346,8 @@ namespace Microsoft.Terminal.Wpf
         protected override void DestroyWindowCore(HandleRef hwnd)
         {
             NativeMethods.DestroyTerminal(this.terminal);
+            this.dispatcherTryEnqueueCalloutGCHandle.Free();
+            this.dispatcherHasThreadAccessCalloutGCHandle.Free();
             this.terminal = IntPtr.Zero;
         }
 

src/cascadia/WpfTerminalControl/WpfTerminalControl.csproj

@@ -5,6 +5,7 @@
     <RootNamespace>Microsoft.Terminal.Wpf</RootNamespace>
     <AssemblyName>Microsoft.Terminal.Wpf</AssemblyName>
     <UseWpf>true</UseWpf>
+    <LangVersion>9.0</LangVersion>
 
     <RepoBinPath>$(MSBuildThisFileDirectory)..\..\..\bin\</RepoBinPath>
     <IntermediateOutputPath>$(RepoBinPath)..\obj\$(Platform)\$(Configuration)\$(MSBuildProjectName)</IntermediateOutputPath>