diff --git a/.vscode/launch.json b/.vscode/launch.json index e9928444..22c76d85 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -34,18 +34,19 @@ ] }, { - "name": "MochaTest", + "name": "vcpkg-artifacts mocha", "type": "node", - "request": "attach", - "port": 9229, - "continueOnAttach": true, - "autoAttachChildProcesses": false, + "request": "launch", + "cwd": "${workspaceFolder}/vcpkg-artifacts", + "program": "${workspaceFolder}/vcpkg-artifacts/node_modules/mocha/bin/mocha.js", + "args": [ + "--import=tsx", + "--no-timeout" + ], + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen", "skipFiles": [ "/**" - ], - "outFiles": [ - "${workspaceFolder}/ce/*/dist/**/*.js", - "!**/node_modules/**" ] } ] diff --git a/.vscode/settings.json b/.vscode/settings.json index 60fa0d61..5abf7c82 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,29 +1,9 @@ { - "typescript.tsdk": "ce/ce/common/temp/node_modules/typescript/lib", - "mochaExplorer.files": "./ce/test/**/*.js", - "mochaExplorer.ignore": [ - "./ce/test/**/*.d.ts", - "**/node_modules/**", - "./ce/ce/**", - "./ce/common/**/*.ts", - "./ce/common/**/*.js" - ], - "mochaExplorer.cwd": "./ce/test", - "mochaExplorer.mochaPath": "./ce/test/node_modules/mocha", - "mochaExplorer.timeout": 500000, - "mochaExplorer.require": [ - "source-map-support/register", - ], - "mochaExplorer.monkeyPatch": true, - "mochaExplorer.debuggerConfig": "MochaTest", + "js/ts.tsdk.path": "./vcpkg-artifacts/node_modules/typescript/lib", "eslint.workingDirectories": [ { "changeProcessCWD": true, - "directory": "./ce/ce", - }, - { - "changeProcessCWD": true, - "directory": "./ce/test", + "directory": "./vcpkg-artifacts", } ], "editor.codeActionsOnSave": { @@ -100,5 +80,20 @@ "typeinfo": "cpp", "variant": "cpp", "__bit_reference": "cpp" - } + }, + "C_Cpp.default.compileCommands": [ + "${workspaceFolder}/out/build/Win-x64-Debug/compile_commands.json", + "${workspaceFolder}/out/build/Win-x64-Debug-WithArtifacts/compile_commands.json", + "${workspaceFolder}/out/build/Win-x64-Debug-NoAnalyze/compile_commands.json", + "${workspaceFolder}/out/build/Win-x64-Debug-Official/compile_commands.json", + "${workspaceFolder}/out/build/Win-x64-Release/compile_commands.json", + "${workspaceFolder}/out/build/Win-x64-Release-Official/compile_commands.json", + "${workspaceFolder}/out/build/windows-ci/compile_commands.json", + "${workspaceFolder}/out/build/linux-with-artifacts/compile_commands.json", + "${workspaceFolder}/out/build/linux-debug/compile_commands.json", + "${workspaceFolder}/out/build/linux-ci/compile_commands.json", + "${workspaceFolder}/out/build/linux-codespaces/compile_commands.json", + "${workspaceFolder}/out/build/macos/compile_commands.json", + "${workspaceFolder}/out/build/macos-ci/compile_commands.json" + ] } diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/baseline.txt b/azure-pipelines/e2e-assets/ci-regression-reports/baseline.txt new file mode 100644 index 00000000..1aaf8c1a --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/baseline.txt @@ -0,0 +1,12 @@ +depends-on-fail-but-unsupported:cross=fail +dynamic-cascade-required-to-pass:cross=pass +fail-but-unsupported:cross=fail +fail-expected:cross=fail +pass-but-fails:cross=pass +pass-but-marked-fail:cross=fail +pass-but-unsupported:cross=pass +skip:cross=skip +pass-but-depends-on-skipped:cross=pass +static-fail-cascade-required-to-pass:cross=pass +static-skipped-cascade-required-to-pass:cross=pass +static-unsupported-cascade-required-to-pass:cross=pass diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/depends-on-fail-but-unsupported/portfile.cmake b/azure-pipelines/e2e-assets/ci-regression-reports/depends-on-fail-but-unsupported/portfile.cmake new file mode 100644 index 00000000..065116c2 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/depends-on-fail-but-unsupported/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/depends-on-fail-but-unsupported/vcpkg.json b/azure-pipelines/e2e-assets/ci-regression-reports/depends-on-fail-but-unsupported/vcpkg.json new file mode 100644 index 00000000..3be4c2f5 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/depends-on-fail-but-unsupported/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "depends-on-fail-but-unsupported", + "version": "1.0.0", + "description": "this port is marked fail but is unsupported", + "dependencies": [ + "fail-but-unsupported" + ] +} diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/dynamic-cascade-required-to-pass/portfile.cmake b/azure-pipelines/e2e-assets/ci-regression-reports/dynamic-cascade-required-to-pass/portfile.cmake new file mode 100644 index 00000000..7168678b --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/dynamic-cascade-required-to-pass/portfile.cmake @@ -0,0 +1 @@ +message(FATAL_ERROR "This port is expected to cascade and this statement will never be reached.") diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/dynamic-cascade-required-to-pass/vcpkg.json b/azure-pipelines/e2e-assets/ci-regression-reports/dynamic-cascade-required-to-pass/vcpkg.json new file mode 100644 index 00000000..0c7983b5 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/dynamic-cascade-required-to-pass/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "dynamic-cascade-required-to-pass", + "version": "1.0.0", + "description": "this port fails", + "dependencies": [ + "fail" + ] +} diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/fail-but-unsupported/portfile.cmake b/azure-pipelines/e2e-assets/ci-regression-reports/fail-but-unsupported/portfile.cmake new file mode 100644 index 00000000..e8405ee4 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/fail-but-unsupported/portfile.cmake @@ -0,0 +1 @@ +message(FATAL_ERROR "This port is always unsupported so this statement should never be reached.") diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/fail-but-unsupported/vcpkg.json b/azure-pipelines/e2e-assets/ci-regression-reports/fail-but-unsupported/vcpkg.json new file mode 100644 index 00000000..c2a704b0 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/fail-but-unsupported/vcpkg.json @@ -0,0 +1,6 @@ +{ + "name": "fail-but-unsupported", + "version": "1.0.0", + "description": "this port is marked fail but is unsupported", + "supports": "native & !native" +} diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/fail-expected/portfile.cmake b/azure-pipelines/e2e-assets/ci-regression-reports/fail-expected/portfile.cmake new file mode 100644 index 00000000..985392b0 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/fail-expected/portfile.cmake @@ -0,0 +1 @@ +message(FATAL_ERROR "This port is expected to fail.") diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/fail-expected/vcpkg.json b/azure-pipelines/e2e-assets/ci-regression-reports/fail-expected/vcpkg.json new file mode 100644 index 00000000..f637502a --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/fail-expected/vcpkg.json @@ -0,0 +1,5 @@ +{ + "name": "fail-expected", + "version": "1.0.0", + "description": "this port fails" +} diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/fail/portfile.cmake b/azure-pipelines/e2e-assets/ci-regression-reports/fail/portfile.cmake new file mode 100644 index 00000000..985392b0 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/fail/portfile.cmake @@ -0,0 +1 @@ +message(FATAL_ERROR "This port is expected to fail.") diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/fail/vcpkg.json b/azure-pipelines/e2e-assets/ci-regression-reports/fail/vcpkg.json new file mode 100644 index 00000000..488ae8e1 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/fail/vcpkg.json @@ -0,0 +1,5 @@ +{ + "name": "fail", + "version": "1.0.0", + "description": "this port fails" +} diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-fails/portfile.cmake b/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-fails/portfile.cmake new file mode 100644 index 00000000..985392b0 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-fails/portfile.cmake @@ -0,0 +1 @@ +message(FATAL_ERROR "This port is expected to fail.") diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-fails/vcpkg.json b/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-fails/vcpkg.json new file mode 100644 index 00000000..b078b6bc --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-fails/vcpkg.json @@ -0,0 +1,5 @@ +{ + "name": "pass-but-fails", + "version": "1.0.0", + "description": "this port is marked pass but fails" +} diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-marked-fail/portfile.cmake b/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-marked-fail/portfile.cmake new file mode 100644 index 00000000..065116c2 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-marked-fail/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-marked-fail/vcpkg.json b/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-marked-fail/vcpkg.json new file mode 100644 index 00000000..657b5a7b --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-marked-fail/vcpkg.json @@ -0,0 +1,5 @@ +{ + "name": "pass-but-marked-fail", + "version": "1.0.0", + "description": "this port passes but is marked to fail" +} diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-unsupported/portfile.cmake b/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-unsupported/portfile.cmake new file mode 100644 index 00000000..e8405ee4 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-unsupported/portfile.cmake @@ -0,0 +1 @@ +message(FATAL_ERROR "This port is always unsupported so this statement should never be reached.") diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-unsupported/vcpkg.json b/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-unsupported/vcpkg.json new file mode 100644 index 00000000..7718d6be --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/pass-but-unsupported/vcpkg.json @@ -0,0 +1,6 @@ +{ + "name": "pass-but-unsupported", + "version": "1.0.0", + "description": "this port is marked pass but is unsupported", + "supports": "native & !native" +} diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/skip/portfile.cmake b/azure-pipelines/e2e-assets/ci-regression-reports/skip/portfile.cmake new file mode 100644 index 00000000..ff2a5b69 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/skip/portfile.cmake @@ -0,0 +1 @@ +message(FATAL_ERROR "This port is skipped so this statement should never be reached.") diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/skip/vcpkg.json b/azure-pipelines/e2e-assets/ci-regression-reports/skip/vcpkg.json new file mode 100644 index 00000000..3f739106 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/skip/vcpkg.json @@ -0,0 +1,5 @@ +{ + "name": "skip", + "version": "1.0.0", + "description": "this port is skipped" +} diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/static-fail-cascade-required-to-pass/portfile.cmake b/azure-pipelines/e2e-assets/ci-regression-reports/static-fail-cascade-required-to-pass/portfile.cmake new file mode 100644 index 00000000..7168678b --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/static-fail-cascade-required-to-pass/portfile.cmake @@ -0,0 +1 @@ +message(FATAL_ERROR "This port is expected to cascade and this statement will never be reached.") diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/static-fail-cascade-required-to-pass/vcpkg.json b/azure-pipelines/e2e-assets/ci-regression-reports/static-fail-cascade-required-to-pass/vcpkg.json new file mode 100644 index 00000000..a3664c97 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/static-fail-cascade-required-to-pass/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "static-fail-cascade-required-to-pass", + "version": "1.0.0", + "description": "this port depends on a port (correctly) expected to fail, so it cascades statically", + "dependencies": [ + "fail-expected" + ] +} diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/static-skipped-cascade-required-to-pass/portfile.cmake b/azure-pipelines/e2e-assets/ci-regression-reports/static-skipped-cascade-required-to-pass/portfile.cmake new file mode 100644 index 00000000..7168678b --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/static-skipped-cascade-required-to-pass/portfile.cmake @@ -0,0 +1 @@ +message(FATAL_ERROR "This port is expected to cascade and this statement will never be reached.") diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/static-skipped-cascade-required-to-pass/vcpkg.json b/azure-pipelines/e2e-assets/ci-regression-reports/static-skipped-cascade-required-to-pass/vcpkg.json new file mode 100644 index 00000000..5991c278 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/static-skipped-cascade-required-to-pass/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "static-skipped-cascade-required-to-pass", + "version": "1.0.0", + "description": "this port depends on a port skipped so it cascades but is marked as needing to pass", + "dependencies": [ + "skip" + ] +} diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/static-unsupported-cascade-required-to-pass/portfile.cmake b/azure-pipelines/e2e-assets/ci-regression-reports/static-unsupported-cascade-required-to-pass/portfile.cmake new file mode 100644 index 00000000..7168678b --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/static-unsupported-cascade-required-to-pass/portfile.cmake @@ -0,0 +1 @@ +message(FATAL_ERROR "This port is expected to cascade and this statement will never be reached.") diff --git a/azure-pipelines/e2e-assets/ci-regression-reports/static-unsupported-cascade-required-to-pass/vcpkg.json b/azure-pipelines/e2e-assets/ci-regression-reports/static-unsupported-cascade-required-to-pass/vcpkg.json new file mode 100644 index 00000000..8e77d84b --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-regression-reports/static-unsupported-cascade-required-to-pass/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "static-unsupported-cascade-required-to-pass", + "version": "1.0.0", + "description": "this port depends on a port statically known to be unsupported so it cascades but is marked as needing to pass", + "dependencies": [ + "fail-but-unsupported" + ] +} diff --git a/azure-pipelines/e2e-assets/ci-skip-failures-features/baseline.txt b/azure-pipelines/e2e-assets/ci-skip-failures-features/baseline.txt new file mode 100644 index 00000000..81f5d77c --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-skip-failures-features/baseline.txt @@ -0,0 +1,9 @@ +fail-port:cross=fail +skip-port:cross=skip + +skip-port-host:cross=skip +skip-port-host:arm64-osx=skip +skip-port-host:x64-linux=skip +skip-port-host:arm64-linux=skip +skip-port-host:x64-windows=skip +skip-port-host:x86-windows=skip diff --git a/azure-pipelines/e2e-assets/ci-skip-failures-features/fail-port/portfile.cmake b/azure-pipelines/e2e-assets/ci-skip-failures-features/fail-port/portfile.cmake new file mode 100644 index 00000000..e33e0396 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-skip-failures-features/fail-port/portfile.cmake @@ -0,0 +1 @@ +message(FATAL_ERROR "This port fails to build at runtime.") diff --git a/azure-pipelines/e2e-assets/ci-skip-failures-features/fail-port/vcpkg.json b/azure-pipelines/e2e-assets/ci-skip-failures-features/fail-port/vcpkg.json new file mode 100644 index 00000000..66cce145 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-skip-failures-features/fail-port/vcpkg.json @@ -0,0 +1,17 @@ +{ + "name": "fail-port", + "version": "1.0.0", + "dependencies": [ + { + "name": "shared-dependency", + "default-features": false, + "features": [ + "depended-on-by-fail" + ] + }, + { + "name": "host-dependency", + "host": true + } + ] +} \ No newline at end of file diff --git a/azure-pipelines/e2e-assets/ci-skip-failures-features/host-dependency/portfile.cmake b/azure-pipelines/e2e-assets/ci-skip-failures-features/host-dependency/portfile.cmake new file mode 100644 index 00000000..0015715f --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-skip-failures-features/host-dependency/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) \ No newline at end of file diff --git a/azure-pipelines/e2e-assets/ci-skip-failures-features/host-dependency/vcpkg.json b/azure-pipelines/e2e-assets/ci-skip-failures-features/host-dependency/vcpkg.json new file mode 100644 index 00000000..6a1d42ae --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-skip-failures-features/host-dependency/vcpkg.json @@ -0,0 +1,4 @@ +{ + "name": "host-dependency", + "version": "1.0.0" +} \ No newline at end of file diff --git a/azure-pipelines/e2e-assets/ci-skip-failures-features/normal-port/portfile.cmake b/azure-pipelines/e2e-assets/ci-skip-failures-features/normal-port/portfile.cmake new file mode 100644 index 00000000..0015715f --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-skip-failures-features/normal-port/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) \ No newline at end of file diff --git a/azure-pipelines/e2e-assets/ci-skip-failures-features/normal-port/vcpkg.json b/azure-pipelines/e2e-assets/ci-skip-failures-features/normal-port/vcpkg.json new file mode 100644 index 00000000..09ef4d56 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-skip-failures-features/normal-port/vcpkg.json @@ -0,0 +1,12 @@ +{ + "name": "normal-port", + "version": "1.0.0", + "dependencies": [ + { + "name": "shared-dependency", + "features": [ + "depended-on-by-normal" + ] + } + ] +} \ No newline at end of file diff --git a/azure-pipelines/e2e-assets/ci-skip-failures-features/shared-dependency/portfile.cmake b/azure-pipelines/e2e-assets/ci-skip-failures-features/shared-dependency/portfile.cmake new file mode 100644 index 00000000..ce1b6205 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-skip-failures-features/shared-dependency/portfile.cmake @@ -0,0 +1,4 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) +if ("depended-on-by-skip" IN_LIST FEATURES) + message(FATAL_ERROR "The feature 'depended-on-by-skip' should not be included.") +endif() \ No newline at end of file diff --git a/azure-pipelines/e2e-assets/ci-skip-failures-features/shared-dependency/vcpkg.json b/azure-pipelines/e2e-assets/ci-skip-failures-features/shared-dependency/vcpkg.json new file mode 100644 index 00000000..6d835adc --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-skip-failures-features/shared-dependency/vcpkg.json @@ -0,0 +1,24 @@ +{ + "name": "shared-dependency", + "version": "1.0.0", + "default-features": [ + "default-feature" + ], + "features": { + "default-feature": { + "description": "Default feature that keeps the target dependency in the plan.", + "dependencies": [ + "host-dependency" + ] + }, + "depended-on-by-fail": { + "description": "Selected by fail-port." + }, + "depended-on-by-normal": { + "description": "Selected by normal-port." + }, + "depended-on-by-skip": { + "description": "Selected only by skipped ports." + } + } +} \ No newline at end of file diff --git a/azure-pipelines/e2e-assets/ci-skip-failures-features/skip-port-host/portfile.cmake b/azure-pipelines/e2e-assets/ci-skip-failures-features/skip-port-host/portfile.cmake new file mode 100644 index 00000000..8088f5d1 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-skip-failures-features/skip-port-host/portfile.cmake @@ -0,0 +1 @@ +message(FATAL_ERROR "This port must be skipped.") diff --git a/azure-pipelines/e2e-assets/ci-skip-failures-features/skip-port-host/vcpkg.json b/azure-pipelines/e2e-assets/ci-skip-failures-features/skip-port-host/vcpkg.json new file mode 100644 index 00000000..ac68fc2a --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-skip-failures-features/skip-port-host/vcpkg.json @@ -0,0 +1,14 @@ +{ + "name": "skip-port-host", + "version": "1.0.0", + "dependencies": [ + { + "name": "shared-dependency", + "default-features": false, + "features": [ + "depended-on-by-skip" + ], + "host": true + } + ] +} \ No newline at end of file diff --git a/azure-pipelines/e2e-assets/ci-skip-failures-features/skip-port/portfile.cmake b/azure-pipelines/e2e-assets/ci-skip-failures-features/skip-port/portfile.cmake new file mode 100644 index 00000000..8088f5d1 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-skip-failures-features/skip-port/portfile.cmake @@ -0,0 +1 @@ +message(FATAL_ERROR "This port must be skipped.") diff --git a/azure-pipelines/e2e-assets/ci-skip-failures-features/skip-port/vcpkg.json b/azure-pipelines/e2e-assets/ci-skip-failures-features/skip-port/vcpkg.json new file mode 100644 index 00000000..cda728b3 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-skip-failures-features/skip-port/vcpkg.json @@ -0,0 +1,17 @@ +{ + "name": "skip-port", + "version": "1.0.0", + "dependencies": [ + { + "name": "shared-dependency", + "default-features": false, + "features": [ + "depended-on-by-skip" + ] + }, + { + "name": "skip-port-host", + "host": true + } + ] +} \ No newline at end of file diff --git a/azure-pipelines/e2e-assets/ci-skipped-ports/baseline.fail.txt b/azure-pipelines/e2e-assets/ci-skipped-ports/baseline.fail.txt index 39f3e77f..ba40ab78 100644 --- a/azure-pipelines/e2e-assets/ci-skipped-ports/baseline.fail.txt +++ b/azure-pipelines/e2e-assets/ci-skipped-ports/baseline.fail.txt @@ -1,15 +1,13 @@ -# fail means skip with --skip-failures -always-skip:cross=fail -always-skip:arm64-linux=fail -always-skip:arm64-osx=fail -always-skip:x64-linux=fail -always-skip:x64-windows=fail -always-skip:x86-windows=fail +always-skip:cross=skip +always-skip:arm64-linux=skip +always-skip:arm64-osx=skip +always-skip:x64-linux=skip +always-skip:x64-windows=skip +always-skip:x86-windows=skip -# fail means skip here, but cached artifacts may exist # not skipping maybe-skip for "cross" triplet -maybe-skip:arm64-linux=fail -maybe-skip:arm64-osx=fail -maybe-skip:x64-linux=fail -maybe-skip:x64-windows=fail -maybe-skip:x86-windows=fail +maybe-skip:arm64-linux=skip +maybe-skip:arm64-osx=skip +maybe-skip:x64-linux=skip +maybe-skip:x64-windows=skip +maybe-skip:x86-windows=skip diff --git a/azure-pipelines/e2e-assets/ci-transitive/transitive-1/portfile.cmake b/azure-pipelines/e2e-assets/ci-transitive/transitive-1/portfile.cmake new file mode 100644 index 00000000..065116c2 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-transitive/transitive-1/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/azure-pipelines/e2e-assets/ci-transitive/transitive-1/vcpkg.json b/azure-pipelines/e2e-assets/ci-transitive/transitive-1/vcpkg.json new file mode 100644 index 00000000..5c471423 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-transitive/transitive-1/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "transitive-1", + "version": "1.0.0", + "description": "transitive-1", + "dependencies": [ + "transitive-2" + ] +} diff --git a/azure-pipelines/e2e-assets/ci-transitive/transitive-2/portfile.cmake b/azure-pipelines/e2e-assets/ci-transitive/transitive-2/portfile.cmake new file mode 100644 index 00000000..065116c2 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-transitive/transitive-2/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/azure-pipelines/e2e-assets/ci-transitive/transitive-2/vcpkg.json b/azure-pipelines/e2e-assets/ci-transitive/transitive-2/vcpkg.json new file mode 100644 index 00000000..d37af6c0 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-transitive/transitive-2/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "transitive-2", + "version": "1.0.0", + "description": "transitive-2", + "dependencies": [ + "transitive-3" + ] +} diff --git a/azure-pipelines/e2e-assets/ci-transitive/transitive-3/portfile.cmake b/azure-pipelines/e2e-assets/ci-transitive/transitive-3/portfile.cmake new file mode 100644 index 00000000..065116c2 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-transitive/transitive-3/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/azure-pipelines/e2e-assets/ci-transitive/transitive-3/vcpkg.json b/azure-pipelines/e2e-assets/ci-transitive/transitive-3/vcpkg.json new file mode 100644 index 00000000..0026e870 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-transitive/transitive-3/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "transitive-3", + "version": "1.0.0", + "description": "transitive-3", + "dependencies": [ + "transitive-4" + ] +} diff --git a/azure-pipelines/e2e-assets/ci-transitive/transitive-4/portfile.cmake b/azure-pipelines/e2e-assets/ci-transitive/transitive-4/portfile.cmake new file mode 100644 index 00000000..065116c2 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-transitive/transitive-4/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/azure-pipelines/e2e-assets/ci-transitive/transitive-4/vcpkg.json b/azure-pipelines/e2e-assets/ci-transitive/transitive-4/vcpkg.json new file mode 100644 index 00000000..3db6c96c --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-transitive/transitive-4/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "transitive-4", + "version": "1.0.0", + "description": "transitive-4", + "dependencies": [ + "transitive-5" + ] +} diff --git a/azure-pipelines/e2e-assets/ci-transitive/transitive-5/portfile.cmake b/azure-pipelines/e2e-assets/ci-transitive/transitive-5/portfile.cmake new file mode 100644 index 00000000..065116c2 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-transitive/transitive-5/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/azure-pipelines/e2e-assets/ci-transitive/transitive-5/vcpkg.json b/azure-pipelines/e2e-assets/ci-transitive/transitive-5/vcpkg.json new file mode 100644 index 00000000..40c664a6 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-transitive/transitive-5/vcpkg.json @@ -0,0 +1,8 @@ +{ + "name": "transitive-5", + "version": "1.0.0", + "description": "transitive-5", + "dependencies": [ + "transitive-6" + ] +} diff --git a/azure-pipelines/e2e-assets/ci-transitive/transitive-6/portfile.cmake b/azure-pipelines/e2e-assets/ci-transitive/transitive-6/portfile.cmake new file mode 100644 index 00000000..065116c2 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-transitive/transitive-6/portfile.cmake @@ -0,0 +1 @@ +set(VCPKG_POLICY_EMPTY_PACKAGE enabled) diff --git a/azure-pipelines/e2e-assets/ci-transitive/transitive-6/vcpkg.json b/azure-pipelines/e2e-assets/ci-transitive/transitive-6/vcpkg.json new file mode 100644 index 00000000..aaacfb89 --- /dev/null +++ b/azure-pipelines/e2e-assets/ci-transitive/transitive-6/vcpkg.json @@ -0,0 +1,5 @@ +{ + "name": "transitive-6", + "version": "1.0.0", + "description": "transitive-6" +} diff --git a/azure-pipelines/end-to-end-tests-dir/ci.ps1 b/azure-pipelines/end-to-end-tests-dir/ci.ps1 index 05f24489..82f8e1eb 100644 --- a/azure-pipelines/end-to-end-tests-dir/ci.ps1 +++ b/azure-pipelines/end-to-end-tests-dir/ci.ps1 @@ -1,9 +1,7 @@ . $PSScriptRoot/../end-to-end-tests-prelude.ps1 # test skipped ports -$Output = Run-VcpkgAndCaptureOutput ci --dry-run --triplet=$Triplet --x-builtin-ports-root="$PSScriptRoot/../e2e-ports/ci" --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci/ci.baseline.txt" -Throw-IfNotFailed -$ErrorOutput = Run-VcpkgAndCaptureStdErr ci --dry-run --triplet=$Triplet --x-builtin-ports-root="$PSScriptRoot/../e2e-ports/ci" --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci/ci.baseline.txt" +$Output = Run-VcpkgAndCaptureBoth ci --dry-run --triplet=$Triplet --x-builtin-ports-root="$PSScriptRoot/../e2e-ports/ci" --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci/ci.baseline.txt" Throw-IfNotFailed # dep-on-feature-not-sup must cascade because it depends on a features that is not supported Throw-IfNonContains -Actual $Output -Expected "dep-on-feature-not-sup:${Triplet}: cascade" @@ -18,24 +16,22 @@ if ($Output.Split("*").Length -ne 4) { } Throw-IfNonContains -Actual $Output -Expected @" SUMMARY FOR $Triplet - CASCADED_DUE_TO_MISSING_DEPENDENCIES: 1 + CASCADED_DUE_TO_SUPPORTS: 1 SKIPPED_BY_DRY_RUN: 3 UNSUPPORTED: 1 "@ # feature-not-sup's baseline fail entry should result in a regression because the port is not supported -Throw-IfNonContains -Actual $ErrorOutput -Expected "REGRESSION: not-sup-host-b:${Triplet} is marked as fail but not supported for ${Triplet}." +Throw-IfNonContains -Actual $Output -Expected "REGRESSION: not-sup-host-b:${Triplet} is marked as fail but not supported for ${Triplet}." # feature-not-sup's baseline fail entry should result in a regression because the port is cascade for this triplet -Throw-IfNonContains -Actual $ErrorOutput -Expected "REGRESSION: dep-on-feature-not-sup:${Triplet} is marked as fail but one dependency is not supported for ${Triplet}." +Throw-IfNonContains -Actual $Output -Expected "REGRESSION: dep-on-feature-not-sup:${Triplet} is marked as fail but one dependency is not supported for ${Triplet}." # pass means pass -Run-Vcpkg ci --dry-run --triplet=$Triplet --x-builtin-ports-root="$PSScriptRoot/../e2e-ports/ci" --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci/ci.baseline.txt" -Throw-IfNotFailed -$ErrorOutput = Run-VcpkgAndCaptureStdErr ci --dry-run --triplet=$Triplet --x-builtin-ports-root="$PSScriptRoot/../e2e-ports/ci" --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci/ci.pass.baseline.txt" +$Output = Run-VcpkgAndCaptureBoth ci --dry-run --triplet=$Triplet --x-builtin-ports-root="$PSScriptRoot/../e2e-ports/ci" --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci/ci.pass.baseline.txt" Throw-IfNotFailed # feature-not-sup's baseline pass entry should result in a regression because the port is not supported -Throw-IfNonContains -Actual $ErrorOutput -Expected "REGRESSION: not-sup-host-b:${Triplet} is marked as pass but not supported for ${Triplet}." +Throw-IfNonContains -Actual $Output -Expected "REGRESSION: not-sup-host-b:${Triplet} is marked as pass but not supported for ${Triplet}." # feature-not-sup's baseline pass entry should result in a regression because the port is cascade for this triplet -Throw-IfNonContains -Actual $ErrorOutput -Expected "REGRESSION: dep-on-feature-not-sup:${Triplet} cascaded, but it is required to pass. (" +Throw-IfNonContains -Actual $Output -Expected "REGRESSION: dep-on-feature-not-sup:${Triplet} cascaded, but it is required to pass. (" # any invalid manifest must raise an error Remove-Problem-Matchers @@ -86,7 +82,7 @@ Throw-IfFailed Throw-IfNonContains -Actual $Output -Expected "base-port:${Triplet}: parent: " Throw-IfNonContains -Actual $Output -Expected @" SUMMARY FOR $Triplet - CASCADED_DUE_TO_MISSING_DEPENDENCIES: 1 + CASCADED_DUE_TO_SUPPORTS: 1 SKIPPED_BY_PARENT_HASHES: 3 UNSUPPORTED: 1 "@ @@ -116,7 +112,7 @@ if (-not ($Output -match 'Building maybe-cross-cascade:[^@]+@1\.0\.0\.\.\.')) { Throw-IfNonContains -Actual $Output -Expected @" SUMMARY FOR $Triplet SUCCEEDED: 4 - CASCADED_DUE_TO_MISSING_DEPENDENCIES: 1 + CASCADED_DUE_TO_BASELINE: 1 SKIPPED: 2 UNSUPPORTED: 5 "@ @@ -143,7 +139,7 @@ Throw-IfNonContains -Actual $Output -Expected "never-built-unsupported-host:$($T Throw-IfNonContains -Actual $Output -Expected "never-built-unsupported-host-feature:$($Triplet): unsupported" Throw-IfNonContains -Actual $Output -Expected @" SUMMARY FOR $Triplet - CASCADED_DUE_TO_MISSING_DEPENDENCIES: 4 + CASCADED_DUE_TO_BASELINE: 4 SKIPPED: 2 UNSUPPORTED: 5 CACHED: 1 @@ -164,7 +160,7 @@ Throw-IfNonContains -Actual $Output -Expected 'never-built-unsupported-host:cros Throw-IfNonContains -Actual $Output -Expected 'never-built-unsupported-host-feature:cross: unsupported' Throw-IfNonContains -Actual $Output -Expected @" SUMMARY FOR cross - CASCADED_DUE_TO_MISSING_DEPENDENCIES: 2 + CASCADED_DUE_TO_BASELINE: 2 SKIPPED: 1 SKIPPED_BY_DRY_RUN: 4 UNSUPPORTED: 5 @@ -186,7 +182,7 @@ Throw-IfContains -Actual $Output -Expected 'Building skipped-depends' Throw-IfNonContains -Actual $Output -Expected @" SUMMARY FOR $Triplet SUCCEEDED: 1 - CASCADED_DUE_TO_MISSING_DEPENDENCIES: 1 + CASCADED_DUE_TO_BASELINE: 1 SKIPPED: 1 "@ @@ -220,7 +216,7 @@ $expected = @" - + @@ -232,3 +228,183 @@ if (-not ($xunitContent -match $expected)) { Write-Diff -Actual $xunitContent -Expected $expected throw 'xUnit output did not match expected output' } + +# test that dependencies of failure baselines still contribute features under --skip-failures +Refresh-TestRoot +Copy-Item $tripletFile "$TestingRoot/cross.cmake" +Remove-Problem-Matchers +$Output = Run-VcpkgAndCaptureOutput ci --triplet cross --overlay-triplets $TestingRoot @directoryArgs --x-builtin-ports-root="$PSScriptRoot/../e2e-assets/ci-skip-failures-features" --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci-skip-failures-features/baseline.txt" +Restore-Problem-Matchers +Throw-IfFailed +# note that depended-on-by-skip is not included but depended-on-by-fail is +Throw-IfNonContains -Actual $Output -Expected 'Building shared-dependency[core,default-feature,depended-on-by-fail,depended-on-by-normal]:cross@1.0.0...' +Throw-IfNonContains -Actual $Output -Expected 'Building host-dependency:cross@1.0.0...' +Throw-IfNonContains -Actual $Output -Expected "Building fail-port:cross@1.0.0..." +Throw-IfNonContains -Actual $Output -Expected "Building host-dependency:$Triplet@1.0.0..." +Throw-IfContains -Actual $Output -Expected "Building skip-port" +Throw-IfContains -Actual $Output -Expected "Building skip-port-host" +Throw-IfContains -Actual $Output -Expected "fail-port:$($Triplet):" +Throw-IfContains -Actual $Output -Expected "normal-port:$($Triplet):" +Throw-IfContains -Actual $Output -Expected "shared-dependency:$($Triplet):" +Throw-IfContains -Actual $Output -Expected "skip-port:$($Triplet):" +Throw-IfNonContains -Actual $Output -Expected @" +SUMMARY FOR cross + SUCCEEDED: 3 + BUILD_FAILED: 1 + SKIPPED: 2 +"@ + +Throw-IfNonContains -Actual $Output -Expected @" +SUMMARY FOR $Triplet + SUCCEEDED: 1 +"@ + +Refresh-TestRoot +Copy-Item $tripletFile "$TestingRoot/cross.cmake" +Remove-Problem-Matchers +$Output = Run-VcpkgAndCaptureOutput ci --skip-failures --triplet cross --overlay-triplets $TestingRoot @directoryArgs --x-builtin-ports-root="$PSScriptRoot/../e2e-assets/ci-skip-failures-features" --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci-skip-failures-features/baseline.txt" +Restore-Problem-Matchers +Throw-IfFailed +Throw-IfNonContains -Actual $Output -Expected 'Building shared-dependency[core,default-feature,depended-on-by-fail,depended-on-by-normal]:cross@1.0.0...' +Throw-IfNonContains -Actual $Output -Expected 'Building host-dependency:cross@1.0.0...' +Throw-IfContains -Actual $Output -Expected "Building fail-port:cross@1.0.0..." +Throw-IfNonContains -Actual $Output -Expected "Building host-dependency:$($Triplet)@1.0.0..." +Throw-IfContains -Actual $Output -Expected "Building skip-port" +Throw-IfContains -Actual $Output -Expected "Building skip-port-host" +Throw-IfContains -Actual $Output -Expected "fail-port:$($Triplet):" +Throw-IfContains -Actual $Output -Expected "normal-port:$($Triplet):" +Throw-IfContains -Actual $Output -Expected "shared-dependency:$($Triplet):" +Throw-IfContains -Actual $Output -Expected "skip-port:$($Triplet):" +Throw-IfNonContains -Actual $Output -Expected @" +SUMMARY FOR cross + SUCCEEDED: 3 + SKIPPED: 2 + SKIPPED_BY_SKIP_FAILURES: 1 +"@ + +Throw-IfNonContains -Actual $Output -Expected @" +SUMMARY FOR $Triplet + SUCCEEDED: 1 +"@ + +# test every "REGRESSION: " outcome +Refresh-TestRoot +Copy-Item $tripletFile "$TestingRoot/cross.cmake" +Remove-Problem-Matchers +$Output = Run-VcpkgAndCaptureBoth ci --triplet cross --overlay-triplets $TestingRoot @directoryArgs --x-builtin-ports-root="$PSScriptRoot/../e2e-assets/ci-regression-reports" --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci-regression-reports/baseline.txt" +Restore-Problem-Matchers +Throw-IfNotFailed +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: depends-on-fail-but-unsupported:cross is marked as fail but one dependency is not supported for cross.' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: dynamic-cascade-required-to-pass:cross cascaded, but it is required to pass.' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: fail:cross failed with BUILD_FAILED. If expected, add fail:cross=fail to ' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: fail-but-unsupported:cross is marked as fail but not supported for cross.' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: pass-but-fails:cross failed with BUILD_FAILED. If expected, add pass-but-fails:cross=fail to ' +Throw-IfNonContains -Actual $Output -Expected 'PASSING, REMOVE FROM FAIL LIST: pass-but-marked-fail:cross (' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: pass-but-unsupported:cross is marked as pass but not supported for cross.' +Throw-IfNonContains -Actual $Output -Expected 'skip:cross: skip' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: static-fail-cascade-required-to-pass:cross cascaded, but it is required to pass. (' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: static-skipped-cascade-required-to-pass:cross cascaded, but it is required to pass. (' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: static-unsupported-cascade-required-to-pass:cross cascaded, but it is required to pass. (' +Throw-IfNonContains -Actual $Output -Expected @" +SUMMARY FOR cross + SUCCEEDED: 1 + BUILD_FAILED: 3 + CASCADED_DUE_TO_MISSING_DEPENDENCIES: 2 + CASCADED_DUE_TO_SUPPORTS: 2 + CASCADED_DUE_TO_BASELINE: 1 + SKIPPED: 1 + UNSUPPORTED: 2 +"@ + +# ... and with --allow-unexpected-passing +Refresh-TestRoot +Copy-Item $tripletFile "$TestingRoot/cross.cmake" +Remove-Problem-Matchers +$Output = Run-VcpkgAndCaptureBoth ci --triplet cross --overlay-triplets $TestingRoot @directoryArgs --x-builtin-ports-root="$PSScriptRoot/../e2e-assets/ci-regression-reports" --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci-regression-reports/baseline.txt" --allow-unexpected-passing +Restore-Problem-Matchers +Throw-IfNotFailed +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: depends-on-fail-but-unsupported:cross is marked as fail but one dependency is not supported for cross.' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: dynamic-cascade-required-to-pass:cross cascaded, but it is required to pass.' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: fail:cross failed with BUILD_FAILED. If expected, add fail:cross=fail to ' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: fail-but-unsupported:cross is marked as fail but not supported for cross.' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: pass-but-fails:cross failed with BUILD_FAILED. If expected, add pass-but-fails:cross=fail to ' +Throw-IfContains -Actual $Output -Expected 'PASSING, REMOVE FROM FAIL LIST' # : pass-but-marked-fail:cross ( +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: pass-but-unsupported:cross is marked as pass but not supported for cross.' +Throw-IfNonContains -Actual $Output -Expected 'skip:cross: skip' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: static-fail-cascade-required-to-pass:cross cascaded, but it is required to pass. (' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: static-skipped-cascade-required-to-pass:cross cascaded, but it is required to pass. (' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: static-unsupported-cascade-required-to-pass:cross cascaded, but it is required to pass. (' +# note that --skip-failures means we did build pass-but-marked-fail but we didn't report it as a regression (it is SUCCEEDED) +Throw-IfNonContains -Actual $Output -Expected @" +SUMMARY FOR cross + SUCCEEDED: 1 + BUILD_FAILED: 3 + CASCADED_DUE_TO_MISSING_DEPENDENCIES: 2 + CASCADED_DUE_TO_SUPPORTS: 2 + CASCADED_DUE_TO_BASELINE: 1 + SKIPPED: 1 + UNSUPPORTED: 2 +"@ + +# ... and with --skip-failures +Refresh-TestRoot +Copy-Item $tripletFile "$TestingRoot/cross.cmake" +Remove-Problem-Matchers +$Output = Run-VcpkgAndCaptureBoth ci --triplet cross --overlay-triplets $TestingRoot @directoryArgs --x-builtin-ports-root="$PSScriptRoot/../e2e-assets/ci-regression-reports" --binarysource=clear --ci-baseline="$PSScriptRoot/../e2e-assets/ci-regression-reports/baseline.txt" --skip-failures +Restore-Problem-Matchers +Throw-IfNotFailed +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: depends-on-fail-but-unsupported:cross is marked as fail but one dependency is not supported for cross.' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: dynamic-cascade-required-to-pass:cross cascaded, but it is required to pass.' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: fail:cross failed with BUILD_FAILED. If expected, add fail:cross=fail to ' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: fail-but-unsupported:cross is marked as fail but not supported for cross.' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: pass-but-fails:cross failed with BUILD_FAILED. If expected, add pass-but-fails:cross=fail to ' +Throw-IfContains -Actual $Output -Expected 'PASSING, REMOVE FROM FAIL LIST' # : pass-but-marked-fail:cross ( +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: pass-but-unsupported:cross is marked as pass but not supported for cross.' +Throw-IfNonContains -Actual $Output -Expected 'skip:cross: skip' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: static-fail-cascade-required-to-pass:cross cascaded, but it is required to pass. (' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: static-skipped-cascade-required-to-pass:cross cascaded, but it is required to pass. (' +Throw-IfNonContains -Actual $Output -Expected 'REGRESSION: static-unsupported-cascade-required-to-pass:cross cascaded, but it is required to pass. (' +# note that --skip-failures means we didn't even try to build pass-but-marked-fail, it was SKIPPED_BY_SKIP_FAILURES +Throw-IfNonContains -Actual $Output -Expected @" +SUMMARY FOR cross + BUILD_FAILED: 3 + CASCADED_DUE_TO_MISSING_DEPENDENCIES: 2 + CASCADED_DUE_TO_SUPPORTS: 2 + CASCADED_DUE_TO_BASELINE: 1 + SKIPPED: 1 + SKIPPED_BY_SKIP_FAILURES: 1 + UNSUPPORTED: 2 +"@ + +# also ensure that transitive dependencies are kept in the plan even if a direct dependency is unnecessary +Refresh-TestRoot +Run-Vcpkg install transitive-3 --binarysource="clear;files,$ArchiveRoot,readwrite" @directoryArgs --x-builtin-ports-root="$PSScriptRoot/../e2e-assets/ci-transitive" +Throw-IfFailed +Run-Vcpkg remove transitive-3 transitive-4 transitive-5 transitive-6 @directoryArgs +Throw-IfFailed +$Output = Run-VcpkgAndCaptureBoth ci --binarysource="clear;files,$ArchiveRoot" @directoryArgs --x-builtin-ports-root="$PSScriptRoot/../e2e-assets/ci-transitive" +Throw-IfFailed + +Throw-IfNonContains -Actual $Output -Expected "Installing 1/6 transitive-6:$($Triplet)@1.0.0..." +Throw-IfContains -Actual $Output -Expected "Building transitive-6:$($Triplet)@1.0.0..." +Throw-IfNonContains -Actual $Output -Expected "Installing 2/6 transitive-5:$($Triplet)@1.0.0..." +Throw-IfContains -Actual $Output -Expected "Building transitive-5:$($Triplet)@1.0.0..." +Throw-IfNonContains -Actual $Output -Expected "Installing 3/6 transitive-4:$($Triplet)@1.0.0..." +Throw-IfContains -Actual $Output -Expected "Building transitive-4:$($Triplet)@1.0.0..." +Throw-IfNonContains -Actual $Output -Expected "Installing 4/6 transitive-3:$($Triplet)@1.0.0..." +Throw-IfContains -Actual $Output -Expected "Building transitive-3:$($Triplet)@1.0.0..." +Throw-IfNonContains -Actual $Output -Expected "Installing 5/6 transitive-2:$($Triplet)@1.0.0..." +Throw-IfNonContains -Actual $Output -Expected "Building transitive-2:$($Triplet)@1.0.0..." +Throw-IfNonContains -Actual $Output -Expected "Installing 6/6 transitive-1:$($Triplet)@1.0.0..." +Throw-IfNonContains -Actual $Output -Expected "Building transitive-1:$($Triplet)@1.0.0..." +Throw-IfNonContains -Actual $Output -Expected "transitive-1:$($Triplet): SUCCEEDED:" +Throw-IfNonContains -Actual $Output -Expected "transitive-2:$($Triplet): SUCCEEDED:" +Throw-IfNonContains -Actual $Output -Expected "transitive-3:$($Triplet): SUCCEEDED:" +Throw-IfNonContains -Actual $Output -Expected "transitive-4:$($Triplet): SUCCEEDED:" +Throw-IfNonContains -Actual $Output -Expected "transitive-5:$($Triplet): SUCCEEDED:" +Throw-IfNonContains -Actual $Output -Expected "transitive-6:$($Triplet): SUCCEEDED:" + +Throw-IfNonContains -Actual $Output -Expected @" +SUMMARY FOR $($Triplet) + SUCCEEDED: 6 +"@ diff --git a/azure-pipelines/end-to-end-tests-prelude.ps1 b/azure-pipelines/end-to-end-tests-prelude.ps1 index f5a02478..f3dd3f7b 100644 --- a/azure-pipelines/end-to-end-tests-prelude.ps1 +++ b/azure-pipelines/end-to-end-tests-prelude.ps1 @@ -135,7 +135,7 @@ function Run-VcpkgImplAndCaptureOutput { Write-Host -ForegroundColor red $Script:CurrentTest $result = (& "$ThisVcpkg" @testArgs) | Out-String Write-Host -ForegroundColor Gray $result - $result.Replace("`r`n", "`n") + return $result.Replace("`r`n", "`n") } function Run-VcpkgShellAndCaptureOutput { @@ -172,6 +172,19 @@ function Run-VcpkgAndCaptureStdErr { return $result.Replace("`r`n", "`n") } +function Run-VcpkgAndCaptureBoth { + Param( + [Parameter(ValueFromRemainingArguments)] + [string[]]$TestArgs + ) + + $Script:CurrentTest = "$VcpkgExe $($testArgs -join ' ')" + Write-Host -ForegroundColor red $Script:CurrentTest + $result = (& "$VcpkgExe" @testArgs 2>&1) | Out-String + Write-Host -ForegroundColor Gray $result + return $result.Replace("`r`n", "`n") +} + function Run-VcpkgShell { Param( [Parameter(ValueFromRemainingArguments)] diff --git a/include/vcpkg/base/message-data.inc.h b/include/vcpkg/base/message-data.inc.h index d1f3499f..5a5a972c 100644 --- a/include/vcpkg/base/message-data.inc.h +++ b/include/vcpkg/base/message-data.inc.h @@ -402,6 +402,16 @@ DECLARE_MESSAGE(BuildResultCascadeDueToMissingDependencies, "Printed after the name of an installed entity to indicate that it could not attempt " "to be installed because one of its transitive dependencies failed to install.", "CASCADED_DUE_TO_MISSING_DEPENDENCIES") +DECLARE_MESSAGE(BuildResultCascadeDueToSupports, + (), + "Printed after the name of an installed entity to indicate that it could not attempt " + "to be installed because one of its transitive dependencies' supports clause fails", + "CASCADED_DUE_TO_SUPPORTS") +DECLARE_MESSAGE(BuildResultCascadeDueToBaseline, + (), + "Printed after the name of an installed entity to indicate that it could not attempt " + "to be installed because one of its transitive dependencies is skipped in ci.baseline.txt.", + "CASCADED_DUE_TO_BASELINE") DECLARE_MESSAGE(BuildResultDownloaded, (), "Printed after the name of an installed entity to indicate that it was successfully " @@ -416,6 +426,11 @@ DECLARE_MESSAGE(BuildResultSkippedByDryRun, (), "Printed after the name of an entity that would be installed, but is not due to --dry-run.", "SKIPPED_BY_DRY_RUN") +DECLARE_MESSAGE(BuildResultSkippedBySkipFailures, + (), + "Printed after the name of an entity that would be attempted to be installed, but is known to fail and " + "vcpkg ci is configured in --skip-failures mode", + "SKIPPED_BY_SKIP_FAILURES") DECLARE_MESSAGE(BuildResultSkippedByParentHashes, (), "Printed after the name of an installed entity to indicate that it isn't tested due to an ABI hash in " diff --git a/include/vcpkg/ci-baseline.h b/include/vcpkg/ci-baseline.h index f353d27a..33c8e767 100644 --- a/include/vcpkg/ci-baseline.h +++ b/include/vcpkg/ci-baseline.h @@ -48,15 +48,14 @@ namespace vcpkg std::vector parse_ci_baseline(StringView text, StringView origin, ParseMessages& messages); + // assertions from a ci-baseline file struct CiBaselineData { - SortedVector expected_failures; - SortedVector required_success; + SortedVector expected_fail; + SortedVector required_pass; }; - CiBaselineData parse_and_apply_ci_baseline(View lines, - SkipsMap& skips_map, - SkipFailures skip_failures); + CiBaselineData parse_and_apply_ci_baseline(View lines, SkipsMap& skips_map); LocalizedString format_ci_result(const PackageSpec& spec, BuildResult result, diff --git a/include/vcpkg/commands.build.h b/include/vcpkg/commands.build.h index 76da098f..e0d4708e 100644 --- a/include/vcpkg/commands.build.h +++ b/include/vcpkg/commands.build.h @@ -118,9 +118,12 @@ namespace vcpkg int post_build_checks_failed = 0; int file_conflicts = 0; int cascaded_due_to_missing_dependencies = 0; + int cascaded_due_to_supports = 0; + int cascaded_due_to_baseline = 0; int skipped = 0; int skipped_by_parent_hashes = 0; int skipped_by_dry_run = 0; + int skipped_by_skip_failures = 0; int unsupported = 0; int cache_missing = 0; int cached = 0; diff --git a/include/vcpkg/fwd/build.h b/include/vcpkg/fwd/build.h index 5e3955e7..f66a0faf 100644 --- a/include/vcpkg/fwd/build.h +++ b/include/vcpkg/fwd/build.h @@ -9,9 +9,12 @@ namespace vcpkg PostBuildChecksFailed, FileConflicts, CascadedDueToMissingDependencies, + CascadedDueToSupports, + CascadedDueToBaseline, Skipped, SkippedByParentHashes, SkippedByDryRun, + SkippedBySkipFailures, Unsupported, CacheMissing, Cached, diff --git a/include/vcpkg/fwd/ci-baseline.h b/include/vcpkg/fwd/ci-baseline.h index 62f09b87..083d431d 100644 --- a/include/vcpkg/fwd/ci-baseline.h +++ b/include/vcpkg/fwd/ci-baseline.h @@ -13,9 +13,4 @@ namespace vcpkg Fail, Pass, }; - enum class SkipFailures : bool - { - No, - Yes, - }; } diff --git a/locales/messages.json b/locales/messages.json index 91aad651..64430ccd 100644 --- a/locales/messages.json +++ b/locales/messages.json @@ -233,8 +233,12 @@ "_BuildResultCacheMissing.comment": "Printed after the name of an installed entity to indicate that it was not present in the binary cache when the user has requested that things may only be installed from the cache rather than built.", "BuildResultCached": "CACHED", "_BuildResultCached.comment": "Printed after the name of an installed entity to indicate that it was not installed because it already existed in a binary cache.", + "BuildResultCascadeDueToBaseline": "CASCADED_DUE_TO_BASELINE", + "_BuildResultCascadeDueToBaseline.comment": "Printed after the name of an installed entity to indicate that it could not attempt to be installed because one of its transitive dependencies is skipped in ci.baseline.txt.", "BuildResultCascadeDueToMissingDependencies": "CASCADED_DUE_TO_MISSING_DEPENDENCIES", "_BuildResultCascadeDueToMissingDependencies.comment": "Printed after the name of an installed entity to indicate that it could not attempt to be installed because one of its transitive dependencies failed to install.", + "BuildResultCascadeDueToSupports": "CASCADED_DUE_TO_SUPPORTS", + "_BuildResultCascadeDueToSupports.comment": "Printed after the name of an installed entity to indicate that it could not attempt to be installed because one of its transitive dependencies' supports clause fails", "BuildResultDownloaded": "DOWNLOADED", "_BuildResultDownloaded.comment": "Printed after the name of an installed entity to indicate that it was successfully downloaded but no build or install was requested.", "BuildResultFileConflicts": "FILE_CONFLICTS", @@ -249,6 +253,8 @@ "_BuildResultSkippedByDryRun.comment": "Printed after the name of an entity that would be installed, but is not due to --dry-run.", "BuildResultSkippedByParentHashes": "SKIPPED_BY_PARENT_HASHES", "_BuildResultSkippedByParentHashes.comment": "Printed after the name of an installed entity to indicate that it isn't tested due to an ABI hash in --parent-hashes.", + "BuildResultSkippedBySkipFailures": "SKIPPED_BY_SKIP_FAILURES", + "_BuildResultSkippedBySkipFailures.comment": "Printed after the name of an entity that would be attempted to be installed, but is known to fail and vcpkg ci is configured in --skip-failures mode", "BuildResultSucceeded": "SUCCEEDED", "_BuildResultSucceeded.comment": "Printed after the name of an installed entity to indicate that it was built and installed successfully.", "BuildResultSummaryHeader": "SUMMARY FOR {triplet}", diff --git a/src/vcpkg-test/ci-baseline.cpp b/src/vcpkg-test/ci-baseline.cpp index 7cb26ffa..6a1ea775 100644 --- a/src/vcpkg-test/ci-baseline.cpp +++ b/src/vcpkg-test/ci-baseline.cpp @@ -208,7 +208,7 @@ bill-made-up-another-skip:x64-linux=skip)"; // note no trailing newline SkipsMap skips; skips.insert(x64_uwp, {}); // example triplet skips.insert(x64_linux, {}); // example host triplet - auto actual = parse_and_apply_ci_baseline(expected_from_example_input, skips, SkipFailures::No); + auto actual = parse_and_apply_ci_baseline(expected_from_example_input, skips); const SortedVector expected_expected_failures{ PackageSpec{"aubio", x64_uwp}, PackageSpec{"bde", x64_linux}, @@ -223,23 +223,10 @@ bill-made-up-another-skip:x64-linux=skip)"; // note no trailing newline PackageSpec{"casclib", x64_uwp}, }; - CHECK(actual.expected_failures == expected_expected_failures); + CHECK(actual.expected_fail == expected_expected_failures); CHECK(skips.triplets.size() == 2); CHECK(skips.triplets[0].skips == SortedVector{"catch-classic"}); CHECK(skips.triplets[1].skips == SortedVector{"catch-classic", "bill-made-up-another-skip"}); - - skips.triplets[0].skips.clear(); - skips.triplets[1].skips.clear(); - - actual = parse_and_apply_ci_baseline(expected_from_example_input, skips, SkipFailures::Yes); - CHECK(actual.expected_failures == expected_expected_failures); - CHECK(skips.triplets.size() == 2); - CHECK(skips.triplets[0].skips == - SortedVector{ - "aubio", "blitz", "blosc", "bond", "botan", "c-ares", "caf", "casclib", "catch-classic"}); - CHECK(skips.triplets[1].skips == - SortedVector{ - "bde", "buck-yeh-bux", "buck-yeh-bux-mariadb-client", "catch-classic", "bill-made-up-another-skip"}); } } @@ -319,15 +306,22 @@ TEST_CASE ("Parse Errors", "[ci-baseline]") TEST_CASE ("format_ci_result 1", "[ci-baseline]") { CiBaselineData cidata; - cidata.expected_failures = { + cidata.expected_fail = { PackageSpec{"fail", Test::X64_UWP}, }; - cidata.required_success = { + cidata.required_pass = { PackageSpec{"pass", Test::X64_UWP}, }; constexpr const char failmsg[] = "REGRESSION: {0} failed with BUILD_FAILED. If expected, add {0}=fail to cifile."; + constexpr const char postbuildmsg[] = + "REGRESSION: {0} failed with POST_BUILD_CHECKS_FAILED. If expected, add {0}=fail to cifile."; + constexpr const char fileconflictsmsg[] = + "REGRESSION: {0} failed with FILE_CONFLICTS. If expected, add {0}=fail to cifile."; + constexpr const char regression_no_path_msg[] = "REGRESSION: {0} failed with {1}."; constexpr const char cascademsg[] = "REGRESSION: {0} cascaded, but it is required to pass. (cifile)."; constexpr const char passmsg[] = "PASSING, REMOVE FROM FAIL LIST: {0} (cifile)."; + constexpr const char unsupported_failmsg[] = "REGRESSION: {0} is marked as fail but not supported for x64-uwp."; + constexpr const char unsupported_passmsg[] = "REGRESSION: {0} is marked as pass but not supported for x64-uwp."; std::string cifile = "cifile"; SECTION ("SUCCEEDED") { @@ -343,6 +337,20 @@ TEST_CASE ("format_ci_result 1", "[ci-baseline]") CHECK(test({"neither", Test::X64_UWP}, false) == ""); } + SECTION ("CACHED") + { + const auto test = [&](PackageSpec s, bool allow_unexpected_passing) { + return format_ci_result(s, BuildResult::Cached, cidata, &cifile, allow_unexpected_passing); + }; + CHECK(test({"pass", Test::X64_UWP}, true) == ""); + CHECK(test({"pass", Test::X64_UWP}, false) == ""); + CHECK(test({"fail", Test::X64_UWP}, true) == ""); + CHECK(test({"fail", Test::X64_UWP}, false) == fmt::format(passmsg, "fail:x64-uwp")); + CHECK(test({"fail", Test::ARM_UWP}, false) == ""); + CHECK(test({"neither", Test::X64_UWP}, true) == ""); + CHECK(test({"neither", Test::X64_UWP}, false) == ""); + } + SECTION ("BUILD_FAILED") { const auto test = [&](PackageSpec s) { @@ -353,14 +361,91 @@ TEST_CASE ("format_ci_result 1", "[ci-baseline]") CHECK(test({"neither", Test::X64_UWP}) == fmt::format(failmsg, "neither:x64-uwp")); } - SECTION ("CASCADED_DUE_TO_MISSING_DEPENDENCIES") + SECTION ("POST_BUILD_CHECKS_FAILED") { const auto test = [&](PackageSpec s) { - return format_ci_result(s, BuildResult::CascadedDueToMissingDependencies, cidata, &cifile, false); + return format_ci_result(s, BuildResult::PostBuildChecksFailed, cidata, &cifile, false); + }; + CHECK(test({"pass", Test::X64_UWP}) == fmt::format(postbuildmsg, "pass:x64-uwp")); + CHECK(test({"fail", Test::X64_UWP}) == ""); + CHECK(test({"neither", Test::X64_UWP}) == fmt::format(postbuildmsg, "neither:x64-uwp")); + } + + SECTION ("FILE_CONFLICTS") + { + const auto test = [&](PackageSpec s) { + return format_ci_result(s, BuildResult::FileConflicts, cidata, &cifile, false); + }; + CHECK(test({"pass", Test::X64_UWP}) == fmt::format(fileconflictsmsg, "pass:x64-uwp")); + CHECK(test({"fail", Test::X64_UWP}) == ""); + CHECK(test({"neither", Test::X64_UWP}) == fmt::format(fileconflictsmsg, "neither:x64-uwp")); + } + + SECTION ("BUILD_FAILED WITHOUT CI FILE") + { + const auto test = [&](PackageSpec s, BuildResult result) { + return format_ci_result(s, result, cidata, nullptr, false); + }; + CHECK(test({"pass", Test::X64_UWP}, BuildResult::BuildFailed) == + fmt::format(regression_no_path_msg, "pass:x64-uwp", "BUILD_FAILED")); + CHECK(test({"pass", Test::X64_UWP}, BuildResult::PostBuildChecksFailed) == + fmt::format(regression_no_path_msg, "pass:x64-uwp", "POST_BUILD_CHECKS_FAILED")); + CHECK(test({"pass", Test::X64_UWP}, BuildResult::FileConflicts) == + fmt::format(regression_no_path_msg, "pass:x64-uwp", "FILE_CONFLICTS")); + } + + SECTION ("CASCADED_DUE_TO_SUPPORTS") + { + const auto test = [&](PackageSpec s) { + return format_ci_result(s, BuildResult::CascadedDueToSupports, cidata, &cifile, false); }; CHECK(test({"pass", Test::X64_UWP}) == fmt::format(cascademsg, "pass:x64-uwp")); CHECK(test({"fail", Test::X64_UWP}) == "REGRESSION: fail:x64-uwp is marked as fail but one dependency is not supported for x64-uwp."); CHECK(test({"neither", Test::X64_UWP}) == ""); } + + SECTION ("CASCADED_DUE_TO_MISSING_DEPENDENCIES") + { + const auto test = [&](PackageSpec s) { + return format_ci_result(s, BuildResult::CascadedDueToMissingDependencies, cidata, &cifile, false); + }; + CHECK(test({"pass", Test::X64_UWP}) == fmt::format(cascademsg, "pass:x64-uwp")); + CHECK(test({"fail", Test::X64_UWP}) == ""); + CHECK(test({"neither", Test::X64_UWP}) == ""); + } + + SECTION ("CASCADED_DUE_TO_BASELINE") + { + const auto test = [&](PackageSpec s) { + return format_ci_result(s, BuildResult::CascadedDueToBaseline, cidata, &cifile, false); + }; + CHECK(test({"pass", Test::X64_UWP}) == fmt::format(cascademsg, "pass:x64-uwp")); + CHECK(test({"fail", Test::X64_UWP}) == ""); + CHECK(test({"neither", Test::X64_UWP}) == ""); + } + + SECTION ("UNSUPPORTED") + { + const auto test = [&](PackageSpec s) { + return format_ci_result(s, BuildResult::Unsupported, cidata, &cifile, false); + }; + CHECK(test({"pass", Test::X64_UWP}) == fmt::format(unsupported_passmsg, "pass:x64-uwp")); + CHECK(test({"fail", Test::X64_UWP}) == fmt::format(unsupported_failmsg, "fail:x64-uwp")); + CHECK(test({"neither", Test::X64_UWP}) == ""); + } + + SECTION ("SKIPPED") + { + const auto test = [&](PackageSpec s, BuildResult result) { + return format_ci_result(s, result, cidata, &cifile, false); + }; + CHECK(test({"pass", Test::X64_UWP}, BuildResult::Skipped) == ""); + CHECK(test({"fail", Test::X64_UWP}, BuildResult::SkippedByParentHashes) == ""); + CHECK(test({"neither", Test::X64_UWP}, BuildResult::SkippedByDryRun) == ""); + CHECK(test({"pass", Test::X64_UWP}, BuildResult::SkippedBySkipFailures) == ""); + } + + // CacheMissing, Downloaded, and Removed are semantically invalid here; format_ci_result only accepts CI result + // states that participate in baseline evaluation, and these values intentionally hit unreachable. } diff --git a/src/vcpkg/ci-baseline.cpp b/src/vcpkg/ci-baseline.cpp index ce627679..a337249c 100644 --- a/src/vcpkg/ci-baseline.cpp +++ b/src/vcpkg/ci-baseline.cpp @@ -138,12 +138,10 @@ namespace vcpkg return result; } - CiBaselineData parse_and_apply_ci_baseline(View lines, - SkipsMap& skips_map, - SkipFailures skip_failures) + CiBaselineData parse_and_apply_ci_baseline(View lines, SkipsMap& skips_map) { - std::vector expected_failures; - std::vector required_success; + std::vector expected_fail; + std::vector required_pass; std::map> added_skips; for (const auto& triplet_entry : skips_map.triplets) { @@ -155,21 +153,13 @@ namespace vcpkg auto triplet_match = added_skips.find(line.triplet); if (triplet_match != added_skips.end()) { - if (line.state == CiBaselineState::Pass) + switch (line.state) { - required_success.emplace_back(line.port_name, line.triplet); - continue; + case CiBaselineState::Pass: required_pass.emplace_back(line.port_name, line.triplet); break; + case CiBaselineState::Fail: expected_fail.emplace_back(line.port_name, line.triplet); break; + case CiBaselineState::Skip: triplet_match->second.push_back(line.port_name); break; + default: Checks::unreachable(VCPKG_LINE_INFO); } - if (line.state == CiBaselineState::Fail) - { - expected_failures.emplace_back(line.port_name, line.triplet); - if (skip_failures == SkipFailures::No) - { - continue; - } - } - - triplet_match->second.push_back(line.port_name); } } @@ -180,37 +170,37 @@ namespace vcpkg } return CiBaselineData{ - SortedVector(std::move(expected_failures)), - SortedVector(std::move(required_success)), + SortedVector(std::move(expected_fail)), + SortedVector(std::move(required_pass)), }; } LocalizedString format_ci_result(const PackageSpec& spec, BuildResult result, - const CiBaselineData& cidata, - const std::string* cifile, + const CiBaselineData& ci_data, + const std::string* ci_file, bool allow_unexpected_passing) { switch (result) { case BuildResult::Succeeded: case BuildResult::Cached: - if (!allow_unexpected_passing && cidata.expected_failures.contains(spec)) + if (!allow_unexpected_passing && ci_data.expected_fail.contains(spec)) { - return msg::format(msgCiBaselineUnexpectedPass, msg::spec = spec, msg::path = *cifile); + return msg::format(msgCiBaselineUnexpectedPass, msg::spec = spec, msg::path = *ci_file); } break; case BuildResult::BuildFailed: case BuildResult::PostBuildChecksFailed: case BuildResult::FileConflicts: - if (!cidata.expected_failures.contains(spec)) + if (!ci_data.expected_fail.contains(spec)) { - if (cifile) + if (ci_file) { return msg::format(msgCiBaselineRegression, msg::spec = spec, msg::build_result = to_string_locale_invariant(result), - msg::path = *cifile); + msg::path = *ci_file); } return msg::format(msgCiBaselineRegressionNoPath, @@ -218,25 +208,28 @@ namespace vcpkg msg::build_result = to_string_locale_invariant(result)); } break; - case BuildResult::CascadedDueToMissingDependencies: - if (cidata.expected_failures.contains(spec)) + case BuildResult::CascadedDueToSupports: + if (ci_data.expected_fail.contains(spec)) { return msg::format( msgCiBaselineUnexpectedFailCascade, msg::spec = spec, msg::triplet = spec.triplet()); } - if (cidata.required_success.contains(spec)) + [[fallthrough]]; + case BuildResult::CascadedDueToMissingDependencies: + case BuildResult::CascadedDueToBaseline: + if (ci_data.required_pass.contains(spec)) { - return msg::format(msgCiBaselineDisallowedCascade, msg::spec = spec, msg::path = *cifile); + return msg::format(msgCiBaselineDisallowedCascade, msg::spec = spec, msg::path = *ci_file); } break; case BuildResult::Unsupported: - if (cidata.expected_failures.contains(spec)) + if (ci_data.expected_fail.contains(spec)) { return msg::format(msgCiBaselineUnexpectedFail, msg::spec = spec, msg::triplet = spec.triplet()); } - if (cidata.required_success.contains(spec)) + if (ci_data.required_pass.contains(spec)) { return msg::format( msgCiBaselineUnexpectedPassUnsupported, msg::spec = spec, msg::triplet = spec.triplet()); @@ -244,7 +237,8 @@ namespace vcpkg break; case BuildResult::Skipped: case BuildResult::SkippedByParentHashes: - case BuildResult::SkippedByDryRun: break; + case BuildResult::SkippedByDryRun: + case BuildResult::SkippedBySkipFailures: break; case BuildResult::CacheMissing: case BuildResult::Downloaded: case BuildResult::Removed: diff --git a/src/vcpkg/commands.build.cpp b/src/vcpkg/commands.build.cpp index d2b693d4..00b7a3ea 100644 --- a/src/vcpkg/commands.build.cpp +++ b/src/vcpkg/commands.build.cpp @@ -336,8 +336,11 @@ namespace vcpkg case BuildResult::Skipped: case BuildResult::SkippedByParentHashes: case BuildResult::SkippedByDryRun: + case BuildResult::SkippedBySkipFailures: case BuildResult::Unsupported: case BuildResult::Cached: + case BuildResult::CascadedDueToSupports: + case BuildResult::CascadedDueToBaseline: default: Checks::unreachable(VCPKG_LINE_INFO); } } @@ -1685,9 +1688,12 @@ namespace vcpkg case BuildResult::PostBuildChecksFailed: ++post_build_checks_failed; return; case BuildResult::FileConflicts: ++file_conflicts; return; case BuildResult::CascadedDueToMissingDependencies: ++cascaded_due_to_missing_dependencies; return; + case BuildResult::CascadedDueToSupports: ++cascaded_due_to_supports; return; + case BuildResult::CascadedDueToBaseline: ++cascaded_due_to_baseline; return; case BuildResult::Skipped: ++skipped; return; case BuildResult::SkippedByParentHashes: ++skipped_by_parent_hashes; return; case BuildResult::SkippedByDryRun: ++skipped_by_dry_run; return; + case BuildResult::SkippedBySkipFailures: ++skipped_by_skip_failures; return; case BuildResult::Unsupported: ++unsupported; return; case BuildResult::CacheMissing: ++cache_missing; return; case BuildResult::Cached: ++cached; return; @@ -1720,9 +1726,12 @@ namespace vcpkg append_build_result_summary_line(msgBuildResultFileConflicts, file_conflicts, str); append_build_result_summary_line( msgBuildResultCascadeDueToMissingDependencies, cascaded_due_to_missing_dependencies, str); + append_build_result_summary_line(msgBuildResultCascadeDueToSupports, cascaded_due_to_supports, str); + append_build_result_summary_line(msgBuildResultCascadeDueToBaseline, cascaded_due_to_baseline, str); append_build_result_summary_line(msgBuildResultSkipped, skipped, str); append_build_result_summary_line(msgBuildResultSkippedByParentHashes, skipped_by_parent_hashes, str); append_build_result_summary_line(msgBuildResultSkippedByDryRun, skipped_by_dry_run, str); + append_build_result_summary_line(msgBuildResultSkippedBySkipFailures, skipped_by_skip_failures, str); append_build_result_summary_line(msgBuildResultUnsupported, unsupported, str); append_build_result_summary_line(msgBuildResultCacheMissing, cache_missing, str); append_build_result_summary_line(msgBuildResultCached, cached, str); @@ -1740,9 +1749,12 @@ namespace vcpkg case BuildResult::PostBuildChecksFailed: return "POST_BUILD_CHECKS_FAILED"; case BuildResult::FileConflicts: return "FILE_CONFLICTS"; case BuildResult::CascadedDueToMissingDependencies: return "CASCADED_DUE_TO_MISSING_DEPENDENCIES"; + case BuildResult::CascadedDueToSupports: return "CASCADED_DUE_TO_SUPPORTS"; + case BuildResult::CascadedDueToBaseline: return "CASCADED_DUE_TO_BASELINE"; case BuildResult::Skipped: return "SKIPPED"; case BuildResult::SkippedByParentHashes: return "SKIPPED_BY_PARENT_HASHES"; case BuildResult::SkippedByDryRun: return "SKIPPED_BY_DRY_RUN"; + case BuildResult::SkippedBySkipFailures: return "SKIPPED_BY_SKIP_FAILURES"; case BuildResult::Unsupported: return "UNSUPPORTED"; case BuildResult::CacheMissing: return "CACHE_MISSING"; case BuildResult::Cached: return "CACHED"; @@ -1762,9 +1774,12 @@ namespace vcpkg case BuildResult::FileConflicts: return msg::format(msgBuildResultFileConflicts); case BuildResult::CascadedDueToMissingDependencies: return msg::format(msgBuildResultCascadeDueToMissingDependencies); + case BuildResult::CascadedDueToSupports: return msg::format(msgBuildResultCascadeDueToSupports); + case BuildResult::CascadedDueToBaseline: return msg::format(msgBuildResultCascadeDueToBaseline); case BuildResult::Skipped: return msg::format(msgBuildResultSkipped); case BuildResult::SkippedByParentHashes: return msg::format(msgBuildResultSkippedByParentHashes); case BuildResult::SkippedByDryRun: return msg::format(msgBuildResultSkippedByDryRun); + case BuildResult::SkippedBySkipFailures: return msg::format(msgBuildResultSkippedBySkipFailures); case BuildResult::Unsupported: return msg::format(msgBuildResultUnsupported); case BuildResult::CacheMissing: return msg::format(msgBuildResultCacheMissing); case BuildResult::Cached: return msg::format(msgBuildResultCached); diff --git a/src/vcpkg/commands.ci.cpp b/src/vcpkg/commands.ci.cpp index 8ea188d0..ca5f9966 100644 --- a/src/vcpkg/commands.ci.cpp +++ b/src/vcpkg/commands.ci.cpp @@ -33,6 +33,12 @@ using namespace vcpkg; namespace { + enum class SkipFailures : bool + { + No, + Yes, + }; + constexpr CommandSetting CI_SETTINGS[] = { {SwitchXXUnit, msgCISettingsOptXUnit}, {SwitchCIBaseline, msgCISettingsOptCIBase}, @@ -60,8 +66,9 @@ namespace enum class SkipReason { Baseline, + BaselineCascade, Supports, - Cascade + SupportsCascade }; struct CiSpecsResult @@ -115,6 +122,7 @@ namespace const std::vector& precheck_results, const std::unordered_set& known_failure_abis, const std::unordered_set& parent_hashes, + const SortedVector& skipped_failures, const ActionPlan& action_plan) { static constexpr StringLiteral STATE_ABI_FAIL = "fail"; @@ -123,6 +131,7 @@ namespace static constexpr StringLiteral STATE_PARENT = "parent"; static constexpr StringLiteral STATE_UNKNOWN = "*"; static constexpr StringLiteral STATE_SKIP = "skip"; + static constexpr StringLiteral STATE_SKIP_FAILURE = "s-fail"; static constexpr StringLiteral STATE_CASCADE = "cascade"; CIPreBuildStatus ret; @@ -139,23 +148,34 @@ namespace const std::string& public_abi = action.package_abi_or_exit(VCPKG_LINE_INFO); const StringLiteral* state; BuildResult known_result; + // check in order from most known to least known information if (Util::Sets::contains(known_failure_abis, public_abi)) { + // positively known broken state = &STATE_ABI_FAIL; known_result = BuildResult::BuildFailed; } else if (precheck_results[action_idx] == CacheAvailability::available) { + // positively known not broken state = &STATE_CACHED; known_result = BuildResult::Cached; } + else if (skipped_failures.contains(action.spec)) + { + // positively skipped due to expected failure and --skip-failures + state = &STATE_SKIP_FAILURE; + known_result = BuildResult::SkippedBySkipFailures; + } else if (Util::Sets::contains(parent_hashes, public_abi)) { + // likely to work but positively skipped by --parent-hashes state = &STATE_PARENT; known_result = BuildResult::SkippedByParentHashes; } else { + // we know nothing state = &STATE_UNKNOWN; known_result = BuildResult::SkippedByDryRun; } @@ -189,19 +209,21 @@ namespace BuildResult known_result; switch (skip.second) { - // it probably makes sense to distinguish between "=skip" and "=fail but --skip-failures" - // but we don't preserve that information right now, so all these cases report as "skip" case SkipReason::Baseline: state = &STATE_SKIP; known_result = BuildResult::Skipped; break; + case SkipReason::BaselineCascade: + state = &STATE_CASCADE; + known_result = BuildResult::CascadedDueToBaseline; + break; case SkipReason::Supports: state = &STATE_UNSUPPORTED; known_result = BuildResult::Unsupported; break; - case SkipReason::Cascade: + case SkipReason::SupportsCascade: state = &STATE_CASCADE; - known_result = BuildResult::CascadedDueToMissingDependencies; + known_result = BuildResult::CascadedDueToSupports; break; default: Checks::unreachable(VCPKG_LINE_INFO); } @@ -225,19 +247,50 @@ namespace Checks::unreachable(VCPKG_LINE_INFO); } - if (it_known->second != BuildResult::SkippedByParentHashes) + switch (it_known->second) { - it->request_type = RequestType::USER_REQUESTED; - if (it_known->second == BuildResult::SkippedByDryRun) - { + case BuildResult::BuildFailed: + // expected to fail but we're going to try to build it anyway to see if we get PASSING REMOVE FROM + // FAIL LIST + case BuildResult::SkippedByDryRun: + // "excluded by dry run" is the default state where we need to try to build that action in order to + // see what it means to_keep.insert(it->spec); - } - } - - if (Util::Sets::contains(to_keep, it->spec) && it_known->second != BuildResult::Skipped && - it_known->second != BuildResult::Unsupported) - { - to_keep.insert(it->package_dependencies.begin(), it->package_dependencies.end()); + to_keep.insert(it->package_dependencies.begin(), it->package_dependencies.end()); + break; + case BuildResult::Skipped: + case BuildResult::CascadedDueToSupports: + case BuildResult::CascadedDueToBaseline: + case BuildResult::Unsupported: + if (Util::Sets::contains(to_keep, it->spec)) + { + Checks::unreachable(VCPKG_LINE_INFO, + "A spec is in the action plan that should have been skipped earlier; bug " + "in calculate_ci_specs ?"); + } + break; + case BuildResult::SkippedByParentHashes: + // parent hashes means that we want to avoid trying to build it but if it's a transitive dependency + // that will be built it's fair game + case BuildResult::SkippedBySkipFailures: + // a skipped failure means we don't want to try to build it but it may be transitive + case BuildResult::Cached: + // if it's already in the binary cache we don't want to attempt to restore it unless it's + // transitively depended upon + if (Util::Sets::contains(to_keep, it->spec)) + { + to_keep.insert(it->package_dependencies.begin(), it->package_dependencies.end()); + } + break; + case BuildResult::Succeeded: + case BuildResult::CascadedDueToMissingDependencies: + case BuildResult::PostBuildChecksFailed: + case BuildResult::FileConflicts: + case BuildResult::Downloaded: + case BuildResult::Removed: + case BuildResult::CacheMissing: + // these cases all only happen after a install/removal occurs + default: Checks::unreachable(VCPKG_LINE_INFO); } } @@ -316,15 +369,11 @@ namespace auto full_package_spec = FullPackageSpec{PackageSpec{scfl->to_name(), target_triplet}, InternalFeatureSet{FeatureNameCore.to_string(), FeatureNameDefault.to_string()}}; - if (target_triplet_skips && target_triplet_skips->contains(scfl->to_name())) - { - result.skipped.insert_or_assign(std::move(full_package_spec.package_spec), SkipReason::Baseline); - continue; - } - PackagesDirAssigner this_packages_dir_not_used{""}; const ActionPlan action_plan = create_feature_install_plan( provider, var_provider, {&full_package_spec, 1}, {}, this_packages_dir_not_used, serialize_options); + // check "supports" before the baseline because the caller can check for baseline consistency later but + // can't check consistency with "supports" because we don't persist the plan if (!action_plan.unsupported_features.empty()) { result.skipped.insert_or_assign( @@ -334,17 +383,22 @@ namespace return p.first.spec().name() == full_package_spec.package_spec.name(); }) ? SkipReason::Supports - : SkipReason::Cascade); + : SkipReason::SupportsCascade); continue; } + if (target_triplet_skips && target_triplet_skips->contains(scfl->to_name())) + { + result.skipped.insert_or_assign(std::move(full_package_spec.package_spec), SkipReason::Baseline); + continue; + } if (cascade_for_triplet(action_plan.install_actions, target_triplet, target_triplet_skips, host_triplet, host_triplet_skips)) { - result.skipped.insert_or_assign(std::move(full_package_spec.package_spec), SkipReason::Cascade); + result.skipped.insert_or_assign(std::move(full_package_spec.package_spec), SkipReason::BaselineCascade); continue; } @@ -437,6 +491,8 @@ namespace vcpkg const std::string* ci_baseline_file_name = nullptr; const bool allow_unexpected_passing = Util::Sets::contains(options.switches, SwitchAllowUnexpectedPassing); CiBaselineData baseline_data; + auto skip_failures = + Util::Sets::contains(options.switches, SwitchSkipFailures) ? SkipFailures::Yes : SkipFailures::No; if (baseline_iter == settings.end()) { if (allow_unexpected_passing) @@ -446,14 +502,12 @@ namespace vcpkg } else { - auto skip_failures = - Util::Sets::contains(options.switches, SwitchSkipFailures) ? SkipFailures::Yes : SkipFailures::No; ci_baseline_file_name = &baseline_iter->second; const auto ci_baseline_file_contents = fs.read_contents(*ci_baseline_file_name, VCPKG_LINE_INFO); ParseMessages ci_parse_messages; const auto lines = parse_ci_baseline(ci_baseline_file_contents, *ci_baseline_file_name, ci_parse_messages); ci_parse_messages.exit_if_errors_or_warnings(); - baseline_data = parse_and_apply_ci_baseline(lines, skips_map, skip_failures); + baseline_data = parse_and_apply_ci_baseline(lines, skips_map); } std::unordered_set known_failure_abis; @@ -511,8 +565,11 @@ namespace vcpkg auto install_actions = Util::fmap(action_plan.install_actions, [](const InstallPlanAction& action) { return &action; }); const auto precheck_results = binary_cache.precheck(console_diagnostic_context, fs, install_actions); - const auto pre_build_status = - compute_pre_build_statuses(ci_specs, precheck_results, known_failure_abis, parent_hashes, action_plan); + SortedVector empty_skipped_failures; + const SortedVector& skipped_failures = + skip_failures == SkipFailures::Yes ? baseline_data.expected_fail : empty_skipped_failures; + const auto pre_build_status = compute_pre_build_statuses( + ci_specs, precheck_results, known_failure_abis, parent_hashes, skipped_failures, action_plan); { std::string msg; for (auto&& line : pre_build_status.report_lines) diff --git a/src/vcpkg/commands.install.cpp b/src/vcpkg/commands.install.cpp index 12da7a9c..79c7f27b 100644 --- a/src/vcpkg/commands.install.cpp +++ b/src/vcpkg/commands.install.cpp @@ -726,10 +726,13 @@ namespace vcpkg case BuildResult::Skipped: case BuildResult::SkippedByParentHashes: case BuildResult::SkippedByDryRun: + case BuildResult::SkippedBySkipFailures: case BuildResult::Cached: break; case BuildResult::BuildFailed: case BuildResult::PostBuildChecksFailed: case BuildResult::FileConflicts: + case BuildResult::CascadedDueToSupports: + case BuildResult::CascadedDueToBaseline: case BuildResult::CascadedDueToMissingDependencies: case BuildResult::Unsupported: case BuildResult::CacheMissing: summary.failed = true; break; diff --git a/src/vcpkg/commands.test-features.cpp b/src/vcpkg/commands.test-features.cpp index b01a52c8..3d013b27 100644 --- a/src/vcpkg/commands.test-features.cpp +++ b/src/vcpkg/commands.test-features.cpp @@ -942,6 +942,9 @@ namespace vcpkg case BuildResult::Skipped: case BuildResult::SkippedByParentHashes: case BuildResult::SkippedByDryRun: + case BuildResult::SkippedBySkipFailures: + case BuildResult::CascadedDueToSupports: + case BuildResult::CascadedDueToBaseline: case BuildResult::Cached: default: Checks::unreachable(VCPKG_LINE_INFO); } diff --git a/src/vcpkg/xunitwriter.cpp b/src/vcpkg/xunitwriter.cpp index bca83084..7a17807c 100644 --- a/src/vcpkg/xunitwriter.cpp +++ b/src/vcpkg/xunitwriter.cpp @@ -31,6 +31,9 @@ namespace case BuildResult::Skipped: case BuildResult::SkippedByParentHashes: case BuildResult::SkippedByDryRun: + case BuildResult::SkippedBySkipFailures: + case BuildResult::CascadedDueToSupports: + case BuildResult::CascadedDueToBaseline: case BuildResult::Unsupported: case BuildResult::Cached: result_string = "Skip"; break; case BuildResult::CacheMissing: