修复 Studio 失败测试并新增合并门禁
把丢失的 box 录像放回 apps/studio/tests/data/，修掉第 3 节 3–10 号与两处过时测试桩， 注册 timing 标记并让根目录收集零错误；新增 tools/gate.ps1 与可选的 pre-push 钩子。 验证：studio 524 通过 19 跳过、shared 467 通过 18 跳过、ui_check 632 项 0 失败、gate.ps1 退出码 0。 细节见 docs/test-repair-plan.zh.md。
This commit is contained in:
ykcheungab
+297
@@ -0,0 +1,297 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Merge gate for orisys-app: lint the change, run both test suites, run the UI check.
|
||||
|
||||
.DESCRIPTION
|
||||
One command that a change cannot skip. It runs, in order:
|
||||
|
||||
1. tools/lint_gate.py the narrow lint rule set over the changed files
|
||||
2. pytest apps/studio/tests full, with the ``timing`` tests deselected
|
||||
3. pytest shared/tests full, with the ``timing`` tests deselected
|
||||
4. apps/studio/tools/ui_check.py the layout / theme / record-channel checks
|
||||
|
||||
and then holds the run to three contracts:
|
||||
|
||||
* tools/known_failures.txt must be empty. Any content fails the gate: a known
|
||||
failure list is the thing this gate exists to make unnecessary.
|
||||
* the skip count per suite must stay under tools/skip_budget.txt. A skip that
|
||||
grows past the budget is a suite that stopped testing what it claims.
|
||||
* the collected test count per suite must stay above the floor in the same file.
|
||||
A module that becomes uncollectable silently shrinks the suite; the floor is
|
||||
how that shows up as a failure instead of a faster green run.
|
||||
|
||||
Every command's output goes to <repo>/.scratch/<Task>/gate-<run-id>/, and the
|
||||
one-page summary is both printed and written there as summary.md.
|
||||
|
||||
Platform: **Windows**. The suite renders real Qt widgets offscreen and two tests
|
||||
are platform-sensitive (see docs/test-repair-plan.zh.md §3 #11 and #13); Linux
|
||||
numbers are not equivalent.
|
||||
|
||||
.PARAMETER Task
|
||||
Scratch task id, i.e. the <task-id> in .scratch/<task-id>/. Defaults to "gate".
|
||||
|
||||
.PARAMETER Python
|
||||
The interpreter to run the suites with. Defaults to .venv-build/Scripts/python.exe.
|
||||
|
||||
.PARAMETER SkipLint
|
||||
Run only the suites and the UI check (used when diagnosing a red gate).
|
||||
|
||||
.EXAMPLE
|
||||
powershell -NoProfile -ExecutionPolicy Bypass -File tools/gate.ps1
|
||||
|
||||
.NOTES
|
||||
Exit codes: 0 gate passed, 1 gate failed, 2 environment or configuration problem.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$Task = 'gate',
|
||||
[string]$Python = '',
|
||||
[switch]$SkipLint
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$Repo = Split-Path -Parent $PSScriptRoot
|
||||
Set-Location $Repo
|
||||
|
||||
function Write-Step([string]$Text) {
|
||||
Write-Host ''
|
||||
Write-Host "== $Text" -ForegroundColor Cyan
|
||||
}
|
||||
|
||||
function Invoke-Native([string]$Exe, [string[]]$Arguments, [string]$LogPath) {
|
||||
# Every tool here writes progress and Qt diagnostics to stderr, and Windows
|
||||
# PowerShell turns a native command's stderr into error records: with
|
||||
# ``$ErrorActionPreference = 'Stop'`` a Qt warning ("This plugin does not support
|
||||
# raise()") aborted the whole gate. The exit code is the signal; stderr is data.
|
||||
$previous = $ErrorActionPreference
|
||||
$ErrorActionPreference = 'Continue'
|
||||
try {
|
||||
& $Exe @Arguments 2>&1 | Tee-Object -FilePath $LogPath | Out-Host
|
||||
return $LASTEXITCODE
|
||||
} finally {
|
||||
$ErrorActionPreference = $previous
|
||||
}
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ environment
|
||||
if ([string]::IsNullOrWhiteSpace($Python)) {
|
||||
$Python = Join-Path $Repo '.venv-build\Scripts\python.exe'
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $Python)) {
|
||||
Write-Host "gate: no interpreter at $Python (set -Python or build .venv-build)" -ForegroundColor Red
|
||||
exit 2
|
||||
}
|
||||
|
||||
$RunId = Get-Date -Format 'yyyyMMddTHHmmssZ'
|
||||
$TaskDir = Join-Path $Repo (Join-Path '.scratch' $Task)
|
||||
$Scratch = Join-Path $TaskDir "gate-$RunId"
|
||||
New-Item -ItemType Directory -Force -Path $Scratch | Out-Null
|
||||
# AGENTS.md rule 1: the scratch directory is marked while a task is using it.
|
||||
Set-Content -LiteralPath (Join-Path $TaskDir '.active') -Value "gate $RunId"
|
||||
|
||||
$KnownFailures = Join-Path $Repo 'tools\known_failures.txt'
|
||||
$BudgetFile = Join-Path $Repo 'tools\skip_budget.txt'
|
||||
|
||||
$Results = New-Object System.Collections.ArrayList
|
||||
function Add-Result([string]$Name, [string]$Status, [string]$Detail) {
|
||||
[void]$Results.Add([pscustomobject]@{ Name = $Name; Status = $Status; Detail = $Detail })
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ budgets
|
||||
function Read-Budget([string]$Path) {
|
||||
$values = @{}
|
||||
if (-not (Test-Path -LiteralPath $Path)) {
|
||||
return $values
|
||||
}
|
||||
foreach ($line in Get-Content -LiteralPath $Path) {
|
||||
$text = $line.Trim()
|
||||
if ($text -eq '' -or $text.StartsWith('#')) { continue }
|
||||
if ($text -notmatch '^([A-Za-z_]+)\s*=\s*(\d+)$') {
|
||||
Write-Host "gate: unreadable line in ${Path}: $text" -ForegroundColor Red
|
||||
throw "bad budget line: $text"
|
||||
}
|
||||
$values[$Matches[1]] = [int]$Matches[2]
|
||||
}
|
||||
return $values
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ known failures
|
||||
Write-Step 'known failures'
|
||||
$KnownStatus = 'passed'
|
||||
$KnownDetail = 'tools/known_failures.txt is empty'
|
||||
if (-not (Test-Path -LiteralPath $KnownFailures)) {
|
||||
$KnownStatus = 'failed'
|
||||
$KnownDetail = 'tools/known_failures.txt is missing'
|
||||
} else {
|
||||
$known = (Get-Content -LiteralPath $KnownFailures -Raw)
|
||||
if ($null -eq $known) { $known = '' }
|
||||
if ($known.Trim() -ne '') {
|
||||
$KnownStatus = 'failed'
|
||||
$KnownDetail = 'tools/known_failures.txt is not empty: ' + ($known.Trim() -replace '\r?\n', ' | ')
|
||||
}
|
||||
}
|
||||
Add-Result 'known failures' $KnownStatus $KnownDetail
|
||||
if ($KnownStatus -ne 'passed') {
|
||||
Write-Host $KnownDetail -ForegroundColor Red
|
||||
} else {
|
||||
Write-Host $KnownDetail
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ lint
|
||||
if (-not $SkipLint) {
|
||||
Write-Step 'lint (changed files)'
|
||||
$lintLog = Join-Path $Scratch 'lint.log'
|
||||
$lintExit = Invoke-Native $Python @((Join-Path $Repo 'tools\lint_gate.py')) $lintLog
|
||||
if ($lintExit -eq 0) {
|
||||
Add-Result 'lint' 'passed' 'no findings'
|
||||
} elseif ($lintExit -eq 1) {
|
||||
Add-Result 'lint' 'failed' 'findings in the changed files (see lint.log)'
|
||||
} else {
|
||||
Add-Result 'lint' 'failed' "lint_gate.py exited $lintExit (environment problem, e.g. ruff missing)"
|
||||
}
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ test suites
|
||||
function Invoke-Suite([string]$Name, [string]$Path) {
|
||||
$safe = $Name.Replace(' ', '-')
|
||||
$log = Join-Path $Scratch "$safe.log"
|
||||
$xml = Join-Path $Scratch "$safe.xml"
|
||||
$base = Join-Path $Scratch "pytest-$safe"
|
||||
Write-Step "$Name"
|
||||
$exit = Invoke-Native $Python @(
|
||||
'-m', 'pytest', $Path,
|
||||
'-m', 'not timing',
|
||||
'-p', 'no:cacheprovider',
|
||||
"--basetemp=$base",
|
||||
"--junitxml=$xml"
|
||||
) $log
|
||||
|
||||
$counts = [pscustomobject]@{
|
||||
Tests = 0; Failures = 0; Errors = 0; Skipped = 0; Exit = $exit; Xml = (Test-Path -LiteralPath $xml)
|
||||
}
|
||||
if ($counts.Xml) {
|
||||
try {
|
||||
[xml]$doc = Get-Content -LiteralPath $xml
|
||||
$suites = @($doc.testsuites.testsuite)
|
||||
foreach ($suite in $suites) {
|
||||
$counts.Tests += [int]$suite.tests
|
||||
$counts.Failures += [int]$suite.failures
|
||||
$counts.Errors += [int]$suite.errors
|
||||
$counts.Skipped += [int]$suite.skipped
|
||||
}
|
||||
} catch {
|
||||
$counts.Xml = $false
|
||||
}
|
||||
}
|
||||
return $counts
|
||||
}
|
||||
|
||||
function Test-Suite([string]$Name, $Counts, $Budget, [string]$MinKey, [string]$MaxKey) {
|
||||
# JUnit is authoritative for the result: the interpreter has been observed to crash
|
||||
# in Qt teardown *after* reporting every test (exit 0xC0000005 with "23 passed"), and
|
||||
# a crash that happens after the last assertion is not a failed assertion. The exit
|
||||
# code is still recorded, and a disagreement is called out.
|
||||
if (-not $Counts.Xml) {
|
||||
Add-Result $Name 'failed' "no JUnit report (pytest exit $($Counts.Exit))"
|
||||
return
|
||||
}
|
||||
$detail = "$($Counts.Tests) tests, $($Counts.Failures) failed, $($Counts.Errors) errors, $($Counts.Skipped) skipped"
|
||||
if ($Counts.Failures -ne 0 -or $Counts.Errors -ne 0) {
|
||||
Add-Result $Name 'failed' $detail
|
||||
return
|
||||
}
|
||||
$note = $detail
|
||||
$ok = $true
|
||||
if ($Counts.Exit -ne 0) {
|
||||
$note = "$detail; process exit $($Counts.Exit) (Qt teardown after the summary?)"
|
||||
}
|
||||
if ($Budget.ContainsKey($MinKey)) {
|
||||
$floor = [int]$Budget[$MinKey]
|
||||
if ($Counts.Tests -lt $floor) {
|
||||
$ok = $false
|
||||
$note = "$detail; only $($Counts.Tests) tests collected, floor is $floor"
|
||||
}
|
||||
}
|
||||
if ($Budget.ContainsKey($MaxKey)) {
|
||||
$ceiling = [int]$Budget[$MaxKey]
|
||||
if ($Counts.Skipped -gt $ceiling) {
|
||||
$ok = $false
|
||||
$note = "$detail; more than the $ceiling skips allowed"
|
||||
}
|
||||
}
|
||||
if ($ok) {
|
||||
Add-Result $Name 'passed' $note
|
||||
} else {
|
||||
Add-Result $Name 'failed' $note
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
$budget = Read-Budget $BudgetFile
|
||||
} catch {
|
||||
Write-Host "gate: $($_.Exception.Message)" -ForegroundColor Red
|
||||
exit 2
|
||||
}
|
||||
if ($budget.Count -eq 0) {
|
||||
Write-Host "gate: no budgets read from $BudgetFile (floors and skip ceilings are part of the gate)" -ForegroundColor Red
|
||||
exit 2
|
||||
}
|
||||
|
||||
Write-Step 'studio suite'
|
||||
$studio = Invoke-Suite 'studio suite' 'apps/studio/tests'
|
||||
Test-Suite 'studio suite' $studio $budget 'studio_min_tests' 'studio_max_skips'
|
||||
|
||||
Write-Step 'shared suite'
|
||||
$shared = Invoke-Suite 'shared suite' 'shared/tests'
|
||||
Test-Suite 'shared suite' $shared $budget 'shared_min_tests' 'shared_max_skips'
|
||||
|
||||
# ------------------------------------------------------------------ ui_check
|
||||
Write-Step 'ui_check'
|
||||
$uiLog = Join-Path $Scratch 'ui_check.log'
|
||||
$uiExit = Invoke-Native $Python @((Join-Path $Repo 'apps\studio\tools\ui_check.py')) $uiLog
|
||||
# ui_check prints "scenes: 4 checks: 631 failures: 2"; the last number is the verdict.
|
||||
$uiSummary = Select-String -LiteralPath $uiLog -Pattern 'checks:\s*\d+.*failures:\s*(\d+)'
|
||||
if ($uiSummary) {
|
||||
$match = $uiSummary[-1].Matches[0]
|
||||
$failures = [int]$match.Groups[1].Value
|
||||
$line = $uiSummary[-1].Line.Trim()
|
||||
if ($failures -eq 0) {
|
||||
Add-Result 'ui_check' 'passed' $line
|
||||
} else {
|
||||
Add-Result 'ui_check' 'failed' $line
|
||||
}
|
||||
} else {
|
||||
Add-Result 'ui_check' 'failed' "no 'checks: N ... failures: M' summary (exit $uiExit)"
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ summary
|
||||
$failed = @($Results | Where-Object { $_.Status -ne 'passed' })
|
||||
$verdict = if ($failed.Count -eq 0) { 'PASS' } else { 'FAIL' }
|
||||
|
||||
Write-Step "summary: $verdict"
|
||||
foreach ($result in $Results) {
|
||||
$colour = if ($result.Status -eq 'passed') { 'Green' } else { 'Red' }
|
||||
Write-Host (" {0,-16} {1,-7} {2}" -f $result.Name, $result.Status, $result.Detail) -ForegroundColor $colour
|
||||
}
|
||||
|
||||
$lines = New-Object System.Collections.ArrayList
|
||||
[void]$lines.Add("# Gate $RunId : $verdict")
|
||||
[void]$lines.Add('')
|
||||
[void]$lines.Add("Platform : Windows")
|
||||
[void]$lines.Add("Python : $Python")
|
||||
[void]$lines.Add("Scratch : $Scratch")
|
||||
[void]$lines.Add('')
|
||||
[void]$lines.Add('| Check | Result | Detail |')
|
||||
[void]$lines.Add('|---|---|---|')
|
||||
foreach ($result in $Results) {
|
||||
[void]$lines.Add("| $($result.Name) | $($result.Status) | $($result.Detail) |")
|
||||
}
|
||||
[void]$lines.Add('')
|
||||
[void]$lines.Add('Logs: lint.log, studio-suite.log, shared-suite.log, ui_check.log, and the JUnit XMLs, in this directory.')
|
||||
$summaryPath = Join-Path $Scratch 'summary.md'
|
||||
Set-Content -LiteralPath $summaryPath -Value $lines -Encoding UTF8
|
||||
Write-Host ''
|
||||
Write-Host "gate: $verdict (summary: $summaryPath)"
|
||||
|
||||
if ($failed.Count -eq 0) { exit 0 }
|
||||
exit 1
|
||||
@@ -0,0 +1,22 @@
|
||||
# Budgets for tools/gate.ps1. One key per line, `name = integer`; `#` starts a comment.
|
||||
#
|
||||
# <suite>_min_tests the fewest tests the suite may collect. A model that becomes
|
||||
# uncollectable (a broken import, a renamed module) shrinks the
|
||||
# suite silently; this floor turns that into a failed gate instead
|
||||
# of a faster green run.
|
||||
# <suite>_max_skips the most tests the suite may skip. Skipping is how an environment
|
||||
# problem or a missing input quietly stops testing anything.
|
||||
#
|
||||
# Measured on Windows 2026-10-18, after docs/test-repair-plan.zh.md §4 stage 3:
|
||||
# studio 543 collected, 19 skipped (all 19 are "no platform GL context" under the
|
||||
# offscreen plugin; they are the same on every machine without a GL context)
|
||||
# shared 486 collected, 18 skipped (3 `timing` tests are deselected, not skipped)
|
||||
#
|
||||
# The ceilings sit a few above today's numbers so a legitimate extra skip does not block
|
||||
# work, but a whole module's worth of new skips does. Lower them when skips are fixed;
|
||||
# never raise them to make a red gate green.
|
||||
|
||||
studio_min_tests = 530
|
||||
studio_max_skips = 25
|
||||
shared_min_tests = 470
|
||||
shared_max_skips = 24
|
||||
Reference in New Issue
Block a user