mirror of
https://github.com/nmasur/dotfiles
synced 2026-10-18 20:29:54 +00:00
remove post-TUI lag triage tooling (root cause was starship notifications)
This commit is contained in:
Noah Masur
@@ -1,5 +1,10 @@
|
|||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## 2026-10-17: removed post-TUI lag tooling
|
||||||
|
|
||||||
|
- With the root cause fixed (starship notifications, below), removed the `lag-triage` module: `lag-triage`, `unlag`, `lag-sample`, `term-probe`, the PTY repro script, the flight recorder filter and the Ctrl-b `heal-autosuggest` binding. Also deleted its `power-user.nix` enable and `~/.local/state/lag-triage`.
|
||||||
|
- The zellij `fish-no-query-term` wrapper now only exports `fish_features=no-query-term` before starting fish; the recorder is gone.
|
||||||
|
|
||||||
## 2026-10-17: post-TUI lag ROOT CAUSE — starship cmd_duration notifications
|
## 2026-10-17: post-TUI lag ROOT CAUSE — starship cmd_duration notifications
|
||||||
|
|
||||||
- The first timestamped episode (fish 4.9.3, fresh zellij session) showed fish handling keys in ~1ms, but every prompt took **~2s**, blocked on one child process. After Ctrl-b, prompts dropped to ~50ms.
|
- The first timestamped episode (fish 4.9.3, fresh zellij session) showed fish handling keys in ~1ms, but every prompt took **~2s**, blocked on one child process. After Ctrl-b, prompts dropped to ~50ms.
|
||||||
|
|||||||
@@ -1,90 +0,0 @@
|
|||||||
{
|
|
||||||
config,
|
|
||||||
pkgs,
|
|
||||||
lib,
|
|
||||||
...
|
|
||||||
}:
|
|
||||||
|
|
||||||
let
|
|
||||||
cfg = config.nmasur.presets.programs.lag-triage;
|
|
||||||
|
|
||||||
term-probe = pkgs.writeScriptBin "term-probe" ''
|
|
||||||
#!${lib.getExe pkgs.python3}
|
|
||||||
${builtins.readFile ./term_probe.py}
|
|
||||||
'';
|
|
||||||
in
|
|
||||||
|
|
||||||
{
|
|
||||||
|
|
||||||
options.nmasur.presets.programs.lag-triage.enable =
|
|
||||||
lib.mkEnableOption "Terminal input-lag triage tools";
|
|
||||||
|
|
||||||
config = lib.mkIf cfg.enable {
|
|
||||||
|
|
||||||
home.packages = [ term-probe ];
|
|
||||||
|
|
||||||
# The zellij pane wrapper (fish-no-query-term) exports FISH_DEBUG and
|
|
||||||
# FISH_DEBUG_OUTPUT so the pane shell records itself. fish reads them once
|
|
||||||
# at startup, so drop them here to keep subshells and `fish -c` children
|
|
||||||
# from writing into the recorder's FIFO path.
|
|
||||||
programs.fish.shellInit = lib.mkBefore ''
|
|
||||||
if set -q FISH_FLIGHT_LOG
|
|
||||||
set -e FISH_DEBUG
|
|
||||||
set -e FISH_DEBUG_OUTPUT
|
|
||||||
end
|
|
||||||
'';
|
|
||||||
|
|
||||||
# Ctrl-b: EXECUTE the proven manual cure as a real commandline. The cure
|
|
||||||
# is not the variable's end value (it starts and ends at 1) — it is the
|
|
||||||
# reader fully exiting readline and re-entering, which only command
|
|
||||||
# EXECUTION does. Setting the variable inline (a plain binding body, or the
|
|
||||||
# old fish_postexec/fish_cancel hook) never makes the reader exit/re-enter,
|
|
||||||
# so it never cured and, with extra repaints on a wedged reader, made it
|
|
||||||
# worse. `commandline -f execute` reproduces exactly what typing the cure
|
|
||||||
# and pressing Enter does — indistinguishable to fish from the manual cure.
|
|
||||||
# Note: this submits the current commandline, so it runs the cure in place
|
|
||||||
# of whatever is typed (fine for a rescue key hit at an empty prompt).
|
|
||||||
# Ctrl-b chosen because it is otherwise unbound (Ctrl-g is taken).
|
|
||||||
nmasur.presets.programs.fish.fish_user_key_bindings = # fish
|
|
||||||
''
|
|
||||||
for mode in insert default visual
|
|
||||||
bind -M $mode \cb heal-autosuggest
|
|
||||||
end
|
|
||||||
'';
|
|
||||||
|
|
||||||
programs.fish.functions = {
|
|
||||||
lag-triage = {
|
|
||||||
description = "Diagnose post-TUI typing lag in the current shell";
|
|
||||||
body = builtins.readFile ./lag-triage.fish;
|
|
||||||
};
|
|
||||||
unlag = {
|
|
||||||
description = "Reset terminal state left behind by a TUI";
|
|
||||||
body = builtins.readFile ./unlag.fish;
|
|
||||||
};
|
|
||||||
lag-sample = {
|
|
||||||
description = "Stack-sample fish and zellij while typing lag is happening";
|
|
||||||
body = builtins.readFile ./lag-sample.fish;
|
|
||||||
};
|
|
||||||
heal-autosuggest = {
|
|
||||||
description = "Heal post-TUI typing lag by executing the autosuggestion-toggle cure (bind to a key)";
|
|
||||||
# Replace the commandline with the exact cure the user runs by hand and
|
|
||||||
# execute it. Executing (not inline-setting) is what cures: it forces
|
|
||||||
# the reader to leave and re-enter readline. Runs in place of whatever
|
|
||||||
# is currently typed.
|
|
||||||
# When the flight recorder is armed, first copy the recent timestamped
|
|
||||||
# log so the lag that made you press Ctrl-b is preserved for analysis.
|
|
||||||
body = # fish
|
|
||||||
''
|
|
||||||
if set -q FISH_FLIGHT_LOG; and test -f "$FISH_FLIGHT_LOG"
|
|
||||||
set -l dir ~/.local/state/lag-triage/episodes
|
|
||||||
mkdir -p $dir
|
|
||||||
tail -n 20000 $FISH_FLIGHT_LOG >$dir/episode-(date +%Y%m%d-%H%M%S)-$fish_pid.log
|
|
||||||
end
|
|
||||||
commandline -r 'set -g fish_autosuggestion_enabled 0; and set -g fish_autosuggestion_enabled 1'
|
|
||||||
commandline -f execute
|
|
||||||
'';
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
};
|
|
||||||
}
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
# Flight-recorder filter for fish's FISH_DEBUG stream (see fish-no-query-term
|
|
||||||
# in zellij.nix). Reads the FIFO fish writes its debug log to, prefixes every
|
|
||||||
# line with a millisecond wall-clock timestamp, and redacts anything typed or
|
|
||||||
# pasted so the always-on log never holds secrets.
|
|
||||||
#
|
|
||||||
# Timing is the whole point: the gap between "Read byte" and the following
|
|
||||||
# "Writing terminal command" is how long fish itself took to handle a key. If
|
|
||||||
# that gap stays small while the pane feels laggy, the delay is downstream of
|
|
||||||
# fish (zellij/ghostty), not in fish.
|
|
||||||
#
|
|
||||||
# Usage: perl flight_filter.pl <fifo> <logfile>
|
|
||||||
use strict;
|
|
||||||
use warnings;
|
|
||||||
use Time::HiRes qw(time);
|
|
||||||
use POSIX qw(strftime);
|
|
||||||
|
|
||||||
my ($fifo, $logfile) = @ARGV;
|
|
||||||
open(my $in, '<', $fifo) or exit 1;
|
|
||||||
# fish has the FIFO open now; nothing else needs the path.
|
|
||||||
unlink $fifo;
|
|
||||||
open(my $out, '>>', $logfile) or exit 1;
|
|
||||||
$out->autoflush(1);
|
|
||||||
|
|
||||||
# Commands worth naming in the log; everything else is redacted.
|
|
||||||
my $keep_cmd = qr/^(heal-autosuggest|set -g fish_autosuggestion_enabled|exec fish|lag-)/;
|
|
||||||
|
|
||||||
while (my $line = <$in>) {
|
|
||||||
$line =~ s/\0//g;
|
|
||||||
# Keystrokes: keep control/escape identities, drop printable content.
|
|
||||||
if ($line =~ /^(reader: Read byte) (.*)$/) {
|
|
||||||
my ($prefix, $b) = ($1, $2);
|
|
||||||
$b = '<printable>' unless $b =~ /^\\(e|r|t|n|x[0-9a-fA-F]+)$/;
|
|
||||||
$line = "$prefix $b\n";
|
|
||||||
}
|
|
||||||
elsif ($line =~ /^(reader: Read char) (\S+)/) {
|
|
||||||
my ($prefix, $k) = ($1, $2);
|
|
||||||
$k = '<printable>'
|
|
||||||
unless $k =~ /^(enter|tab|escape|backspace|delete|space|up|down|left|right|home|end)$|^(ctrl|alt|shift|super)-/;
|
|
||||||
$line = "$prefix $k\n";
|
|
||||||
}
|
|
||||||
elsif ($line =~ /^reader: adding peeked /) {
|
|
||||||
$line = "reader: adding peeked <redacted>\n";
|
|
||||||
}
|
|
||||||
elsif ($line =~ /^reader: Found binding .*?commands: (\[[^\]]*\])/) {
|
|
||||||
$line = "reader: Found binding $1\n";
|
|
||||||
}
|
|
||||||
$line =~ s/(Error while expanding string ).*/$1<redacted>/;
|
|
||||||
$line =~ s/(CommandStart\(")(.*)("\))/$1 . ($2 =~ $keep_cmd ? $2 : '<cmd>') . $3/e;
|
|
||||||
$line =~ s/(WindowTitle\().*(\))/$1<title>$2/;
|
|
||||||
my $t = time;
|
|
||||||
printf $out "%s.%03d %s", strftime('%H:%M:%S', localtime $t), ($t - int $t) * 1000, $line;
|
|
||||||
}
|
|
||||||
@@ -1,53 +0,0 @@
|
|||||||
# Capture stack samples of this fish process, the zellij server, and the
|
|
||||||
# zellij client WHILE the typing lag is happening. This names the guilty
|
|
||||||
# component directly: if fish's main thread is busy/blocked per keystroke the
|
|
||||||
# stacks show exactly where; if fish is idle while typing feels laggy, the
|
|
||||||
# delay is in zellij's render path instead.
|
|
||||||
#
|
|
||||||
# CAUTION (learned 2026-08-30): attaching the sampler to a lagging fish CURES
|
|
||||||
# the lag (thread suspend/resume unwedges it), so run this from a DIFFERENT
|
|
||||||
# pane with the lagging shell's pid: `lag-sample <pid>` (get it in the lagging
|
|
||||||
# shell with the builtin-only `echo $fish_pid`). Have someone type in the
|
|
||||||
# lagging pane while sampling runs — the first samples may catch the wedge.
|
|
||||||
# With no argument it samples the current shell.
|
|
||||||
|
|
||||||
set -l target $fish_pid
|
|
||||||
if test (count $argv) -ge 1; and test -n "$argv[1]"
|
|
||||||
set target $argv[1]
|
|
||||||
end
|
|
||||||
|
|
||||||
set -l outdir ~/.local/state/lag-triage
|
|
||||||
mkdir -p $outdir
|
|
||||||
set -l ts (date +%Y%m%d-%H%M%S)
|
|
||||||
set -l dur 8
|
|
||||||
|
|
||||||
set -l fishfile $outdir/sample-$ts-fish-$target.txt
|
|
||||||
/usr/bin/sample $target $dur 1 -file $fishfile &>/dev/null &
|
|
||||||
disown
|
|
||||||
|
|
||||||
# this session's zellij server (socket path ends in the session name)
|
|
||||||
set -l serverpid (pgrep -f "zellij --server.*/$ZELLIJ_SESSION_NAME\$")
|
|
||||||
test -z "$serverpid"; and set serverpid (pgrep -f "zellij --server" | head -3)
|
|
||||||
for pid in $serverpid
|
|
||||||
/usr/bin/sample $pid $dur 1 -file $outdir/sample-$ts-zellij-server-$pid.txt &>/dev/null &
|
|
||||||
disown
|
|
||||||
end
|
|
||||||
|
|
||||||
# zellij clients (attached to ghostty): named zellij but without --server args
|
|
||||||
set -l allserver (pgrep -f "zellij --server")
|
|
||||||
set -l clientpid
|
|
||||||
for pid in (pgrep -x zellij)
|
|
||||||
contains $pid $allserver; or set -a clientpid $pid
|
|
||||||
end
|
|
||||||
for pid in $clientpid[1..3]
|
|
||||||
/usr/bin/sample $pid $dur 1 -file $outdir/sample-$ts-zellij-client-$pid.txt &>/dev/null &
|
|
||||||
disown
|
|
||||||
end
|
|
||||||
|
|
||||||
# notify when done, without occupying the commandline
|
|
||||||
fish -c "sleep (math $dur + 2); echo; echo '== lag-sample done: '$outdir'/sample-$ts-*.txt =='" &
|
|
||||||
disown
|
|
||||||
|
|
||||||
echo "Sampling fish (pid $target), zellij server(s) [$serverpid], client(s) [$clientpid] for $dur s."
|
|
||||||
echo ">>> TYPE CONTINUOUSLY IN THE LAGGING PANE NOW (junk text is fine) <<<"
|
|
||||||
echo "Files: $outdir/sample-$ts-*.txt"
|
|
||||||
@@ -1,182 +0,0 @@
|
|||||||
# Guided diagnosis for the post-TUI typing-lag problem (Ghostty + Zellij + fish).
|
|
||||||
# Run this IN THE LAGGING SHELL the moment you notice the lag, BEFORE starting
|
|
||||||
# a new shell. It captures evidence, then applies targeted resets one at a time
|
|
||||||
# so the stage that cures the lag identifies the layer holding stuck state.
|
|
||||||
# Everything is logged for filing an upstream issue.
|
|
||||||
|
|
||||||
set -l logdir ~/.local/state/lag-triage
|
|
||||||
mkdir -p $logdir
|
|
||||||
set -l logfile $logdir/(date +%Y%m%d-%H%M%S).log
|
|
||||||
|
|
||||||
function _lt --inherit-variable logfile
|
|
||||||
echo $argv | tee -a $logfile
|
|
||||||
end
|
|
||||||
|
|
||||||
function _lt_ask --inherit-variable logfile
|
|
||||||
# usage: _lt_ask VARNAME prompt... -> sets global $VARNAME (default: skip)
|
|
||||||
set -l __name $argv[1]
|
|
||||||
read -g -P "$argv[2..] " $__name
|
|
||||||
or set -g $__name skip
|
|
||||||
test -z "$$__name"; and set -g $__name skip
|
|
||||||
echo "ANSWER $__name: $$__name" >>$logfile
|
|
||||||
end
|
|
||||||
|
|
||||||
_lt "== lag-triage "(date)" =="
|
|
||||||
_lt "Log: $logfile"
|
|
||||||
_lt "Answer y / n, or press Enter to skip a question."
|
|
||||||
_lt ""
|
|
||||||
|
|
||||||
# ---- 1. Context -------------------------------------------------------------
|
|
||||||
_lt_ask ans_tui "Which TUI did you just exit (nvim/jjui/yazi/other)?"
|
|
||||||
_lt_ask ans_launch "Launched via (f)loating-pane keybind or (c)ommand typed in this shell?"
|
|
||||||
|
|
||||||
# ---- 2. Snapshot ------------------------------------------------------------
|
|
||||||
begin
|
|
||||||
echo "-- snapshot --"
|
|
||||||
fish --version
|
|
||||||
echo "fish pid: $fish_pid, started: "(ps -o lstart= -p $fish_pid 2>/dev/null)
|
|
||||||
zellij --version 2>/dev/null
|
|
||||||
echo "escape delay: '$fish_escape_delay_ms' sequence delay: '$fish_sequence_key_delay_ms'"
|
|
||||||
env | grep -iE '^(TERM|ZELLIJ|GHOSTTY|COLORTERM)' | sort
|
|
||||||
echo "-- status features --"
|
|
||||||
status features
|
|
||||||
echo "-- stty -a --"
|
|
||||||
stty -a
|
|
||||||
end >>$logfile 2>&1
|
|
||||||
_lt "Captured shell + environment snapshot."
|
|
||||||
|
|
||||||
# Proven root cause of the 2026-08 lag (see docs/CHANGELOG.md 2026-08-29):
|
|
||||||
# fish latches feature flags from its startup env before config.fish runs, so
|
|
||||||
# a shell with query-term ON sends terminal queries after every command; one
|
|
||||||
# reply zellij fails to relay permanently degrades this process's reader.
|
|
||||||
if status features | string match -qr '^query-term\s+on'
|
|
||||||
_lt ""
|
|
||||||
_lt "!! query-term is ON in this shell: fish did NOT get fish_features="
|
|
||||||
_lt "!! no-query-term in its STARTUP environment (config.fish is too late)."
|
|
||||||
_lt "!! This is the proven root cause of the post-TUI lag — a query reply"
|
|
||||||
_lt "!! lost by zellij permanently degrades this fish process's reader."
|
|
||||||
_lt "!! Fix: spawn fish with the variable exported (zellij default_shell"
|
|
||||||
_lt "!! wrapper fish-no-query-term). Subshells are immune because they"
|
|
||||||
_lt "!! inherit the exported variable — that's why a new shell 'fixes' it."
|
|
||||||
else
|
|
||||||
_lt "query-term is off in this shell (good — the known root cause is ruled out)."
|
|
||||||
end
|
|
||||||
|
|
||||||
# ---- 3. Terminal state below the shell --------------------------------------
|
|
||||||
_lt ""
|
|
||||||
_lt "Querying terminal state (takes a few seconds)..."
|
|
||||||
term-probe report 2>&1 | tee -a $logfile
|
|
||||||
_lt ""
|
|
||||||
_lt " ^ Things to look for: kitty flags with a reply > 0, modifyOtherKeys > 1,"
|
|
||||||
_lt " any mouse/alternate-screen mode SET while at a shell prompt, or a slow"
|
|
||||||
_lt " DA1 round-trip (> 100 ms means the input path itself is delayed)."
|
|
||||||
|
|
||||||
# ---- 4. Raw keystroke capture (bypasses fish entirely) ----------------------
|
|
||||||
_lt ""
|
|
||||||
_lt "Raw input capture: type ~10 characters at a steady pace, including one"
|
|
||||||
_lt "ESC press and one arrow key. This shows the exact bytes this pane delivers"
|
|
||||||
_lt "and their timing, with fish's input handling out of the picture."
|
|
||||||
term-probe keylog 2>&1 | tee -a $logfile
|
|
||||||
_lt_ask ans_keylog_instant "Did each keypress appear INSTANTLY in the capture? (y/n)"
|
|
||||||
_lt_ask ans_keylog_plain "Were plain letters single plain bytes like b'a' (not escape sequences)? (y/n)"
|
|
||||||
|
|
||||||
# ---- 5. Scope ---------------------------------------------------------------
|
|
||||||
_lt ""
|
|
||||||
_lt_ask ans_scope_pane "Optional: open a NEW zellij pane/tab and type — laggy there too? (y/n)"
|
|
||||||
_lt_ask ans_scope_window "Optional: type in a separate Ghostty window (outside this zellij session) — laggy? (y/n)"
|
|
||||||
|
|
||||||
# ---- 6. Staged resets -------------------------------------------------------
|
|
||||||
# Each stage resets one category of state a TUI could have left behind.
|
|
||||||
# The first stage that cures the lag names the culprit.
|
|
||||||
set -l fixed none
|
|
||||||
|
|
||||||
_lt ""
|
|
||||||
_lt "Now applying resets one at a time. After each, type into the test prompt"
|
|
||||||
_lt "to judge whether the lag is gone."
|
|
||||||
_lt "CAVEAT: fish's read prompt may NOT exhibit lag even when the main"
|
|
||||||
_lt "commandline does. If typing at these test prompts never feels laggy at"
|
|
||||||
_lt "all, answer 'u' (unsure) instead of 'y' — a 'y' here is only meaningful"
|
|
||||||
_lt "if you could feel the lag at the test prompts before the reset."
|
|
||||||
|
|
||||||
if test $fixed = none
|
|
||||||
_lt ""
|
|
||||||
_lt "Stage A - kitty keyboard protocol: pop stack + clear all flags"
|
|
||||||
printf '\e[<9u\e[=0;1u'
|
|
||||||
_lt_ask ans_stage_a " Test typing here, then Enter — lag gone? (y/n)"
|
|
||||||
test "$ans_stage_a" = y; and set fixed "A (kitty keyboard state)"
|
|
||||||
end
|
|
||||||
|
|
||||||
if test $fixed = none
|
|
||||||
_lt "Stage B - modifyOtherKeys off"
|
|
||||||
printf '\e[>4;0m'
|
|
||||||
_lt_ask ans_stage_b " Test typing here, then Enter — lag gone? (y/n)"
|
|
||||||
test "$ans_stage_b" = y; and set fixed "B (modifyOtherKeys)"
|
|
||||||
end
|
|
||||||
|
|
||||||
if test $fixed = none
|
|
||||||
_lt "Stage C - normal keypad + normal cursor keys"
|
|
||||||
printf '\e>\e[?1l'
|
|
||||||
_lt_ask ans_stage_c " Test typing here, then Enter — lag gone? (y/n)"
|
|
||||||
test "$ans_stage_c" = y; and set fixed "C (application keypad/cursor mode)"
|
|
||||||
end
|
|
||||||
|
|
||||||
if test $fixed = none
|
|
||||||
_lt "Stage D - disable mouse, focus reporting, synchronized output"
|
|
||||||
printf '\e[?1000l\e[?1001l\e[?1002l\e[?1003l\e[?1005l\e[?1006l\e[?1015l\e[?1016l\e[?1004l\e[?2026l'
|
|
||||||
_lt_ask ans_stage_d " Test typing here, then Enter — lag gone? (y/n)"
|
|
||||||
test "$ans_stage_d" = y; and set fixed "D (mouse/focus/sync modes)"
|
|
||||||
end
|
|
||||||
|
|
||||||
if test $fixed = none
|
|
||||||
_lt "Stage E - leave alternate screen"
|
|
||||||
printf '\e[?1049l'
|
|
||||||
_lt_ask ans_stage_e " Test typing here, then Enter — lag gone? (y/n)"
|
|
||||||
test "$ans_stage_e" = y; and set fixed "E (alternate screen)"
|
|
||||||
end
|
|
||||||
|
|
||||||
if test $fixed = none
|
|
||||||
_lt "Stage F - stty sane (line-discipline reset)"
|
|
||||||
stty sane
|
|
||||||
_lt_ask ans_stage_f " Test typing here, then Enter — lag gone? (y/n)"
|
|
||||||
test "$ans_stage_f" = y; and set fixed "F (termios/line discipline)"
|
|
||||||
end
|
|
||||||
|
|
||||||
if test $fixed = none
|
|
||||||
_lt "Stage G - DECSTR soft terminal reset"
|
|
||||||
printf '\e[!p'
|
|
||||||
_lt_ask ans_stage_g " Test typing here, then Enter — lag gone? (y/n)"
|
|
||||||
test "$ans_stage_g" = y; and set fixed "G (DECSTR-resettable mode)"
|
|
||||||
end
|
|
||||||
|
|
||||||
# ---- 7. Verdict --------------------------------------------------------------
|
|
||||||
_lt ""
|
|
||||||
_lt "== Verdict =="
|
|
||||||
if test $fixed != none
|
|
||||||
_lt "Lag cleared by stage $fixed."
|
|
||||||
_lt "That state was stuck BELOW fish — in the Zellij pane or relayed to"
|
|
||||||
_lt "Ghostty — and the TUI you exited ($ans_tui) failed to restore it, or"
|
|
||||||
_lt "Zellij failed to restore it when the pane closed."
|
|
||||||
_lt "Re-probing terminal state after the fix for comparison:"
|
|
||||||
term-probe report 2>&1 | tee -a $logfile
|
|
||||||
_lt ""
|
|
||||||
_lt "-> File this log against zellij (or ghostty, if a separate window also"
|
|
||||||
_lt " lagged). The before/after probe diff pinpoints the exact stuck mode."
|
|
||||||
else if test "$ans_keylog_instant" = y; and test "$ans_keylog_plain" = y
|
|
||||||
_lt "Raw input reaches this pane instantly as plain bytes, and no terminal"
|
|
||||||
_lt "state reset helps: the lag lives INSIDE this fish process (reader state)."
|
|
||||||
_lt "Confirm now: run 'exec fish' — if that cures it, it is fish-internal."
|
|
||||||
_lt ""
|
|
||||||
_lt "-> To catch it in the act, run your next long-lived shell as:"
|
|
||||||
_lt " FISH_DEBUG='reader,term-support' FISH_DEBUG_OUTPUT=$logdir/fish-debug.log fish"
|
|
||||||
_lt " then re-run lag-triage when it recurs and file both logs to fish-shell."
|
|
||||||
else
|
|
||||||
_lt "Keystrokes were delayed or arrived as escape sequences BEFORE fish saw"
|
|
||||||
_lt "them: the problem is in Zellij (client stdin parser / server) or Ghostty."
|
|
||||||
_lt " new pane also laggy: $ans_scope_pane (y -> session-wide, not this pane)"
|
|
||||||
_lt " separate window laggy: $ans_scope_window (y -> Ghostty itself)"
|
|
||||||
_lt "-> File this log against zellij; include the keylog byte capture."
|
|
||||||
end
|
|
||||||
_lt ""
|
|
||||||
_lt "Full log: $logfile"
|
|
||||||
|
|
||||||
functions -e _lt _lt_ask
|
|
||||||
@@ -1,165 +0,0 @@
|
|||||||
"""Probe the terminal state of the current pane, below the shell.
|
|
||||||
|
|
||||||
Modes:
|
|
||||||
report - query kitty-keyboard flags, modifyOtherKeys, and DEC private
|
|
||||||
modes directly on /dev/tty, reporting each reply (or lack of
|
|
||||||
one) and its round-trip latency. Answers may come from Zellij
|
|
||||||
(pane state) or be relayed from Ghostty (window state).
|
|
||||||
keylog - raw-mode keystroke capture: prints the exact bytes and
|
|
||||||
inter-key latency for every keypress, bypassing the shell's
|
|
||||||
input machinery entirely. Press q to finish.
|
|
||||||
|
|
||||||
Used by the `lag-triage` fish function to pin down which layer
|
|
||||||
(fish / zellij / ghostty) is holding stuck state when typing lags
|
|
||||||
after a TUI exits.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import select
|
|
||||||
import sys
|
|
||||||
import termios
|
|
||||||
import time
|
|
||||||
import tty
|
|
||||||
|
|
||||||
# DECRQM reply values
|
|
||||||
DECRQM_VALUES = {
|
|
||||||
"0": "not recognized",
|
|
||||||
"1": "SET",
|
|
||||||
"2": "reset",
|
|
||||||
"3": "permanently set",
|
|
||||||
"4": "permanently reset",
|
|
||||||
}
|
|
||||||
|
|
||||||
DEC_MODES = [
|
|
||||||
(1, "application cursor keys (DECCKM)"),
|
|
||||||
(25, "cursor visible"),
|
|
||||||
(1000, "mouse click reporting"),
|
|
||||||
(1002, "mouse drag reporting"),
|
|
||||||
(1003, "mouse all-motion reporting"),
|
|
||||||
(1004, "focus reporting"),
|
|
||||||
(1006, "SGR mouse encoding"),
|
|
||||||
(1049, "alternate screen"),
|
|
||||||
(2004, "bracketed paste"),
|
|
||||||
(2026, "synchronized output"),
|
|
||||||
(2031, "color theme reporting"),
|
|
||||||
]
|
|
||||||
|
|
||||||
QUERIES = [
|
|
||||||
("kitty keyboard flags (\\e[?u)", b"\x1b[?u", rb"\x1b\[\?(\d+)u", None),
|
|
||||||
("modifyOtherKeys (XTQMODKEYS)", b"\x1b[?4m", rb"\x1b\[>4;(\d+)m", None),
|
|
||||||
("background color (OSC 11)", b"\x1b]11;?\x1b\\", rb"\x1b\]11;([^\x07\x1b]+)", None),
|
|
||||||
] + [
|
|
||||||
(
|
|
||||||
f"DEC mode {num} — {desc}",
|
|
||||||
b"\x1b[?%d$p" % num,
|
|
||||||
rb"\x1b\[\?%d;(\d+)\$y" % num,
|
|
||||||
DECRQM_VALUES,
|
|
||||||
)
|
|
||||||
for num, desc in DEC_MODES
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def read_for(fd, seconds):
|
|
||||||
buf = b""
|
|
||||||
end = time.monotonic() + seconds
|
|
||||||
while True:
|
|
||||||
remaining = end - time.monotonic()
|
|
||||||
if remaining <= 0:
|
|
||||||
break
|
|
||||||
r, _, _ = select.select([fd], [], [], remaining)
|
|
||||||
if not r:
|
|
||||||
break
|
|
||||||
buf += os.read(fd, 4096)
|
|
||||||
return buf
|
|
||||||
|
|
||||||
|
|
||||||
def report(fd):
|
|
||||||
lines = []
|
|
||||||
raw_dump = b""
|
|
||||||
for label, query, pattern, value_names in QUERIES:
|
|
||||||
raw_dump += read_for(fd, 0.02) # drain stragglers
|
|
||||||
start = time.monotonic()
|
|
||||||
os.write(fd, query)
|
|
||||||
buf = b""
|
|
||||||
match = None
|
|
||||||
deadline = time.monotonic() + 0.35
|
|
||||||
while time.monotonic() < deadline:
|
|
||||||
buf += read_for(fd, 0.05)
|
|
||||||
match = re.search(pattern, buf)
|
|
||||||
if match:
|
|
||||||
break
|
|
||||||
raw_dump += buf
|
|
||||||
if match:
|
|
||||||
latency = (time.monotonic() - start) * 1000
|
|
||||||
value = match.group(1).decode("ascii", "replace")
|
|
||||||
if value_names:
|
|
||||||
value = f"{value} ({value_names.get(value, '?')})"
|
|
||||||
lines.append(f" {label:45s} = {value:24s} [{latency:6.1f} ms]")
|
|
||||||
else:
|
|
||||||
lines.append(f" {label:45s} = (no reply)")
|
|
||||||
|
|
||||||
# DA1 as a fence: every terminal answers it, so its round-trip time
|
|
||||||
# measures the whole input path (ghostty -> zellij -> pane -> here).
|
|
||||||
start = time.monotonic()
|
|
||||||
os.write(fd, b"\x1b[c")
|
|
||||||
buf = b""
|
|
||||||
match = None
|
|
||||||
deadline = time.monotonic() + 2.0
|
|
||||||
while time.monotonic() < deadline:
|
|
||||||
buf += read_for(fd, 0.05)
|
|
||||||
match = re.search(rb"\x1b\[\?([0-9;]*)c", buf)
|
|
||||||
if match:
|
|
||||||
break
|
|
||||||
raw_dump += buf
|
|
||||||
if match:
|
|
||||||
latency = (time.monotonic() - start) * 1000
|
|
||||||
lines.append(
|
|
||||||
f" {'device attributes (DA1) round-trip':45s} = "
|
|
||||||
f"{match.group(1).decode():24s} [{latency:6.1f} ms]"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
lines.append(f" {'device attributes (DA1) round-trip':45s} = (NO REPLY in 2s!)")
|
|
||||||
lines.append(f" raw bytes received: {raw_dump!r}")
|
|
||||||
return lines
|
|
||||||
|
|
||||||
|
|
||||||
def keylog(fd):
|
|
||||||
sys.stdout.write("keylog: capturing raw bytes from the tty. Press q to finish.\r\n")
|
|
||||||
sys.stdout.flush()
|
|
||||||
last = time.monotonic()
|
|
||||||
while True:
|
|
||||||
select.select([fd], [], [], None)
|
|
||||||
data = os.read(fd, 4096)
|
|
||||||
now = time.monotonic()
|
|
||||||
delta_ms = (now - last) * 1000
|
|
||||||
last = now
|
|
||||||
sys.stdout.write(f" +{delta_ms:8.1f} ms {data!r} hex={data.hex(' ')}\r\n")
|
|
||||||
sys.stdout.flush()
|
|
||||||
if data in (b"q", b"\x03", b"\x04"):
|
|
||||||
break
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
mode = sys.argv[1] if len(sys.argv) > 1 else "report"
|
|
||||||
fd = os.open("/dev/tty", os.O_RDWR)
|
|
||||||
old = termios.tcgetattr(fd)
|
|
||||||
lines = None
|
|
||||||
try:
|
|
||||||
tty.setraw(fd)
|
|
||||||
if mode == "report":
|
|
||||||
lines = report(fd)
|
|
||||||
elif mode == "keylog":
|
|
||||||
keylog(fd)
|
|
||||||
else:
|
|
||||||
raise SystemExit(f"unknown mode: {mode}")
|
|
||||||
finally:
|
|
||||||
termios.tcsetattr(fd, termios.TCSADRAIN, old)
|
|
||||||
os.close(fd)
|
|
||||||
if lines:
|
|
||||||
print("terminal state as seen from this pane:")
|
|
||||||
print("\n".join(lines))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -1,15 +0,0 @@
|
|||||||
# One-shot reset of terminal state a TUI may have left behind (kitty keyboard
|
|
||||||
# flags, modifyOtherKeys, application keypad/cursor, mouse/focus reporting,
|
|
||||||
# alternate screen, termios). fish re-enables the modes it wants at the next
|
|
||||||
# prompt, so this is safe to run any time.
|
|
||||||
#
|
|
||||||
# Diagnostic value: if this cures the lag, the stuck state was below fish
|
|
||||||
# (run lag-triage next time to find which mode). If only `exec fish` cures
|
|
||||||
# it, the lag is inside the fish process itself.
|
|
||||||
printf '\e[<9u\e[=0;1u'
|
|
||||||
printf '\e[>4;0m'
|
|
||||||
printf '\e>\e[?1l'
|
|
||||||
printf '\e[?1000l\e[?1001l\e[?1002l\e[?1003l\e[?1005l\e[?1006l\e[?1015l\e[?1016l\e[?1004l\e[?2026l'
|
|
||||||
printf '\e[?1049l'
|
|
||||||
stty sane
|
|
||||||
echo "terminal state reset — if typing still lags, run lag-triage (before exec fish!)"
|
|
||||||
@@ -1,151 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Deterministic repro of permanent fish reader degradation (fish 4.8.1).
|
|
||||||
|
|
||||||
This is evidence for an upstream fish-shell report, and the proof behind the
|
|
||||||
fish-no-query-term wrapper in presets/programs/zellij.nix. Not installed by
|
|
||||||
the nix module; run directly: python3 upstream_repro.py [queryterm|noqueryterm]
|
|
||||||
|
|
||||||
Finding: with the query-term feature enabled (latched from the startup env,
|
|
||||||
which is fish's default), fish sends OSC 11 + CPR (\e[6n) + DA1 (\e[0c)
|
|
||||||
after every external command and waits for the replies. If the terminal
|
|
||||||
fails to reply during ONE such cycle -- even though it answered every query
|
|
||||||
before and answers every query after -- that fish process's interactive
|
|
||||||
reader is PERMANENTLY degraded: keystrokes are no longer echoed (>3s each,
|
|
||||||
never recovers). In production this happens when zellij drops/mis-relays a
|
|
||||||
reply during TUI teardown or heavy output (cf. zellij-org/zellij#5158), and
|
|
||||||
it presents as permanent typing lag cured only by replacing the process.
|
|
||||||
With fish_features=no-query-term in the startup environment, the same
|
|
||||||
sequence has zero effect (~35ms echo throughout).
|
|
||||||
|
|
||||||
Phases:
|
|
||||||
A. terminal answers all queries -> echo ~35ms (both variants)
|
|
||||||
B. replies dropped for one command -> queryterm: echo dead, permanently
|
|
||||||
C. replies restored, another command -> queryterm: still dead
|
|
||||||
"""
|
|
||||||
import os, pty, re, select, subprocess, sys, time, fcntl, termios
|
|
||||||
|
|
||||||
VARIANT = sys.argv[1] if len(sys.argv) > 1 else "queryterm"
|
|
||||||
|
|
||||||
env = dict(os.environ)
|
|
||||||
env["TERM"] = "xterm-ghostty"
|
|
||||||
env["ZELLIJ"] = "0"
|
|
||||||
env["ZELLIJ_SESSION_NAME"] = "repro"
|
|
||||||
env["FISH_DEBUG"] = "term-support"
|
|
||||||
env["FISH_DEBUG_OUTPUT"] = f"/tmp/fish-lagrepro-{VARIANT}.log"
|
|
||||||
env.pop("fish_features", None)
|
|
||||||
if VARIANT == "noqueryterm":
|
|
||||||
env["fish_features"] = "no-query-term"
|
|
||||||
|
|
||||||
master, slave = pty.openpty()
|
|
||||||
# give it a size
|
|
||||||
fcntl.ioctl(master, termios.TIOCSWINSZ, b"\x00\x28\x00\x78\x00\x00\x00\x00")
|
|
||||||
proc = subprocess.Popen(
|
|
||||||
["fish", "-i", "--no-config"],
|
|
||||||
stdin=slave, stdout=slave, stderr=slave, env=env,
|
|
||||||
preexec_fn=lambda: (os.setsid(), fcntl.ioctl(0, termios.TIOCSCTTY, 0)),
|
|
||||||
close_fds=True,
|
|
||||||
)
|
|
||||||
os.close(slave)
|
|
||||||
|
|
||||||
RESPOND = True
|
|
||||||
transcript = []
|
|
||||||
|
|
||||||
def respond(data):
|
|
||||||
"""Answer terminal queries the way a well-behaved terminal would."""
|
|
||||||
out = b""
|
|
||||||
for m in re.finditer(rb"\x1b\[6n", data):
|
|
||||||
out += b"\x1b[40;1R" # CPR
|
|
||||||
for m in re.finditer(rb"\x1b\[0?c", data):
|
|
||||||
out += b"\x1b[?62;22c" # DA1
|
|
||||||
for m in re.finditer(rb"\x1b\[\?u", data):
|
|
||||||
out += b"\x1b[?0u" # kitty flags
|
|
||||||
for m in re.finditer(rb"\x1b\]11;\?", data):
|
|
||||||
out += b"\x1b]11;rgb:2828/2828/2828\x1b\\" # OSC 11
|
|
||||||
for m in re.finditer(rb"\x1b\[>0?q", data):
|
|
||||||
out += b"\x1bP>|ghostty 1.3.1\x1b\\" # XTVERSION
|
|
||||||
for m in re.finditer(rb"\x1bP\+q[0-9a-fA-F;]+\x1b\\", data):
|
|
||||||
out += b"\x1bP0+r\x1b\\" # XTGETTCAP: not found
|
|
||||||
for m in re.finditer(rb"\x1b\[\?(\d+)\$p", data):
|
|
||||||
out += b"\x1b[?%s;2$y" % m.group(1) # DECRQM: reset
|
|
||||||
return out
|
|
||||||
|
|
||||||
def pump(timeout):
|
|
||||||
"""Read fish output for `timeout` seconds, answering queries if RESPOND."""
|
|
||||||
buf = b""
|
|
||||||
end = time.monotonic() + timeout
|
|
||||||
while time.monotonic() < end:
|
|
||||||
r, _, _ = select.select([master], [], [], 0.03)
|
|
||||||
if r:
|
|
||||||
try:
|
|
||||||
data = os.read(master, 65536)
|
|
||||||
except OSError:
|
|
||||||
return buf
|
|
||||||
buf += data
|
|
||||||
transcript.append(data)
|
|
||||||
if RESPOND:
|
|
||||||
reply = respond(data)
|
|
||||||
if reply:
|
|
||||||
os.write(master, reply)
|
|
||||||
return buf
|
|
||||||
|
|
||||||
def send(s):
|
|
||||||
os.write(master, s if isinstance(s, bytes) else s.encode())
|
|
||||||
|
|
||||||
def measure_echo(chars, settle=0.1):
|
|
||||||
"""Send chars one at a time; measure time until each is echoed."""
|
|
||||||
results = []
|
|
||||||
for ch in chars:
|
|
||||||
pump(settle)
|
|
||||||
t0 = time.monotonic()
|
|
||||||
send(ch)
|
|
||||||
deadline = time.monotonic() + 3.0
|
|
||||||
latency = None
|
|
||||||
buf = b""
|
|
||||||
while time.monotonic() < deadline:
|
|
||||||
buf += pump(0.02)
|
|
||||||
if ch.encode() in buf:
|
|
||||||
latency = (time.monotonic() - t0) * 1000
|
|
||||||
break
|
|
||||||
results.append((ch, latency))
|
|
||||||
return results
|
|
||||||
|
|
||||||
print(f"=== variant: {VARIANT} ===")
|
|
||||||
pump(1.2) # startup, queries answered
|
|
||||||
|
|
||||||
send("echo warmup\r")
|
|
||||||
pump(0.8)
|
|
||||||
|
|
||||||
print("phase A: terminal responsive, echo latency per key:")
|
|
||||||
for ch, ms in measure_echo("abcde"):
|
|
||||||
print(f" {ch}: {ms:.0f} ms" if ms else f" {ch}: NO ECHO in 3s")
|
|
||||||
send("\x15") # ctrl-u clear line
|
|
||||||
pump(0.3)
|
|
||||||
|
|
||||||
# Run external command, then STOP answering queries (simulate lost relay)
|
|
||||||
send("sh -c true\r")
|
|
||||||
time.sleep(0.05)
|
|
||||||
RESPOND = False
|
|
||||||
pump(1.0)
|
|
||||||
|
|
||||||
print("phase B: after external command with query replies DROPPED:")
|
|
||||||
for ch, ms in measure_echo("fghij"):
|
|
||||||
print(f" {ch}: {ms:.0f} ms" if ms else f" {ch}: NO ECHO in 3s")
|
|
||||||
send("\x15")
|
|
||||||
pump(0.3)
|
|
||||||
|
|
||||||
# Does it persist across further commands, with responses restored?
|
|
||||||
RESPOND = True
|
|
||||||
send("sh -c true\r")
|
|
||||||
pump(1.0)
|
|
||||||
print("phase C: responses restored, after another external command:")
|
|
||||||
for ch, ms in measure_echo("klmno"):
|
|
||||||
print(f" {ch}: {ms:.0f} ms" if ms else f" {ch}: NO ECHO in 3s")
|
|
||||||
|
|
||||||
send("\x15")
|
|
||||||
pump(0.2)
|
|
||||||
send("exit\r")
|
|
||||||
pump(0.5)
|
|
||||||
try:
|
|
||||||
proc.wait(timeout=3)
|
|
||||||
except subprocess.TimeoutExpired:
|
|
||||||
proc.kill()
|
|
||||||
@@ -10,42 +10,13 @@ let
|
|||||||
cfg = config.nmasur.presets.programs.zellij;
|
cfg = config.nmasur.presets.programs.zellij;
|
||||||
|
|
||||||
# fish latches feature flags from its startup ENVIRONMENT before config.fish
|
# fish latches feature flags from its startup ENVIRONMENT before config.fish
|
||||||
# runs, so the `set -gx fish_features no-query-term` in config.fish only
|
# runs, so `set -gx fish_features no-query-term` in config.fish only affects
|
||||||
# protects CHILD fish processes — which is why subshells/exec fish were always
|
# child fish processes. Spawning pane shells with the variable already
|
||||||
# immune to the post-TUI typing lag while zellij-spawned pane shells were not.
|
# exported keeps fish from sending terminal queries (OSC 11, CPR, DA1) that
|
||||||
# With query-term latched on, fish sends OSC 11 + CPR + DA1 queries after
|
# zellij may fail to relay; on fish 4.8.1 a single missed reply was shown to
|
||||||
# every command and waits for replies; if zellij fails to relay even one
|
# degrade that shell's reader for good.
|
||||||
# reply (a race during TUI teardown or heavy output), that fish process's
|
|
||||||
# reader is PERMANENTLY degraded — reproduced deterministically in a PTY
|
|
||||||
# harness on fish 4.8.1 (see docs/CHANGELOG.md 2026-08-29 and
|
|
||||||
# presets/programs/lag-triage/upstream_repro.py). Spawning fish with the
|
|
||||||
# variable already exported makes every pane shell immune.
|
|
||||||
# Flight recorder for the still-unsolved post-TUI typing lag: the lag only
|
|
||||||
# shows up in long-lived pane shells and has never reproduced in a fresh
|
|
||||||
# shell, so the recorder must already be running when it starts.
|
|
||||||
# Armed by `touch ~/.local/state/lag-triage/RECORD`; new panes then stream
|
|
||||||
# fish's debug log through flight_filter.pl, which adds millisecond
|
|
||||||
# timestamps (the September logs had none, which made them inconclusive)
|
|
||||||
# and redacts typed/pasted text. Ctrl-b (heal-autosuggest) snapshots the
|
|
||||||
# recent log into ~/.local/state/lag-triage/episodes/ before healing.
|
|
||||||
# Remove the RECORD file to disable (zero overhead when off).
|
|
||||||
fish-no-query-term = pkgs.writeShellScriptBin "fish-no-query-term" ''
|
fish-no-query-term = pkgs.writeShellScriptBin "fish-no-query-term" ''
|
||||||
export fish_features=no-query-term
|
export fish_features=no-query-term
|
||||||
dir="$HOME/.local/state/lag-triage"
|
|
||||||
if [ -e "$dir/RECORD" ]; then
|
|
||||||
mkdir -p "$dir/flight"
|
|
||||||
find "$dir/flight" -type f -mtime +3 -delete 2>/dev/null
|
|
||||||
log="$dir/flight/fish-$(date +%Y%m%d-%H%M%S)-$$.log"
|
|
||||||
fifo="$dir/flight/.fifo-$$"
|
|
||||||
rm -f "$fifo"
|
|
||||||
if mkfifo -m 600 "$fifo"; then
|
|
||||||
# The filter exits on EOF once fish (this pid after exec) exits.
|
|
||||||
${lib.getExe pkgs.perl} ${./lag-triage/flight_filter.pl} "$fifo" "$log" </dev/null >/dev/null 2>&1 &
|
|
||||||
export FISH_DEBUG='reader,reader-render,term-support,proc-termowner,iothread,fd-monitor,topic-monitor,complete,autoload,history,uvar-notifier'
|
|
||||||
export FISH_DEBUG_OUTPUT="$fifo"
|
|
||||||
export FISH_FLIGHT_LOG="$log"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
exec ${lib.getExe pkgs.fish} "$@"
|
exec ${lib.getExe pkgs.fish} "$@"
|
||||||
'';
|
'';
|
||||||
|
|
||||||
|
|||||||
@@ -63,7 +63,6 @@ in
|
|||||||
git.enable = lib.mkDefault true;
|
git.enable = lib.mkDefault true;
|
||||||
helix.enable = lib.mkDefault true;
|
helix.enable = lib.mkDefault true;
|
||||||
jujutsu.enable = lib.mkDefault true;
|
jujutsu.enable = lib.mkDefault true;
|
||||||
lag-triage.enable = lib.mkDefault true;
|
|
||||||
lazygit.enable = lib.mkDefault true;
|
lazygit.enable = lib.mkDefault true;
|
||||||
# neovim.enable = lib.mkDefault true;
|
# neovim.enable = lib.mkDefault true;
|
||||||
nix-index.enable = lib.mkDefault true;
|
nix-index.enable = lib.mkDefault true;
|
||||||
|
|||||||
Reference in New Issue
Block a user