web: non-blocking web_server, browser console mirroring, Tcl 9 fix

src/Main.cc

@@ -418,6 +418,8 @@ static int tclAppInit(int& argc,
     // directly on the main thread, like the non-GUI path.
     const bool gui_enabled = gui::Gui::enabled() && !web_enabled;
 
+    auto* web_server = ord::OpenRoad::openRoad()->getWebServer();
+
     if (read_odb_filename) {
       std::string cmd = fmt::format("read_db {{{}}}", read_odb_filename);
       if (!gui_enabled) {
@@ -476,15 +478,16 @@ static int tclAppInit(int& argc,
       }
     }
 
-    // Block until the web server is stopped (like QApplication::exec()
-    // for the GUI).  After this returns, fall through to readline.
-    if (web_enabled) {
-      auto* server = ord::OpenRoad::openRoad()->getWebServer();
-      server->waitForStop();
-      // `exit` typed in the browser Tcl widget signalled stop; do the
-      // real process exit now from the main thread (worker threads are
-      // already joined by stop()).
-      if (server->exitRequested()) {
+    // If the web server is running at this point — either because of
+    // -web (implicit serve before script) or because the script itself
+    // called `web_server` — park the main thread in Tcl_DoOneEvent so
+    // worker threads can drive browser-typed tcl_eval requests.  The
+    // wait exits when a browser-typed `exit` (requestExit) or
+    // `web_server -stop` (requestStop) wakes the loop.
+    if (web_server->isRunning()) {
+      const bool was_exit = web_server->runEventLoopUntilStop();
+      web_server->stop();
+      if (was_exit) {
         exit(EXIT_SUCCESS);
       }
     }

src/web/include/web/web.h

@@ -5,12 +5,13 @@
 
 #include <spdlog/common.h>
 
-#include <condition_variable>
+#include <atomic>
 #include <cstdint>
 #include <functional>
 #include <memory>
 #include <mutex>
 #include <string>
+#include <string_view>
 #include <thread>
 #include <vector>
 
@@ -64,8 +65,14 @@ ListenerHandle createAndRunListener(
     WebViewerHook* viewer_hook);
 
 // A layout web server.  serve() starts the server in background I/O
-// threads; waitForStop() blocks the calling thread until requestStop()
-// is called, mirroring gui::show / gui::hide.
+// threads and returns immediately.  After serve(), the calling Tcl
+// script (or main-thread REPL) can keep executing — each top-level
+// command on the main thread is broadcast to connected browsers as a
+// `console_input` echo via a Tcl_CreateObjTrace at level 1.  When the
+// script settles, Main.cc parks the main thread in Tcl_DoOneEvent
+// servicing browser-driven Tcl evaluations until exitRequested()
+// or stopRequested() returns true; then the main thread calls
+// stop() to tear down.
 
 class WebServer
 {
@@ -82,21 +89,58 @@ class WebServer
   // the server is already running.
   void serve(int port);
 
-  // True after serve() returns and before stop/destructor.
+  // True after serve() returns and before stop()/destructor.
   bool isRunning() const { return ioc_ != nullptr; }
 
-  // Block the calling thread until requestStop() is called, then
-  // tear down the server.  Typically called on the main/Tcl thread.
-  void waitForStop();
+  // Tears down the I/O threads and cleans up hooks.  Safe to call
+  // multiple times and from any thread (including a worker thread —
+  // stopAndJoinIoThreads detaches the calling thread if it is itself
+  // a worker, to avoid the EDEADLK self-join).  After it returns,
+  // isRunning() is false and serve() may be called again to restart
+  // the server.
+  void stop();
 
-  // Signal waitForStop() to return.  Safe to call from any thread
-  // (e.g. an ASIO worker thread executing a Tcl command).
+  // Park the calling (main) thread in Tcl_DoOneEvent until exitRequested
+  // or stopRequested is set.  Returns true if exit was requested
+  // (caller is expected to stop() then std::exit()), false if stop
+  // was requested (caller stop()s and continues).  No-op (returns false
+  // immediately) if the server isn't running.
+  bool runEventLoopUntilStop();
+
+  // Worker-thread signals to wake the main thread out of Tcl_DoOneEvent.
+  // requestExit() sets the exit flag; the main thread will std::exit(0)
+  // after stop().  requestStop() sets the stop flag; main thread will
+  // tear the server down but not exit the process.  Both queue a no-op
+  // event on the main thread's Tcl event queue so a parked
+  // Tcl_DoOneEvent returns and the wait loop re-checks the flags.
+  void requestExit();
   void requestStop();
 
-  // True if `exit` was invoked from a Tcl command running on a worker
-  // thread.  Main.cc / web.i checks this after waitForStop() returns
-  // and exits the process cleanly from the main thread.
   bool exitRequested() const { return exit_requested_; }
+  bool stopRequested() const { return stop_requested_; }
+
+  // The Tcl thread id captured in serve().  Used by the level-1
+  // command trace to filter out worker-thread Tcl_Eval invocations
+  // (browser-typed tcl_eval requests) — the browser locally echoes
+  // those, so we avoid a duplicate console_input broadcast.
+  Tcl_ThreadId mainThreadId() const { return main_thread_id_; }
+
+  // Lock used to serialize Tcl_Eval / STA access among worker threads.
+  // TclEvaluator::eval() and the STA-touching request handlers in
+  // request_handler.cpp lock this mutex.  Exposed publicly so callers
+  // outside web/ that may directly evaluate Tcl on the same interp
+  // can opt into the same serialization.
+  std::mutex& tclMutex() { return tcl_mutex_; }
+
+  // Broadcasts the given top-level command text to connected browsers
+  // as a `{"type":"console_input","text":...}` push frame.  Called from
+  // the level-1 command trace installed in serve().
+  void broadcastConsoleInput(const std::string& cmd);
+
+  // Broadcasts a chunk of bytes written to Tcl's stdout/stderr channels
+  // as a `{"type":"log","text":...}` push frame.  Called from the
+  // Tcl_StackChannel transform's outputProc installed in serve().
+  void broadcastChannelChunk(std::string_view chunk);
 
   void saveReport(const std::string& filename,
                   int max_setup_paths,
@@ -111,12 +155,13 @@ class WebServer
                  double dbu_per_pixel,
                  const std::string& vis_json);
 
-  // Tears down the I/O threads and cleans up hooks.  Safe to call multiple
-  // times and from any thread; after it returns, isRunning() is false and
-  // serve() may be called again to restart the server.
-  void stop();
-
  private:
+  // Wakes the main thread's Tcl event loop by queuing a no-op Tcl
+  // event so a parked Tcl_DoOneEvent returns and the wait loop in
+  // runEventLoopUntilStop re-checks the exit/stop flags.  Safe to
+  // call from any thread; no-op if main_thread_id_ is unset.
+  void wakeMainEventLoop();
+
   // Stops ioc_, joins every worker thread except the current one, and
   // clears threads_. Detaches the current thread if it happens to be a
   // worker (would otherwise raise EDEADLK on self-join).
@@ -149,14 +194,35 @@ class WebServer
   // Held so stop() can remove it from the Logger before viewer_hook_ is
   // destroyed — the sink stores a raw pointer into the hook.
   spdlog::sink_ptr log_sink_;
-  // Blocking support: waitForStop() sleeps on stop_cv_ until
-  // requestStop() sets stop_requested_.
-  std::mutex stop_mutex_;
-  std::condition_variable stop_cv_;
-  bool stop_requested_ = false;
-
-  // Set by tclExitHandler when `exit` is run on a worker thread.
-  bool exit_requested_ = false;
+
+  // Shared STA / Tcl_Eval serialization mutex (see tclMutex()).
+  // TclEvaluator::eval() and the STA-touching request handlers in
+  // request_handler.cpp lock this mutex.
+  std::mutex tcl_mutex_;
+
+  // Set by tclExitHandler / requestExit() when an exit is being
+  // requested from a non-main thread (browser tcl widget).
+  std::atomic<bool> exit_requested_{false};
+  // Set by requestStop() / web_server_stop_cmd from a worker.
+  std::atomic<bool> stop_requested_{false};
+
+  // Tcl thread id of the thread that called serve().  Used as the
+  // target for Tcl_ThreadQueueEvent / Tcl_ThreadAlert wakeups.
+  Tcl_ThreadId main_thread_id_ = nullptr;
+
+  // Level-1 command trace token; installed in serve(), removed in
+  // teardown().
+  Tcl_Trace trace_token_ = nullptr;
+
+  // Tcl_StackChannel handles + per-channel state for the stdout / stderr
+  // mirror.  The Tcl_Channel pointers are returned by Tcl_StackChannel
+  // and unstacked in teardown(); the void* fields point at heap-
+  // allocated WebChannelMirror structs that the channel type's
+  // outputProc reads as instance data.
+  Tcl_Channel mirror_stdout_chan_ = nullptr;
+  Tcl_Channel mirror_stderr_chan_ = nullptr;
+  void* mirror_stdout_data_ = nullptr;
+  void* mirror_stderr_data_ = nullptr;
 
   // Tcl command override: replaces `exit` while the server is running
   // so a worker-thread `exit` doesn't run Tcl_Exit (which would self-join

src/web/src/main.js

@@ -850,6 +850,13 @@ app.websocketManager.onPush = (msg) => {
         let text = msg.text;
         if (text.endsWith('\n')) text = text.slice(0, -1);
         if (text) tclAppend(text + '\n', '');
+    } else if (msg.type === 'console_input') {
+        // Top-level command executed on the main Tcl thread (script line
+        // or terminal-typed command), broadcast by the level-1 trace
+        // installed in WebServer::serve().  Render it like a
+        // browser-typed command so the browser console shows the same
+        // history as the user driving the process from elsewhere.
+        if (msg.text) tclAppend(`>>> ${msg.text}\n`, 'tcl-cmd');
     } else if (msg.type === 'shutdown') {
         // Server is stopping intentionally (web_server -stop).
         // Disable auto-reconnect and show a clear message. Note that

src/web/src/request_handler.cpp

@@ -6,6 +6,7 @@
 #include <algorithm>
 #include <any>
 #include <cmath>
+#include <condition_variable>
 #include <cstddef>
 #include <cstdint>
 #include <exception>
@@ -40,12 +41,109 @@
 #include "odb/dbTypes.h"
 #include "odb/geom.h"
 #include "request_dispatcher.h"
+#include "tcl.h"
+#include "tclDecls.h"
 #include "tile_generator.h"
 #include "timing_report.h"
 #include "utl/Logger.h"
 
 namespace web {
 
+//------------------------------------------------------------------------------
+// TclEvaluator::eval — main-thread Tcl_Eval with cross-thread marshalling
+//------------------------------------------------------------------------------
+
+namespace {
+
+struct EvalState
+{
+  std::string cmd;
+  TclEvaluator::Result result;
+  std::mutex m;
+  std::condition_variable cv;
+  bool done = false;
+  TclEvaluator* evaluator;
+};
+
+struct EvalEvent : Tcl_Event
+{
+  EvalState* state;
+};
+
+// Run the actual Tcl_Eval under the shared mutex, then drain any
+// buffered log output to clients so the response arrives in-order with
+// the log lines.  Used both as the direct fast-path on the main thread
+// and as the body of the queued event proc when called from a worker.
+//
+// Evaluates at the GLOBAL namespace (TCL_EVAL_GLOBAL) regardless of
+// the current Tcl call-frame.  Without that flag, when web_server is
+// invoked from the launching terminal, web_server_wait_cmd parks in
+// Tcl_DoOneEvent while *still inside the web_server proc body*; the
+// event proc's Tcl_Eval would inherit that proc's local scope and a
+// browser-typed `puts $x` would fail to find globals set at the
+// terminal prompt (Tcl proc scopes don't auto-fall-through to global).
+// Browser commands behave as if typed at the top-level REPL.
+void runEvalLocked(TclEvaluator& e,
+                   const std::string& cmd,
+                   TclEvaluator::Result& out)
+{
+  std::lock_guard<std::mutex> lock(e.mutex);
+  const int rc
+      = Tcl_EvalEx(e.interp, cmd.c_str(), /*numBytes=*/-1, TCL_EVAL_GLOBAL);
+  out.result = Tcl_GetStringResult(e.interp);
+  out.is_error = (rc != TCL_OK);
+  if (e.drain_output) {
+    e.drain_output();
+  }
+}
+
+// Tcl event proc invoked on the main thread.  Runs Tcl_Eval, fills
+// the worker's EvalState, signals the cv so the worker can return.
+extern "C" int evalEventProc(Tcl_Event* ev_, int /*flags*/)
+{
+  auto* ev = static_cast<EvalEvent*>(ev_);
+  EvalState* s = ev->state;
+  runEvalLocked(*s->evaluator, s->cmd, s->result);
+  {
+    std::lock_guard<std::mutex> lk(s->m);
+    s->done = true;
+  }
+  s->cv.notify_one();
+  return 1;
+}
+
+}  // namespace
+
+TclEvaluator::Result TclEvaluator::eval(const std::string& cmd)
+{
+  // Direct fast path when the caller is already on the main thread
+  // (or we don't have a known main thread, e.g. unit tests built
+  // without a running server).
+  if (main_thread_id == nullptr || Tcl_GetCurrentThread() == main_thread_id) {
+    Result r;
+    runEvalLocked(*this, cmd, r);
+    return r;
+  }
+
+  // Worker thread: Tcl 9 isolates per-thread interp state, so
+  // Tcl_Eval here would not see globals set on the main thread.
+  // Marshal the request to the main thread and wait for the result.
+  EvalState state;
+  state.cmd = cmd;
+  state.evaluator = this;
+
+  auto* ev = reinterpret_cast<EvalEvent*>(Tcl_Alloc(sizeof(EvalEvent)));
+  ev->proc = &evalEventProc;
+  ev->nextPtr = nullptr;
+  ev->state = &state;
+  Tcl_ThreadQueueEvent(main_thread_id, ev, TCL_QUEUE_TAIL);
+  Tcl_ThreadAlert(main_thread_id);
+
+  std::unique_lock<std::mutex> lk(state.m);
+  state.cv.wait(lk, [&] { return state.done; });
+  return state.result;
+}
+
 //------------------------------------------------------------------------------
 // ShapeCollector — a gui::Painter that collects rectangles from
 // descriptor->highlight() calls for use in tile rendering.