Add support for a non-terminal console mode

If a process is running under control of winpty, the output of the process is decorated with ESC sequences to control a terminal to print the process output nicely. In some environments however, the client showing the output to the user is not a full terminal emulation, the Eclipse CDT debug console view in example, and the ESC sequences are printed as output additional to the real process output. This commit is adding an API function to switch into a mode where winpty is not decorating the process output with ESC sequences. The console mode is designated to pass on the process output to the client as is.

agent/Agent.cc

@@ -191,6 +191,10 @@ void Agent::handlePacket(ReadBuffer &packet)
         else
             result = GetProcessId(m_childProcess);
         break;
+    case AgentMsg::SetConsoleMode:
+        m_terminal->setConsoleMode(packet.getInt());
+        result = 0;
+        break;
     default:
         trace("Unrecognized message, id:%d", type);
     }

agent/Terminal.cc

@@ -50,13 +50,22 @@ Terminal::Terminal(NamedPipe *output) :
     m_output(output),
     m_remoteLine(0),
     m_cursorHidden(false),
-    m_remoteColor(-1)
+    m_remoteColor(-1),
+    m_consoleMode(false)
 {
 }
 
+void Terminal::setConsoleMode(int mode)
+{
+    if (mode == 1)
+        m_consoleMode = true;
+    else
+        m_consoleMode = false;
+}
+
 void Terminal::reset(bool sendClearFirst, int newLine)
 {
-    if (sendClearFirst)
+    if (sendClearFirst && !m_consoleMode)
         m_output->write(CSI"1;1H"CSI"2J");
     m_remoteLine = newLine;
     m_cursorHidden = false;
@@ -70,7 +79,8 @@ void Terminal::sendLine(int line, CHAR_INFO *lineData, int width)
     moveTerminalToLine(line);
 
     // Erase in Line -- erase entire line.
-    m_output->write(CSI"2K");
+    if (!m_consoleMode)
+        m_output->write(CSI"2K");
 
     std::string termLine;
     termLine.reserve(width + 32);
@@ -125,7 +135,8 @@ void Terminal::sendLine(int line, CHAR_INFO *lineData, int width)
                     strcat(buffer, ";1");
             }
             strcat(buffer, "m");
-            termLine.append(buffer);
+            if (!m_consoleMode)
+                termLine.append(buffer);
             length = termLine.size();
             m_remoteColor = color;
         }
@@ -163,7 +174,8 @@ void Terminal::finishOutput(const std::pair<int, int> &newCursorPos)
         moveTerminalToLine(newCursorPos.second);
         char buffer[32];
         sprintf(buffer, CSI"%dG"CSI"?25h", newCursorPos.first + 1);
-        m_output->write(buffer);
+        if (!m_consoleMode)
+            m_output->write(buffer);
         m_cursorHidden = false;
     }
     m_cursorPos = newCursorPos;
@@ -173,7 +185,8 @@ void Terminal::hideTerminalCursor()
 {
     if (m_cursorHidden)
         return;
-    m_output->write(CSI"?25l");
+    if (!m_consoleMode)
+        m_output->write(CSI"?25l");
     m_cursorHidden = true;
 }
 
@@ -188,11 +201,13 @@ void Terminal::moveTerminalToLine(int line)
         // CUrsor Up (CUU)
         char buffer[32];
         sprintf(buffer, "\r"CSI"%dA", m_remoteLine - line);
-        m_output->write(buffer);
+        if (!m_consoleMode)
+            m_output->write(buffer);
         m_remoteLine = line;
     } else if (line > m_remoteLine) {
         while (line > m_remoteLine) {
-            m_output->write("\r\n");
+            if (!m_consoleMode)
+                m_output->write("\r\n");
             m_remoteLine++;
         }
     } else {

agent/Terminal.h

@@ -34,6 +34,7 @@ public:
     void reset(bool sendClearFirst, int newLine);
     void sendLine(int line, CHAR_INFO *lineData, int width);
     void finishOutput(const std::pair<int, int> &newCursorPos);
+    void setConsoleMode(int mode);
 
 private:
     void hideTerminalCursor();
@@ -45,6 +46,7 @@ private:
     bool m_cursorHidden;
     std::pair<int, int> m_cursorPos;
     int m_remoteColor;
+    bool m_consoleMode;
 };
 
 #endif // TERMINAL_H

include/winpty.h

@@ -91,6 +91,11 @@ WINPTY_API HANDLE winpty_get_data_pipe(winpty_t *pc);
  */
 WINPTY_API int winpty_set_size(winpty_t *pc, int cols, int rows);
 
+/*
+ * Toggle the console mode. If in console mode, no terminal escape sequences are send.
+ */
+WINPTY_API int winpty_set_console_mode(winpty_t *pc, int mode);
+
 /*
  * Closes the winpty.
  */

libwinpty/winpty.cc

@@ -420,3 +420,12 @@ WINPTY_API void winpty_close(winpty_t *pc)
     CloseHandle(pc->dataPipe);
     delete pc;
 }
+
+WINPTY_API int winpty_set_console_mode(winpty_t *pc, int mode)
+{
+    WriteBuffer packet;
+    packet.putInt(AgentMsg::SetConsoleMode);
+    packet.putInt(mode);
+    writePacket(pc, packet);
+    return readInt32(pc);
+}

shared/AgentMsg.h

@@ -28,7 +28,8 @@ struct AgentMsg
         StartProcess,
         SetSize,
         GetExitCode,
-        GetProcessId
+        GetProcessId,
+        SetConsoleMode
     };
 };