WIP: Try to be more correct about 32-bit-vs-size_t/etc

src/agent/ConsoleInput.cc

@@ -80,7 +80,7 @@ const unsigned int kIncompleteEscapeTimeoutMs = 1000u;
         do {                                        \
             (out) = 0;                              \
             CHECK(isdigit(*pch));                   \
-            const char *begin = pch;                \
+            const char *const begin = pch;          \
             do {                                    \
                 CHECK(pch - begin + 1 < maxLen);    \
                 (out) = (out) * 10 + *pch - '0';    \
@@ -106,25 +106,25 @@ const unsigned int kIncompleteEscapeTimeoutMs = 1000u;
 // 0   no match
 // >0  match, returns length of match
 // -1  incomplete match
-static int matchDsr(const char *input, int inputSize)
+static int matchDsr(const char *input, size_t inputSize)
 {
     int32_t dummy = 0;
     const char *pch = input;
-    const char *stop = input + inputSize;
+    const char *const stop = input + inputSize;
     CHECK(*pch == '\x1B');  ADVANCE();
     CHECK(*pch == '[');     ADVANCE();
     SCAN_INT(dummy, 8);
     CHECK(*pch == ';');     ADVANCE();
     SCAN_INT(dummy, 8);
     CHECK(*pch == 'R');
-    return pch - input + 1;
+    return static_cast<int>(pch - input + 1);
 }
 
-static int matchMouseDefault(const char *input, int inputSize,
+static int matchMouseDefault(const char *input, size_t inputSize,
                              MouseRecord &out)
 {
     const char *pch = input;
-    const char *stop = input + inputSize;
+    const char *const stop = input + inputSize;
     CHECK(*pch == '\x1B');              ADVANCE();
     CHECK(*pch == '[');                 ADVANCE();
     CHECK(*pch == 'M');                 ADVANCE();
@@ -133,14 +133,15 @@ static int matchMouseDefault(const char *input, int inputSize,
     ADVANCE();
     out.coord.Y = (*pch - '!') & 0xFF;
     out.release = false;
-    return pch - input + 1;
+    return static_cast<int>(pch - input + 1);
 }
 
-static int matchMouse1006(const char *input, int inputSize, MouseRecord &out)
+static int matchMouse1006(const char *input, size_t inputSize,
+                          MouseRecord &out)
 {
     const char *pch = input;
-    const char *stop = input + inputSize;
-    int32_t temp;
+    const char *const stop = input + inputSize;
+    int32_t temp = 0;
     CHECK(*pch == '\x1B');      ADVANCE();
     CHECK(*pch == '[');         ADVANCE();
     CHECK(*pch == '<');         ADVANCE();
@@ -151,14 +152,15 @@ static int matchMouse1006(const char *input, int inputSize, MouseRecord &out)
     SCAN_SIGNED_INT(temp, 8); out.coord.Y = temp - 1;
     CHECK(*pch == 'M' || *pch == 'm');
     out.release = (*pch == 'm');
-    return pch - input + 1;
+    return static_cast<int>(pch - input + 1);
 }
 
-static int matchMouse1015(const char *input, int inputSize, MouseRecord &out)
+static int matchMouse1015(const char *input, size_t inputSize,
+                          MouseRecord &out)
 {
     const char *pch = input;
-    const char *stop = input + inputSize;
-    int32_t temp;
+    const char *const stop = input + inputSize;
+    int32_t temp = 0;
     CHECK(*pch == '\x1B');      ADVANCE();
     CHECK(*pch == '[');         ADVANCE();
     SCAN_INT(out.flags, 8); out.flags -= 32;
@@ -168,17 +170,18 @@ static int matchMouse1015(const char *input, int inputSize, MouseRecord &out)
     SCAN_SIGNED_INT(temp, 8); out.coord.Y = temp - 1;
     CHECK(*pch == 'M');
     out.release = false;
-    return pch - input + 1;
+    return static_cast<int>(pch - input + 1);
 }
 
 // Match a mouse input escape sequence of any kind.
 // 0   no match
 // >0  match, returns length of match
 // -1  incomplete match
-static int matchMouseRecord(const char *input, int inputSize, MouseRecord &out)
+static int matchMouseRecord(const char *input, size_t inputSize,
+                            MouseRecord &out)
 {
     memset(&out, 0, sizeof(out));
-    int ret;
+    int ret = 0;
     if ((ret = matchMouse1006(input, inputSize, out)) != 0) { return ret; }
     if ((ret = matchMouse1015(input, inputSize, out)) != 0) { return ret; }
     if ((ret = matchMouseDefault(input, inputSize, out)) != 0) { return ret; }
@@ -342,7 +345,12 @@ void ConsoleInput::doWrite(bool isEof)
     m_byteQueue.erase(0, idx);
     DWORD actual = 0;
     if (records.size() > 0) {
-        if (!WriteConsoleInputW(m_conin, records.data(), records.size(), &actual)) {
+        ASSERT(records.size() <= UINT32_MAX);
+        if (!WriteConsoleInputW(
+                m_conin,
+                records.data(),
+                static_cast<uint32_t>(records.size()),
+                &actual)) {
             trace("WriteConsoleInputW failed");
         }
     }
@@ -350,7 +358,7 @@ void ConsoleInput::doWrite(bool isEof)
 
 int ConsoleInput::scanInput(std::vector<INPUT_RECORD> &records,
                             const char *input,
-                            int inputSize,
+                            size_t inputSize,
                             bool isEof)
 {
     ASSERT(inputSize >= 1);
@@ -407,7 +415,7 @@ int ConsoleInput::scanInput(std::vector<INPUT_RECORD> &records,
     // than the DSR flushing mechanism or use a decrepit terminal.  The user
     // might be on a slow network connection.)
     if (input[0] == '\x1B' && inputSize >= 2 && input[1] != '\x1B') {
-        const int len = utf8CharLength(input[1]);
+        const unsigned int len = utf8CharLength(input[1]);
         if (len > 0) {
             if (1 + len > inputSize) {
                 // Incomplete character.
@@ -420,7 +428,7 @@ int ConsoleInput::scanInput(std::vector<INPUT_RECORD> &records,
     }
 
     // A UTF-8 character.
-    const int len = utf8CharLength(input[0]);
+    const unsigned int len = utf8CharLength(input[0]);
     if (len == 0) {
         static bool debugInput = isTracingEnabled() && hasDebugFlag("input");
         if (debugInput) {
@@ -440,9 +448,9 @@ int ConsoleInput::scanInput(std::vector<INPUT_RECORD> &records,
 
 int ConsoleInput::scanMouseInput(std::vector<INPUT_RECORD> &records,
                                  const char *input,
-                                 int inputSize)
+                                 size_t inputSize)
 {
-    MouseRecord record;
+    MouseRecord record = {};
     const int len = matchMouseRecord(input, inputSize, record);
     if (len <= 0) {
         return len;
@@ -553,7 +561,7 @@ int ConsoleInput::scanMouseInput(std::vector<INPUT_RECORD> &records,
 
 void ConsoleInput::appendUtf8Char(std::vector<INPUT_RECORD> &records,
                                   const char *charBuffer,
-                                  const int charLen,
+                                  const size_t charLen,
                                   const uint16_t keyState)
 {
     const uint32_t code = decodeUtf8(charBuffer);
@@ -562,7 +570,7 @@ void ConsoleInput::appendUtf8Char(std::vector<INPUT_RECORD> &records,
         if (debugInput) {
             StringBuilder error(64);
             error << "Discarding invalid UTF-8 sequence:";
-            for (int i = 0; i < charLen; ++i) {
+            for (size_t i = 0; i < charLen; ++i) {
                 error << ' ';
                 error << hexOfInt<true, uint8_t>(charBuffer[i]);
             }

src/agent/ConsoleInput.h

@@ -49,14 +49,14 @@ private:
     void doWrite(bool isEof);
     int scanInput(std::vector<INPUT_RECORD> &records,
                   const char *input,
-                  int inputSize,
+                  size_t inputSize,
                   bool isEof);
     int scanMouseInput(std::vector<INPUT_RECORD> &records,
                        const char *input,
-                       int inputSize);
+                       size_t inputSize);
     void appendUtf8Char(std::vector<INPUT_RECORD> &records,
                         const char *charBuffer,
-                        int charLen,
+                        size_t charLen,
                         uint16_t keyState);
     void appendKeyPress(std::vector<INPUT_RECORD> &records,
                         uint16_t virtualKey,

src/agent/InputMap.cc

@@ -186,7 +186,7 @@ InputMap::Node &InputMap::getOrCreateChild(Node &node, unsigned char ch) {
 }
 
 // Find the longest matching key and node.
-int InputMap::lookupKey(const char *input, int inputSize,
+int InputMap::lookupKey(const char *input, size_t inputSize,
                         Key &keyOut, bool &incompleteOut) const {
     keyOut = kKeyZero;
     incompleteOut = false;
@@ -195,7 +195,7 @@ int InputMap::lookupKey(const char *input, int inputSize,
     InputMap::Key longestMatch = kKeyZero;
     int longestMatchLen = 0;
 
-    for (int i = 0; i < inputSize; ++i) {
+    for (size_t i = 0; i < inputSize; ++i) {
         unsigned char ch = input[i];
         node = getChild(*node, ch);
         if (node == NULL) {

src/libwinpty/winpty.cc

@@ -20,6 +20,7 @@
 
 #include <windows.h>
 
+#include <limits.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <string.h>
@@ -297,7 +298,7 @@ connectControlPipe(winpty_t &wp) {
     }
 }
 
-static void writeData(winpty_t &wp, const void *data, size_t amount) {
+static void writeData(winpty_t &wp, const void *data, uint32_t amount) {
     // Perform a single pipe write.
     DWORD actual = 0;
     OVERLAPPED over = {};
@@ -321,12 +322,16 @@ static inline WriteBuffer newPacket() {
 }
 
 static void writePacket(winpty_t &wp, WriteBuffer &packet) {
+    // The protocol allows a "packet" of 2**64 bytes, but this implementation
+    // uses a single WriteFile call, which limits the packet size it can send.
+    // XXX: do this right...
     const auto &buf = packet.buf();
     packet.replaceRawValue<uint64_t>(0, buf.size());
-    writeData(wp, buf.data(), buf.size());
+    ASSERT(buf.size() <= UINT32_MAX);
+    writeData(wp, buf.data(), static_cast<uint32_t>(buf.size()));
 }
 
-static size_t readData(winpty_t &wp, void *data, size_t amount) {
+static uint32_t readData(winpty_t &wp, void *data, uint32_t amount) {
     DWORD actual = 0;
     OVERLAPPED over = {};
     over.hEvent = wp.ioEvent.get();
@@ -342,7 +347,9 @@ static size_t readData(winpty_t &wp, void *data, size_t amount) {
 
 static void readAll(winpty_t &wp, void *data, size_t amount) {
     while (amount > 0) {
-        const size_t chunk = readData(wp, data, amount);
+        const auto chunk =
+            readData(wp, data, 
+                static_cast<uint32_t>(std::min<size_t>(amount, 1024 * 1024)));
         ASSERT(chunk <= amount && "readData result is larger than amount");
         data = reinterpret_cast<char*>(data) + chunk;
         amount -= chunk;
@@ -357,11 +364,12 @@ static uint64_t readUInt64(winpty_t &wp) {
 
 // Returns a reply packet's payload.
 static ReadBuffer readPacket(winpty_t &wp) {
-    const uint64_t packetSize = readUInt64(wp);
+    const auto packetSize = readUInt64(wp);
     if (packetSize < sizeof(packetSize) || packetSize > SIZE_MAX) {
         throwWinptyException(L"Agent RPC error: invalid packet size");
     }
-    const size_t payloadSize = packetSize - sizeof(packetSize);
+    const auto payloadSize =
+        static_cast<size_t>(packetSize - sizeof(packetSize));
     std::vector<char> bytes(payloadSize);
     readAll(wp, bytes.data(), bytes.size());
     return ReadBuffer(std::move(bytes));

src/shared/StringUtil.cc

@@ -21,6 +21,7 @@
 #include "StringUtil.h"
 
 #include <windows.h>
+#include <limits.h>
 
 #include "WinptyAssert.h"
 
@@ -37,19 +38,20 @@ size_t winpty_wcsnlen(const wchar_t *s, size_t maxlen) {
 }
 
 std::string utf8FromWide(const std::wstring &input) {
-    int mblen = WideCharToMultiByte(
+    ASSERT(input.size() <= INT_MAX);
+    const int mblen = WideCharToMultiByte(
         CP_UTF8, 0,
-        input.data(), input.size(),
+        input.data(), static_cast<int>(input.size()),
         NULL, 0, NULL, NULL);
     if (mblen <= 0) {
-        return std::string();
+        return std::string(); // XXX: Better error handling.
     }
-    std::vector<char> tmp(mblen);
-    int mblen2 = WideCharToMultiByte(
+    std::vector<char> mbstr(mblen);
+    const int mblen2 = WideCharToMultiByte(
         CP_UTF8, 0,
-        input.data(), input.size(),
-        tmp.data(), tmp.size(),
+        input.data(), static_cast<int>(input.size()),
+        mbstr.data(), mblen,
         NULL, NULL);
     ASSERT(mblen2 == mblen);
-    return std::string(tmp.data(), tmp.size());
+    return std::string(mbstr.data(), mbstr.size());
 }