A few more cases for binary conformance tests. (#2500)

* A few more cases for binary conformance tests.

* over-encoded varints (encoded in more bytes than are necessary).
* truncated varints (>32 bits for 32-bit types).

* Fixed Python decoding bug with 32-bit varints.

* Fixed 1L -> 1LL for 32-bit platforms.

conformance/conformance_test.cc

@@ -109,13 +109,18 @@ string cat(const string& a, const string& b,
 // The maximum number of bytes that it takes to encode a 64-bit varint.
 #define VARINT_MAX_LEN 10
 
-size_t vencode64(uint64_t val, char *buf) {
+size_t vencode64(uint64_t val, int over_encoded_bytes, char *buf) {
   if (val == 0) { buf[0] = 0; return 1; }
   size_t i = 0;
   while (val) {
     uint8_t byte = val & 0x7fU;
     val >>= 7;
-    if (val) byte |= 0x80U;
+    if (val || over_encoded_bytes) byte |= 0x80U;
+    buf[i++] = byte;
+  }
+  while (over_encoded_bytes--) {
+    assert(i < 10);
+    uint8_t byte = over_encoded_bytes ? 0x80 : 0;
     buf[i++] = byte;
   }
   return i;
@@ -123,7 +128,15 @@ size_t vencode64(uint64_t val, char *buf) {
 
 string varint(uint64_t x) {
   char buf[VARINT_MAX_LEN];
-  size_t len = vencode64(x, buf);
+  size_t len = vencode64(x, 0, buf);
+  return string(buf, len);
+}
+
+// Encodes a varint that is |extra| bytes longer than it needs to be, but still
+// valid.
+string longvarint(uint64_t x, int extra) {
+  char buf[VARINT_MAX_LEN];
+  size_t len = vencode64(x, extra, buf);
   return string(buf, len);
 }
 
@@ -744,13 +757,23 @@ bool ConformanceTestSuite::RunSuite(ConformanceTestRunner* runner,
   });
   TestValidDataForType(FieldDescriptor::TYPE_INT32, {
     {varint(12345), "12345"},
+    {longvarint(12345, 2), "12345"},
+    {longvarint(12345, 7), "12345"},
     {varint(kInt32Max), std::to_string(kInt32Max)},
     {varint(kInt32Min), std::to_string(kInt32Min)},
+    {varint(1LL << 33), std::to_string(static_cast<int32>(1LL << 33))},
+    {varint((1LL << 33) - 1),
+     std::to_string(static_cast<int32>((1LL << 33) - 1))},
   });
   TestValidDataForType(FieldDescriptor::TYPE_UINT32, {
     {varint(12345), "12345"},
+    {longvarint(12345, 2), "12345"},
+    {longvarint(12345, 7), "12345"},
     {varint(kUint32Max), std::to_string(kUint32Max)},  // UINT32_MAX
-    {varint(0), "0"}
+    {varint(0), "0"},
+    {varint(1LL << 33), std::to_string(static_cast<uint32>(1LL << 33))},
+    {varint((1LL << 33) - 1),
+     std::to_string(static_cast<uint32>((1LL << 33) - 1))},
   });
   TestValidDataForType(FieldDescriptor::TYPE_FIXED64, {
     {u64(12345), "12345"},

python/google/protobuf/internal/decoder.py

@@ -131,9 +131,12 @@ def _VarintDecoder(mask, result_type):
   return DecodeVarint
 
 
-def _SignedVarintDecoder(mask, result_type):
+def _SignedVarintDecoder(bits, result_type):
   """Like _VarintDecoder() but decodes signed values."""
 
+  signbit = 1 << (bits - 1)
+  mask = (1 << bits) - 1
+
   def DecodeVarint(buffer, pos):
     result = 0
     shift = 0
@@ -142,11 +145,8 @@ def _SignedVarintDecoder(mask, result_type):
       result |= ((b & 0x7f) << shift)
       pos += 1
       if not (b & 0x80):
-        if result > 0x7fffffffffffffff:
+        result &= mask
-          result -= (1 << 64)
+        result = (result ^ signbit) - signbit
-          result |= ~mask
-        else:
-          result &= mask
         result = result_type(result)
         return (result, pos)
       shift += 7
@@ -159,11 +159,11 @@ def _SignedVarintDecoder(mask, result_type):
 # (e.g. the C++ implementation) simpler.
 
 _DecodeVarint = _VarintDecoder((1 << 64) - 1, long)
-_DecodeSignedVarint = _SignedVarintDecoder((1 << 64) - 1, long)
+_DecodeSignedVarint = _SignedVarintDecoder(64, long)
 
 # Use these versions for values which must be limited to 32 bits.
 _DecodeVarint32 = _VarintDecoder((1 << 32) - 1, int)
-_DecodeSignedVarint32 = _SignedVarintDecoder((1 << 32) - 1, int)
+_DecodeSignedVarint32 = _SignedVarintDecoder(32, int)
 
 
 def ReadTag(buffer, pos):