Provide a Ruby extension.

This adds a Ruby extension in ruby/ that is based on the 'upb' library
(now included as a submodule), and adds support for Ruby code generation
to the protoc compiler.

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "upb"]
+	path = upb
+	url = https://github.com/haberman/upb

Makefile.am

@@ -238,7 +238,18 @@ python_EXTRA_DIST=                                                           \
   python/stubout.py                                                          \
   python/README.txt
 
-all_EXTRA_DIST=$(java_EXTRA_DIST) $(python_EXTRA_DIST)
+ruby_EXTRA_DIST=                                                             \
+  ruby/ext/defs.c                                                            \
+  ruby/ext/encode_decode.c                                                   \
+  ruby/ext/extconf.rb                                                        \
+  ruby/ext/message.c                                                         \
+  ruby/ext/protobuf.c                                                        \
+  ruby/ext/protobuf.h                                                        \
+  ruby/ext/repeated_field.c                                                  \
+  ruby/ext/storage.c                                                         \
+  ruby/ext/test.rb
+
+all_EXTRA_DIST=$(java_EXTRA_DIST) $(python_EXTRA_DIST) $(ruby_EXTRA_DIST)
 
 EXTRA_DIST = $(@DIST_LANG@_EXTRA_DIST)                                       \
   autogen.sh                                                                 \

autogen.sh

@@ -37,5 +37,9 @@ sed -i -e 's/RuntimeLibrary="5"/RuntimeLibrary="3"/g;
 # TODO(kenton):  Remove the ",no-obsolete" part and fix the resulting warnings.
 autoreconf -f -i -Wall,no-obsolete
 
+# pull down git submodules.
+git submodule init
+git submodule update
+
 rm -rf autom4te.cache config.h.in~
 exit 0

configure.ac

@@ -23,7 +23,7 @@ AC_CONFIG_MACRO_DIR([m4])
 AC_ARG_VAR(DIST_LANG, [language to include in the distribution package (i.e., make dist)])
 case "$DIST_LANG" in
   "") DIST_LANG=cpp ;;
-  all | cpp | java | python | javanano) ;;
+  all | cpp | java | python | javanano | ruby) ;;
   *) AC_MSG_FAILURE([unknown language: $DIST_LANG]) ;;
 esac
 AC_SUBST(DIST_LANG)

ruby/README.md

@@ -0,0 +1,34 @@
+This directory contains the Ruby extension that implements Protocol Buffers
+functionality in Ruby.
+
+The Ruby extension makes use of generated Ruby code that defines message and
+enum types in a Ruby DSL. You may write definitions in this DSL directly, but
+we recommend using protoc's Ruby generation support with .proto files. The
+build process in this directory only installs the extension; you need to
+install protoc as well to have Ruby code generation functionality.
+
+Installation
+------------
+
+To build this Ruby extension, you will need:
+
+* Rake
+* Bundler
+* Ruby development headers
+* a C compiler
+* the upb submodule
+
+First, ensure that upb/ is checked out:
+
+    $ cd ..   # top level protobuf directory
+    $ git submodule init
+    $ git submodule update
+
+Then install the required Ruby gems:
+
+    $ sudo gem install bundler rake rake-compiler rspec rubygems-tasks
+
+Then build the Gem:
+
+    $ rake gem
+    $ gem install pkg/protobuf-$VERSION.gem

ruby/Rakefile

@@ -0,0 +1,37 @@
+require "rake/extensiontask"
+require "rake/testtask"
+
+spec = Gem::Specification.new do |s|
+  s.name        = "protobuf"
+  s.version     = "2.6.2"
+  s.licenses    = ["BSD"]
+  s.summary     = "Protocol Buffers"
+  s.description = "Protocol Buffers are Google's data interchange format."
+  s.authors     = ["Protobuf Authors"]
+  s.email       = "protobuf@googlegroups.com"
+
+  s.files       = ["lib/protobuf_c.so", "lib/protobuf.rb"]
+end
+
+Rake::ExtensionTask.new("protobuf_c", spec) do |ext|
+  ext.lib_dir = "lib"
+  ext.config_script = "extconf.rb"
+end
+
+Rake::TestTask.new(:test => :build) do |t|
+  t.test_files = FileList["tests/*.rb"]
+end
+
+task :chmod do
+  File.chmod(0755, "lib/protobuf_c.so")
+end
+
+Gem::PackageTask.new(spec) do |pkg|
+end
+task :package => :chmod
+task :gem => :chmod
+
+task :build => [:clean, :compile]
+task :default => [:build]
+
+# vim:sw=2:et

ruby/ext/protobuf_c/encode_decode.c

@@ -0,0 +1,755 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2014 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "protobuf.h"
+
+// -----------------------------------------------------------------------------
+// Parsing.
+// -----------------------------------------------------------------------------
+
+#define DEREF(msg, ofs, type) *(type*)(((uint8_t *)msg) + ofs)
+
+// Creates a handlerdata that simply contains the offset for this field.
+static const void* newhandlerdata(upb_handlers* h, uint32_t ofs) {
+  size_t* hd_ofs = ALLOC(size_t);
+  *hd_ofs = ofs;
+  upb_handlers_addcleanup(h, hd_ofs, free);
+  return hd_ofs;
+}
+
+typedef struct {
+  size_t ofs;
+  const upb_msgdef *md;
+} submsg_handlerdata_t;
+
+// Creates a handlerdata that contains offset and submessage type information.
+static const void *newsubmsghandlerdata(upb_handlers* h, uint32_t ofs,
+                                        const upb_fielddef* f) {
+  submsg_handlerdata_t *hd = ALLOC(submsg_handlerdata_t);
+  hd->ofs = ofs;
+  hd->md = upb_fielddef_msgsubdef(f);
+  upb_handlers_addcleanup(h, hd, free);
+  return hd;
+}
+
+// A handler that starts a repeated field.  Gets the Repeated*Field instance for
+// this field (such an instance always exists even in an empty message).
+static void *startseq_handler(void* closure, const void* hd) {
+  MessageHeader* msg = closure;
+  const size_t *ofs = hd;
+  return (void*)DEREF(Message_data(msg), *ofs, VALUE);
+}
+
+// Handlers that append primitive values to a repeated field (a regular Ruby
+// array for now).
+#define DEFINE_APPEND_HANDLER(type, ctype)                 \
+  static bool append##type##_handler(void *closure, const void *hd, \
+                                     ctype val) {                   \
+    VALUE ary = (VALUE)closure;                                     \
+    RepeatedField_push_native(ary, &val);                           \
+    return true;                                                    \
+  }
+
+DEFINE_APPEND_HANDLER(bool,   bool)
+DEFINE_APPEND_HANDLER(int32,  int32_t)
+DEFINE_APPEND_HANDLER(uint32, uint32_t)
+DEFINE_APPEND_HANDLER(float,  float)
+DEFINE_APPEND_HANDLER(int64,  int64_t)
+DEFINE_APPEND_HANDLER(uint64, uint64_t)
+DEFINE_APPEND_HANDLER(double, double)
+
+// Appends a string to a repeated field (a regular Ruby array for now).
+static void* appendstr_handler(void *closure,
+                               const void *hd,
+                               size_t size_hint) {
+  VALUE ary = (VALUE)closure;
+  VALUE str = rb_str_new2("");
+  rb_enc_associate(str, kRubyStringUtf8Encoding);
+  RepeatedField_push(ary, str);
+  return (void*)str;
+}
+
+// Appends a 'bytes' string to a repeated field (a regular Ruby array for now).
+static void* appendbytes_handler(void *closure,
+                                 const void *hd,
+                                 size_t size_hint) {
+  VALUE ary = (VALUE)closure;
+  VALUE str = rb_str_new2("");
+  rb_enc_associate(str, kRubyString8bitEncoding);
+  RepeatedField_push(ary, str);
+  return (void*)str;
+}
+
+// Sets a non-repeated string field in a message.
+static void* str_handler(void *closure,
+                         const void *hd,
+                         size_t size_hint) {
+  MessageHeader* msg = closure;
+  const size_t *ofs = hd;
+  VALUE str = rb_str_new2("");
+  rb_enc_associate(str, kRubyStringUtf8Encoding);
+  DEREF(Message_data(msg), *ofs, VALUE) = str;
+  return (void*)str;
+}
+
+// Sets a non-repeated 'bytes' field in a message.
+static void* bytes_handler(void *closure,
+                           const void *hd,
+                           size_t size_hint) {
+  MessageHeader* msg = closure;
+  const size_t *ofs = hd;
+  VALUE str = rb_str_new2("");
+  rb_enc_associate(str, kRubyString8bitEncoding);
+  DEREF(Message_data(msg), *ofs, VALUE) = str;
+  return (void*)str;
+}
+
+static size_t stringdata_handler(void* closure, const void* hd,
+                                 const char* str, size_t len,
+                                 const upb_bufhandle* handle) {
+  VALUE rb_str = (VALUE)closure;
+  rb_str_cat(rb_str, str, len);
+  return len;
+}
+
+// Appends a submessage to a repeated field (a regular Ruby array for now).
+static void *appendsubmsg_handler(void *closure, const void *hd) {
+  VALUE ary = (VALUE)closure;
+  const submsg_handlerdata_t *submsgdata = hd;
+  VALUE subdesc =
+      get_def_obj((void*)submsgdata->md);
+  VALUE subklass = Descriptor_msgclass(subdesc);
+
+  VALUE submsg_rb = rb_class_new_instance(0, NULL, subklass);
+  RepeatedField_push(ary, submsg_rb);
+
+  MessageHeader* submsg;
+  TypedData_Get_Struct(submsg_rb, MessageHeader, &Message_type, submsg);
+  return submsg;
+}
+
+// Sets a non-repeated submessage field in a message.
+static void *submsg_handler(void *closure, const void *hd) {
+  MessageHeader* msg = closure;
+  const submsg_handlerdata_t* submsgdata = hd;
+  VALUE subdesc =
+      get_def_obj((void*)submsgdata->md);
+  VALUE subklass = Descriptor_msgclass(subdesc);
+
+  if (DEREF(Message_data(msg), submsgdata->ofs, VALUE) == Qnil) {
+    DEREF(Message_data(msg), submsgdata->ofs, VALUE) =
+        rb_class_new_instance(0, NULL, subklass);
+  }
+
+  VALUE submsg_rb = DEREF(Message_data(msg), submsgdata->ofs, VALUE);
+  MessageHeader* submsg;
+  TypedData_Get_Struct(submsg_rb, MessageHeader, &Message_type, submsg);
+  return submsg;
+}
+
+static void add_handlers_for_message(const void *closure, upb_handlers *h) {
+  Descriptor* desc = ruby_to_Descriptor(
+      get_def_obj((void*)upb_handlers_msgdef(h)));
+  // Ensure layout exists. We may be invoked to create handlers for a given
+  // message if we are included as a submsg of another message type before our
+  // class is actually built, so to work around this, we just create the layout
+  // (and handlers, in the class-building function) on-demand.
+  if (desc->layout == NULL) {
+    desc->layout = create_layout(desc->msgdef);
+  }
+
+  upb_msg_iter i;
+
+  for (upb_msg_begin(&i, desc->msgdef);
+       !upb_msg_done(&i);
+       upb_msg_next(&i)) {
+    const upb_fielddef *f = upb_msg_iter_field(&i);
+    size_t offset = desc->layout->offsets[upb_fielddef_index(f)];
+
+    if (upb_fielddef_isseq(f)) {
+      upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
+      upb_handlerattr_sethandlerdata(&attr, newhandlerdata(h, offset));
+      upb_handlers_setstartseq(h, f, startseq_handler, &attr);
+      upb_handlerattr_uninit(&attr);
+
+      switch (upb_fielddef_type(f)) {
+
+#define SET_HANDLER(utype, ltype)                                 \
+  case utype:                                                     \
+    upb_handlers_set##ltype(h, f, append##ltype##_handler, NULL); \
+    break;
+
+        SET_HANDLER(UPB_TYPE_BOOL,   bool);
+        SET_HANDLER(UPB_TYPE_INT32,  int32);
+        SET_HANDLER(UPB_TYPE_UINT32, uint32);
+        SET_HANDLER(UPB_TYPE_ENUM,   int32);
+        SET_HANDLER(UPB_TYPE_FLOAT,  float);
+        SET_HANDLER(UPB_TYPE_INT64,  int64);
+        SET_HANDLER(UPB_TYPE_UINT64, uint64);
+        SET_HANDLER(UPB_TYPE_DOUBLE, double);
+
+#undef SET_HANDLER
+
+        case UPB_TYPE_STRING:
+        case UPB_TYPE_BYTES: {
+          bool is_bytes = upb_fielddef_type(f) == UPB_TYPE_BYTES;
+          upb_handlers_setstartstr(h, f, is_bytes ?
+                                   appendbytes_handler : appendstr_handler,
+                                   NULL);
+          upb_handlers_setstring(h, f, stringdata_handler, NULL);
+        }
+        case UPB_TYPE_MESSAGE: {
+          upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
+          upb_handlerattr_sethandlerdata(&attr, newsubmsghandlerdata(h, 0, f));
+          upb_handlers_setstartsubmsg(h, f, appendsubmsg_handler, &attr);
+          upb_handlerattr_uninit(&attr);
+          break;
+        }
+      }
+    }
+
+    switch (upb_fielddef_type(f)) {
+      case UPB_TYPE_BOOL:
+      case UPB_TYPE_INT32:
+      case UPB_TYPE_UINT32:
+      case UPB_TYPE_ENUM:
+      case UPB_TYPE_FLOAT:
+      case UPB_TYPE_INT64:
+      case UPB_TYPE_UINT64:
+      case UPB_TYPE_DOUBLE:
+        // The shim writes directly at the given offset (instead of using
+        // DEREF()) so we need to add the msg overhead.
+        upb_shim_set(h, f, offset + sizeof(MessageHeader), -1);
+        break;
+      case UPB_TYPE_STRING:
+      case UPB_TYPE_BYTES: {
+        bool is_bytes = upb_fielddef_type(f) == UPB_TYPE_BYTES;
+        upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
+        upb_handlerattr_sethandlerdata(&attr, newhandlerdata(h, offset));
+        upb_handlers_setstartstr(h, f,
+                                 is_bytes ? bytes_handler : str_handler,
+                                 &attr);
+        upb_handlers_setstring(h, f, stringdata_handler, &attr);
+        upb_handlerattr_uninit(&attr);
+        break;
+      }
+      case UPB_TYPE_MESSAGE: {
+        upb_handlerattr attr = UPB_HANDLERATTR_INITIALIZER;
+        upb_handlerattr_sethandlerdata(&attr, newsubmsghandlerdata(h, offset, f));
+        upb_handlers_setstartsubmsg(h, f, submsg_handler, &attr);
+        upb_handlerattr_uninit(&attr);
+        break;
+      }
+    }
+  }
+}
+
+// Creates upb handlers for populating a message.
+static const upb_handlers *new_fill_handlers(Descriptor* desc,
+                                             const void* owner) {
+  // TODO(cfallin, haberman): once upb gets a caching/memoization layer for
+  // handlers, reuse subdef handlers so that e.g. if we already parse
+  // B-with-field-of-type-C, we don't have to rebuild the whole hierarchy to
+  // parse A-with-field-of-type-B-with-field-of-type-C.
+  return upb_handlers_newfrozen(desc->msgdef, owner,
+                                add_handlers_for_message, NULL);
+}
+
+// Constructs the handlers for filling a message's data into an in-memory
+// object.
+const upb_handlers* get_fill_handlers(Descriptor* desc) {
+  if (!desc->fill_handlers) {
+    desc->fill_handlers =
+        new_fill_handlers(desc, &desc->fill_handlers);
+  }
+  return desc->fill_handlers;
+}
+
+// Constructs the upb decoder method for parsing messages of this type.
+// This is called from the message class creation code.
+const upb_pbdecodermethod *new_fillmsg_decodermethod(Descriptor* desc,
+                                                     const void* owner) {
+  const upb_handlers* handlers = get_fill_handlers(desc);
+  upb_pbdecodermethodopts opts;
+  upb_pbdecodermethodopts_init(&opts, handlers);
+
+  const upb_pbdecodermethod *ret = upb_pbdecodermethod_new(&opts, owner);
+  return ret;
+}
+
+static const upb_pbdecodermethod *msgdef_decodermethod(Descriptor* desc) {
+  if (desc->fill_method == NULL) {
+    desc->fill_method = new_fillmsg_decodermethod(
+        desc, &desc->fill_method);
+  }
+  return desc->fill_method;
+}
+
+/*
+ * call-seq:
+ *     MessageClass.decode(data) => message
+ *
+ * Decodes the given data (as a string containing bytes in protocol buffers wire
+ * format) under the interpretration given by this message class's definition
+ * and returns a message object with the corresponding field values.
+ */
+VALUE Message_decode(VALUE klass, VALUE data) {
+  VALUE descriptor = rb_iv_get(klass, kDescriptorInstanceVar);
+  Descriptor* desc = ruby_to_Descriptor(descriptor);
+  VALUE msgklass = Descriptor_msgclass(descriptor);
+
+  if (TYPE(data) != T_STRING) {
+    rb_raise(rb_eArgError, "Expected string for binary protobuf data.");
+  }
+
+  VALUE msg_rb = rb_class_new_instance(0, NULL, msgklass);
+  MessageHeader* msg;
+  TypedData_Get_Struct(msg_rb, MessageHeader, &Message_type, msg);
+
+  const upb_pbdecodermethod* method = msgdef_decodermethod(desc);
+  const upb_handlers* h = upb_pbdecodermethod_desthandlers(method);
+  upb_pbdecoder decoder;
+  upb_sink sink;
+  upb_status status = UPB_STATUS_INIT;
+
+  upb_pbdecoder_init(&decoder, method, &status);
+  upb_sink_reset(&sink, h, msg);
+  upb_pbdecoder_resetoutput(&decoder, &sink);
+  upb_bufsrc_putbuf(RSTRING_PTR(data), RSTRING_LEN(data),
+                    upb_pbdecoder_input(&decoder));
+
+  upb_pbdecoder_uninit(&decoder);
+  if (!upb_ok(&status)) {
+    rb_raise(rb_eRuntimeError, "Error occurred during parsing: %s.",
+             upb_status_errmsg(&status));
+  }
+
+  return msg_rb;
+}
+
+/*
+ * call-seq:
+ *     MessageClass.decode_json(data) => message
+ *
+ * Decodes the given data (as a string containing bytes in protocol buffers wire
+ * format) under the interpretration given by this message class's definition
+ * and returns a message object with the corresponding field values.
+ */
+VALUE Message_decode_json(VALUE klass, VALUE data) {
+  VALUE descriptor = rb_iv_get(klass, kDescriptorInstanceVar);
+  Descriptor* desc = ruby_to_Descriptor(descriptor);
+  VALUE msgklass = Descriptor_msgclass(descriptor);
+
+  if (TYPE(data) != T_STRING) {
+    rb_raise(rb_eArgError, "Expected string for JSON data.");
+  }
+  // TODO(cfallin): Check and respect string encoding. If not UTF-8, we need to
+  // convert, because string handlers pass data directly to message string
+  // fields.
+
+  VALUE msg_rb = rb_class_new_instance(0, NULL, msgklass);
+  MessageHeader* msg;
+  TypedData_Get_Struct(msg_rb, MessageHeader, &Message_type, msg);
+
+  upb_status status = UPB_STATUS_INIT;
+  upb_json_parser parser;
+  upb_json_parser_init(&parser, &status);
+
+  upb_sink sink;
+  upb_sink_reset(&sink, get_fill_handlers(desc), msg);
+  upb_json_parser_resetoutput(&parser, &sink);
+  upb_bufsrc_putbuf(RSTRING_PTR(data), RSTRING_LEN(data),
+                    upb_json_parser_input(&parser));
+
+  upb_json_parser_uninit(&parser);
+  if (!upb_ok(&status)) {
+    rb_raise(rb_eRuntimeError, "Error occurred during parsing: %s.",
+             upb_status_errmsg(&status));
+  }
+
+  return msg_rb;
+}
+
+// -----------------------------------------------------------------------------
+// Serializing.
+// -----------------------------------------------------------------------------
+//
+// The code below also comes from upb's prototype Ruby binding, developed by
+// haberman@.
+
+/* stringsink *****************************************************************/
+
+// This should probably be factored into a common upb component.
+
+typedef struct {
+  upb_byteshandler handler;
+  upb_bytessink sink;
+  char *ptr;
+  size_t len, size;
+} stringsink;
+
+static void *stringsink_start(void *_sink, const void *hd, size_t size_hint) {
+  stringsink *sink = _sink;
+  sink->len = 0;
+  return sink;
+}
+
+static size_t stringsink_string(void *_sink, const void *hd, const char *ptr,
+                                size_t len, const upb_bufhandle *handle) {
+  UPB_UNUSED(hd);
+  UPB_UNUSED(handle);
+
+  stringsink *sink = _sink;
+  size_t new_size = sink->size;
+
+  while (sink->len + len > new_size) {
+    new_size *= 2;
+  }
+
+  if (new_size != sink->size) {
+    sink->ptr = realloc(sink->ptr, new_size);
+    sink->size = new_size;
+  }
+
+  memcpy(sink->ptr + sink->len, ptr, len);
+  sink->len += len;
+
+  return len;
+}
+
+void stringsink_init(stringsink *sink) {
+  upb_byteshandler_init(&sink->handler);
+  upb_byteshandler_setstartstr(&sink->handler, stringsink_start, NULL);
+  upb_byteshandler_setstring(&sink->handler, stringsink_string, NULL);
+
+  upb_bytessink_reset(&sink->sink, &sink->handler, sink);
+
+  sink->size = 32;
+  sink->ptr = malloc(sink->size);
+  sink->len = 0;
+}
+
+void stringsink_uninit(stringsink *sink) {
+  free(sink->ptr);
+}
+
+/* msgvisitor *****************************************************************/
+
+// TODO: If/when we support proto2 semantics in addition to the current proto3
+// semantics, which means that we have true field presence, we will want to
+// modify msgvisitor so that it emits all present fields rather than all
+// non-default-value fields.
+//
+// Likewise, when implementing JSON serialization, we may need to have a
+// 'verbose' mode that outputs all fields and a 'concise' mode that outputs only
+// those with non-default values.
+
+static void putmsg(VALUE msg, const Descriptor* desc,
+                   upb_sink *sink, int depth);
+
+static upb_selector_t getsel(const upb_fielddef *f, upb_handlertype_t type) {
+  upb_selector_t ret;
+  bool ok = upb_handlers_getselector(f, type, &ret);
+  UPB_ASSERT_VAR(ok, ok);
+  return ret;
+}
+
+static void putstr(VALUE str, const upb_fielddef *f, upb_sink *sink) {
+  if (str == Qnil) return;
+
+  assert(BUILTIN_TYPE(str) == RUBY_T_STRING);
+  upb_sink subsink;
+
+  // Ensure that the string has the correct encoding. We also check at field-set
+  // time, but the user may have mutated the string object since then.
+  native_slot_validate_string_encoding(upb_fielddef_type(f), str);
+
+  upb_sink_startstr(sink, getsel(f, UPB_HANDLER_STARTSTR), RSTRING_LEN(str),
+                    &subsink);
+  upb_sink_putstring(&subsink, getsel(f, UPB_HANDLER_STRING), RSTRING_PTR(str),
+                     RSTRING_LEN(str), NULL);
+  upb_sink_endstr(sink, getsel(f, UPB_HANDLER_ENDSTR));
+}
+
+static void putsubmsg(VALUE submsg, const upb_fielddef *f, upb_sink *sink,
+                      int depth) {
+  if (submsg == Qnil) return;
+
+  upb_sink subsink;
+  VALUE descriptor = rb_iv_get(submsg, kDescriptorInstanceVar);
+  Descriptor* subdesc = ruby_to_Descriptor(descriptor);
+
+  upb_sink_startsubmsg(sink, getsel(f, UPB_HANDLER_STARTSUBMSG), &subsink);
+  putmsg(submsg, subdesc, &subsink, depth + 1);
+  upb_sink_endsubmsg(sink, getsel(f, UPB_HANDLER_ENDSUBMSG));
+}
+
+static void putary(VALUE ary, const upb_fielddef *f, upb_sink *sink,
+                   int depth) {
+  if (ary == Qnil) return;
+
+  upb_sink subsink;
+
+  upb_sink_startseq(sink, getsel(f, UPB_HANDLER_STARTSEQ), &subsink);
+
+  upb_fieldtype_t type = upb_fielddef_type(f);
+  upb_selector_t sel = 0;
+  if (upb_fielddef_isprimitive(f)) {
+    sel = getsel(f, upb_handlers_getprimitivehandlertype(f));
+  }
+
+  int size = NUM2INT(RepeatedField_length(ary));
+  for (int i = 0; i < size; i++) {
+    void* memory = RepeatedField_index_native(ary, i);
+    switch (type) {
+#define T(upbtypeconst, upbtype, ctype)                         \
+  case upbtypeconst:                                            \
+    upb_sink_put##upbtype(&subsink, sel, *((ctype *)memory));   \
+    break;
+
+      T(UPB_TYPE_FLOAT,  float,  float)
+      T(UPB_TYPE_DOUBLE, double, double)
+      T(UPB_TYPE_BOOL,   bool,   int8_t)
+      case UPB_TYPE_ENUM:
+      T(UPB_TYPE_INT32,  int32,  int32_t)
+      T(UPB_TYPE_UINT32, uint32, uint32_t)
+      T(UPB_TYPE_INT64,  int64,  int64_t)
+      T(UPB_TYPE_UINT64, uint64, uint64_t)
+
+      case UPB_TYPE_STRING:
+      case UPB_TYPE_BYTES:
+        putstr(*((VALUE *)memory), f, &subsink);
+        break;
+      case UPB_TYPE_MESSAGE:
+        putsubmsg(*((VALUE *)memory), f, &subsink, depth);
+        break;
+
+#undef T
+
+    }
+  }
+  upb_sink_endseq(sink, getsel(f, UPB_HANDLER_ENDSEQ));
+}
+
+static void putmsg(VALUE msg_rb, const Descriptor* desc,
+                   upb_sink *sink, int depth) {
+  upb_sink_startmsg(sink);
+
+  // Protect against cycles (possible because users may freely reassign message
+  // and repeated fields) by imposing a maximum recursion depth.
+  if (depth > UPB_SINK_MAX_NESTING) {
+    rb_raise(rb_eRuntimeError,
+             "Maximum recursion depth exceeded during encoding.");
+  }
+
+  MessageHeader* msg;
+  TypedData_Get_Struct(msg_rb, MessageHeader, &Message_type, msg);
+  void* msg_data = Message_data(msg);
+
+  upb_msg_iter i;
+  for (upb_msg_begin(&i, desc->msgdef);
+       !upb_msg_done(&i);
+       upb_msg_next(&i)) {
+    upb_fielddef *f = upb_msg_iter_field(&i);
+    uint32_t offset = desc->layout->offsets[upb_fielddef_index(f)];
+
+    if (upb_fielddef_isseq(f)) {
+      VALUE ary = DEREF(msg_data, offset, VALUE);
+      if (ary != Qnil) {
+        putary(ary, f, sink, depth);
+      }
+    } else if (upb_fielddef_isstring(f)) {
+      VALUE str = DEREF(msg_data, offset, VALUE);
+      if (RSTRING_LEN(str) > 0) {
+        putstr(str, f, sink);
+      }
+    } else if (upb_fielddef_issubmsg(f)) {
+      putsubmsg(DEREF(msg_data, offset, VALUE), f, sink, depth);
+    } else {
+      upb_selector_t sel = getsel(f, upb_handlers_getprimitivehandlertype(f));
+
+#define T(upbtypeconst, upbtype, ctype, default_value)                \
+  case upbtypeconst: {                                                \
+      ctype value = DEREF(msg_data, offset, ctype);                   \
+      if (value != default_value) {                                   \
+        upb_sink_put##upbtype(sink, sel, value);                      \
+      }                                                               \
+    }                                                                 \
+    break;
+
+      switch (upb_fielddef_type(f)) {
+        T(UPB_TYPE_FLOAT,  float,  float, 0.0)
+        T(UPB_TYPE_DOUBLE, double, double, 0.0)
+        T(UPB_TYPE_BOOL,   bool,   uint8_t, 0)
+        case UPB_TYPE_ENUM:
+        T(UPB_TYPE_INT32,  int32,  int32_t, 0)
+        T(UPB_TYPE_UINT32, uint32, uint32_t, 0)
+        T(UPB_TYPE_INT64,  int64,  int64_t, 0)
+        T(UPB_TYPE_UINT64, uint64, uint64_t, 0)
+
+        case UPB_TYPE_STRING:
+        case UPB_TYPE_BYTES:
+        case UPB_TYPE_MESSAGE: rb_raise(rb_eRuntimeError, "Internal error.");
+      }
+
+#undef T
+
+    }
+  }
+
+  upb_status status;
+  upb_sink_endmsg(sink, &status);
+}
+
+static const upb_handlers* msgdef_pb_serialize_handlers(Descriptor* desc) {
+  if (desc->pb_serialize_handlers == NULL) {
+    desc->pb_serialize_handlers =
+        upb_pb_encoder_newhandlers(desc->msgdef, &desc->pb_serialize_handlers);
+  }
+  return desc->pb_serialize_handlers;
+}
+
+static const upb_handlers* msgdef_json_serialize_handlers(Descriptor* desc) {
+  if (desc->json_serialize_handlers == NULL) {
+    desc->json_serialize_handlers =
+        upb_json_printer_newhandlers(
+            desc->msgdef, &desc->json_serialize_handlers);
+  }
+  return desc->json_serialize_handlers;
+}
+
+/*
+ * call-seq:
+ *     MessageClass.encode(msg) => bytes
+ *
+ * Encodes the given message object to its serialized form in protocol buffers
+ * wire format.
+ */
+VALUE Message_encode(VALUE klass, VALUE msg_rb) {
+  VALUE descriptor = rb_iv_get(klass, kDescriptorInstanceVar);
+  Descriptor* desc = ruby_to_Descriptor(descriptor);
+
+  stringsink sink;
+  stringsink_init(&sink);
+
+  const upb_handlers* serialize_handlers =
+      msgdef_pb_serialize_handlers(desc);
+
+  upb_pb_encoder encoder;
+  upb_pb_encoder_init(&encoder, serialize_handlers);
+  upb_pb_encoder_resetoutput(&encoder, &sink.sink);
+
+  putmsg(msg_rb, desc, upb_pb_encoder_input(&encoder), 0);
+
+  VALUE ret = rb_str_new(sink.ptr, sink.len);
+
+  upb_pb_encoder_uninit(&encoder);
+  stringsink_uninit(&sink);
+
+  return ret;
+}
+
+/*
+ * call-seq:
+ *     MessageClass.encode_json(msg) => json_string
+ *
+ * Encodes the given message object into its serialized JSON representation.
+ */
+VALUE Message_encode_json(VALUE klass, VALUE msg_rb) {
+  VALUE descriptor = rb_iv_get(klass, kDescriptorInstanceVar);
+  Descriptor* desc = ruby_to_Descriptor(descriptor);
+
+  stringsink sink;
+  stringsink_init(&sink);
+
+  const upb_handlers* serialize_handlers =
+      msgdef_json_serialize_handlers(desc);
+
+  upb_json_printer printer;
+  upb_json_printer_init(&printer, serialize_handlers);
+  upb_json_printer_resetoutput(&printer, &sink.sink);
+
+  putmsg(msg_rb, desc, upb_json_printer_input(&printer), 0);
+
+  VALUE ret = rb_str_new(sink.ptr, sink.len);
+
+  upb_json_printer_uninit(&printer);
+  stringsink_uninit(&sink);
+
+  return ret;
+}
+
+/*
+ * call-seq:
+ *     Google::Protobuf.encode(msg) => bytes
+ *
+ * Encodes the given message object to protocol buffers wire format. This is an
+ * alternative to the #encode method on msg's class.
+ */
+VALUE Google_Protobuf_encode(VALUE self, VALUE msg_rb) {
+  VALUE klass = CLASS_OF(msg_rb);
+  return Message_encode(klass, msg_rb);
+}
+
+/*
+ * call-seq:
+ *     Google::Protobuf.encode_json(msg) => json_string
+ *
+ * Encodes the given message object to its JSON representation. This is an
+ * alternative to the #encode_json method on msg's class.
+ */
+VALUE Google_Protobuf_encode_json(VALUE self, VALUE msg_rb) {
+  VALUE klass = CLASS_OF(msg_rb);
+  return Message_encode_json(klass, msg_rb);
+}
+
+/*
+ * call-seq:
+ *     Google::Protobuf.decode(class, bytes) => msg
+ *
+ * Decodes the given bytes as protocol buffers wire format under the
+ * interpretation given by the given class's message definition. This is an
+ * alternative to the #decode method on the given class.
+ */
+VALUE Google_Protobuf_decode(VALUE self, VALUE klass, VALUE msg_rb) {
+  return Message_decode(klass, msg_rb);
+}
+
+/*
+ * call-seq:
+ *     Google::Protobuf.decode_json(class, json_string) => msg
+ *
+ * Decodes the given JSON string under the interpretation given by the given
+ * class's message definition. This is an alternative to the #decode_json method
+ * on the given class.
+ */
+VALUE Google_Protobuf_decode_json(VALUE self, VALUE klass, VALUE msg_rb) {
+  return Message_decode_json(klass, msg_rb);
+}

ruby/ext/protobuf_c/extconf.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/ruby
+
+require 'mkmf'
+
+upb_path = File.absolute_path(File.dirname($0)) + "/../../../upb"
+libs = ["upb_pic", "upb.pb_pic", "upb.json_pic"]
+system("cd #{upb_path}; make " + libs.map{|l| "lib/lib#{l}.a"}.join(" "))
+
+$CFLAGS += " -O3 -std=c99 -Wno-unused-function -DNDEBUG"
+
+find_header("upb/upb.h", upb_path) or
+  raise "Can't find upb headers"
+find_library("upb_pic", "upb_msgdef_new", upb_path + "/lib") or
+  raise "Can't find upb lib"
+find_library("upb.pb_pic", "upb_pbdecoder_init", upb_path + "/lib") or
+  raise "Can't find upb.pb lib"
+find_library("upb.json_pic", "upb_json_printer_init", upb_path + "/lib") or
+  raise "Can't find upb.pb lib"
+
+$objs = ["protobuf.o", "defs.o", "storage.o", "message.o",
+         "repeated_field.o", "encode_decode.o"]
+
+create_makefile("protobuf_c")

ruby/ext/protobuf_c/message.c

@@ -0,0 +1,463 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2014 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "protobuf.h"
+
+// -----------------------------------------------------------------------------
+// Class/module creation from msgdefs and enumdefs, respectively.
+// -----------------------------------------------------------------------------
+
+void* Message_data(void* msg) {
+  return ((uint8_t *)msg) + sizeof(MessageHeader);
+}
+
+void Message_mark(void* _self) {
+  MessageHeader* self = (MessageHeader *)_self;
+  layout_mark(self->descriptor->layout, Message_data(self));
+}
+
+void Message_free(void* self) {
+  xfree(self);
+}
+
+rb_data_type_t Message_type = {
+  "Message",
+  { Message_mark, Message_free, NULL },
+};
+
+VALUE Message_alloc(VALUE klass) {
+  VALUE descriptor = rb_iv_get(klass, kDescriptorInstanceVar);
+  Descriptor* desc = ruby_to_Descriptor(descriptor);
+  MessageHeader* msg = (MessageHeader*)ALLOC_N(
+      uint8_t, sizeof(MessageHeader) + desc->layout->size);
+  memset(Message_data(msg), 0, desc->layout->size);
+
+  // We wrap first so that everything in the message object is GC-rooted in case
+  // a collection happens during object creation in layout_init().
+  VALUE ret = TypedData_Wrap_Struct(klass, &Message_type, msg);
+  msg->descriptor = desc;
+  rb_iv_set(ret, kDescriptorInstanceVar, descriptor);
+
+  layout_init(desc->layout, Message_data(msg));
+
+  return ret;
+}
+
+/*
+ * call-seq:
+ *     Message.method_missing(*args)
+ *
+ * Provides accessors and setters for message fields according to their field
+ * names. For any field whose name does not conflict with a built-in method, an
+ * accessor is provided with the same name as the field, and a setter is
+ * provided with the name of the field plus the '=' suffix. Thus, given a
+ * message instance 'msg' with field 'foo', the following code is valid:
+ *
+ *     msg.foo = 42
+ *     puts msg.foo
+ */
+VALUE Message_method_missing(int argc, VALUE* argv, VALUE _self) {
+  MessageHeader* self;
+  TypedData_Get_Struct(_self, MessageHeader, &Message_type, self);
+  if (argc < 1) {
+    rb_raise(rb_eArgError, "Expected method name as first argument.");
+  }
+  VALUE method_name = argv[0];
+  if (!SYMBOL_P(method_name)) {
+    rb_raise(rb_eArgError, "Expected symbol as method name.");
+  }
+  VALUE method_str = rb_id2str(SYM2ID(method_name));
+  char* name = RSTRING_PTR(method_str);
+  size_t name_len = RSTRING_LEN(method_str);
+  bool setter = false;
+
+  // Setters have names that end in '='.
+  if (name[name_len - 1] == '=') {
+    setter = true;
+    name_len--;
+  }
+
+  const upb_fielddef* f = upb_msgdef_ntof(self->descriptor->msgdef,
+                                          name, name_len);
+
+  if (f == NULL) {
+    rb_raise(rb_eArgError, "Unknown field");
+  }
+
+  if (setter) {
+    if (argc < 2) {
+      rb_raise(rb_eArgError, "No value provided to setter.");
+    }
+    layout_set(self->descriptor->layout, Message_data(self), f, argv[1]);
+    return Qnil;
+  } else {
+    return layout_get(self->descriptor->layout, Message_data(self), f);
+  }
+}
+
+int Message_initialize_kwarg(VALUE key, VALUE val, VALUE _self) {
+  MessageHeader* self;
+  TypedData_Get_Struct(_self, MessageHeader, &Message_type, self);
+
+  if (!SYMBOL_P(key)) {
+    rb_raise(rb_eArgError,
+             "Expected symbols as hash keys in initialization map.");
+  }
+
+  VALUE method_str = rb_id2str(SYM2ID(key));
+  char* name = RSTRING_PTR(method_str);
+  const upb_fielddef* f = upb_msgdef_ntofz(self->descriptor->msgdef, name);
+  if (f == NULL) {
+    rb_raise(rb_eArgError,
+             "Unknown field name in initialization map entry.");
+  }
+
+  if (upb_fielddef_label(f) == UPB_LABEL_REPEATED) {
+    if (TYPE(val) != T_ARRAY) {
+      rb_raise(rb_eArgError,
+               "Expected array as initializer value for repeated field.");
+    }
+    VALUE ary = layout_get(self->descriptor->layout, Message_data(self), f);
+    for (int i = 0; i < RARRAY_LEN(val); i++) {
+      RepeatedField_push(ary, rb_ary_entry(val, i));
+    }
+  } else {
+    layout_set(self->descriptor->layout, Message_data(self), f, val);
+  }
+  return 0;
+}
+
+/*
+ * call-seq:
+ *     Message.new(kwargs) => new_message
+ *
+ * Creates a new instance of the given message class. Keyword arguments may be
+ * provided with keywords corresponding to field names.
+ *
+ * Note that no literal Message class exists. Only concrete classes per message
+ * type exist, as provided by the #msgclass method on Descriptors after they
+ * have been added to a pool. The method definitions described here on the
+ * Message class are provided on each concrete message class.
+ */
+VALUE Message_initialize(int argc, VALUE* argv, VALUE _self) {
+  if (argc == 0) {
+    return Qnil;
+  }
+  if (argc != 1) {
+    rb_raise(rb_eArgError, "Expected 0 or 1 arguments.");
+  }
+  VALUE hash_args = argv[0];
+  if (TYPE(hash_args) != T_HASH) {
+    rb_raise(rb_eArgError, "Expected hash arguments.");
+  }
+
+  rb_hash_foreach(hash_args, Message_initialize_kwarg, _self);
+  return Qnil;
+}
+
+/*
+ * call-seq:
+ *     Message.dup => new_message
+ *
+ * Performs a shallow copy of this message and returns the new copy.
+ */
+VALUE Message_dup(VALUE _self) {
+  MessageHeader* self;
+  TypedData_Get_Struct(_self, MessageHeader, &Message_type, self);
+
+  VALUE new_msg = rb_class_new_instance(0, NULL, CLASS_OF(_self));
+  MessageHeader* new_msg_self;
+  TypedData_Get_Struct(new_msg, MessageHeader, &Message_type, new_msg_self);
+
+  layout_dup(self->descriptor->layout,
+             Message_data(new_msg_self),
+             Message_data(self));
+
+  return new_msg;
+}
+
+// Internal only; used by Google::Protobuf.deep_copy.
+VALUE Message_deep_copy(VALUE _self) {
+  MessageHeader* self;
+  TypedData_Get_Struct(_self, MessageHeader, &Message_type, self);
+
+  VALUE new_msg = rb_class_new_instance(0, NULL, CLASS_OF(_self));
+  MessageHeader* new_msg_self;
+  TypedData_Get_Struct(new_msg, MessageHeader, &Message_type, new_msg_self);
+
+  layout_deep_copy(self->descriptor->layout,
+                   Message_data(new_msg_self),
+                   Message_data(self));
+
+  return new_msg;
+}
+
+/*
+ * call-seq:
+ *     Message.==(other) => boolean
+ *
+ * Performs a deep comparison of this message with another. Messages are equal
+ * if they have the same type and if each field is equal according to the :==
+ * method's semantics (a more efficient comparison may actually be done if the
+ * field is of a primitive type).
+ */
+VALUE Message_eq(VALUE _self, VALUE _other) {
+  MessageHeader* self;
+  TypedData_Get_Struct(_self, MessageHeader, &Message_type, self);
+
+  MessageHeader* other;
+  TypedData_Get_Struct(_other, MessageHeader, &Message_type, other);
+
+  if (self->descriptor != other->descriptor) {
+    return Qfalse;
+  }
+
+  return layout_eq(self->descriptor->layout,
+                   Message_data(self),
+                   Message_data(other));
+}
+
+/*
+ * call-seq:
+ *     Message.hash => hash_value
+ *
+ * Returns a hash value that represents this message's field values.
+ */
+VALUE Message_hash(VALUE _self) {
+  MessageHeader* self;
+  TypedData_Get_Struct(_self, MessageHeader, &Message_type, self);
+
+  return layout_hash(self->descriptor->layout, Message_data(self));
+}
+
+/*
+ * call-seq:
+ *     Message.inspect => string
+ *
+ * Returns a human-readable string representing this message. It will be
+ * formatted as "<MessageType: field1: value1, field2: value2, ...>". Each
+ * field's value is represented according to its own #inspect method.
+ */
+VALUE Message_inspect(VALUE _self) {
+  MessageHeader* self;
+  TypedData_Get_Struct(_self, MessageHeader, &Message_type, self);
+
+  VALUE str = rb_str_new2("<");
+  str = rb_str_append(str, rb_str_new2(rb_class2name(CLASS_OF(_self))));
+  str = rb_str_cat2(str, ": ");
+  str = rb_str_append(str, layout_inspect(
+      self->descriptor->layout, Message_data(self)));
+  str = rb_str_cat2(str, ">");
+  return str;
+}
+
+/*
+ * call-seq:
+ *     Message.[](index) => value
+ *
+ * Accesses a field's value by field name. The provided field name should be a
+ * string.
+ */
+VALUE Message_index(VALUE _self, VALUE field_name) {
+  MessageHeader* self;
+  TypedData_Get_Struct(_self, MessageHeader, &Message_type, self);
+  Check_Type(field_name, T_STRING);
+  const upb_fielddef* field =
+      upb_msgdef_ntofz(self->descriptor->msgdef, RSTRING_PTR(field_name));
+  if (field == NULL) {
+    return Qnil;
+  }
+  return layout_get(self->descriptor->layout, Message_data(self), field);
+}
+
+/*
+ * call-seq:
+ *     Message.[]=(index, value)
+ *
+ * Sets a field's value by field name. The provided field name should be a
+ * string.
+ */
+VALUE Message_index_set(VALUE _self, VALUE field_name, VALUE value) {
+  MessageHeader* self;
+  TypedData_Get_Struct(_self, MessageHeader, &Message_type, self);
+  Check_Type(field_name, T_STRING);
+  const upb_fielddef* field =
+      upb_msgdef_ntofz(self->descriptor->msgdef, RSTRING_PTR(field_name));
+  if (field == NULL) {
+    rb_raise(rb_eArgError, "Unknown field: %s", RSTRING_PTR(field_name));
+  }
+  layout_set(self->descriptor->layout, Message_data(self), field, value);
+  return Qnil;
+}
+
+/*
+ * call-seq:
+ *     Message.descriptor => descriptor
+ *
+ * Class method that returns the Descriptor instance corresponding to this
+ * message class's type.
+ */
+VALUE Message_descriptor(VALUE klass) {
+  return rb_iv_get(klass, kDescriptorInstanceVar);
+}
+
+VALUE build_class_from_descriptor(Descriptor* desc) {
+  if (desc->layout == NULL) {
+    desc->layout = create_layout(desc->msgdef);
+  }
+  if (desc->fill_method == NULL) {
+    desc->fill_method = new_fillmsg_decodermethod(desc, &desc->fill_method);
+  }
+
+  const char* name = upb_msgdef_fullname(desc->msgdef);
+  if (name == NULL) {
+    rb_raise(rb_eRuntimeError, "Descriptor does not have assigned name.");
+  }
+
+  VALUE klass = rb_define_class_id(
+      // Docs say this parameter is ignored. User will assign return value to
+      // their own toplevel constant class name.
+      rb_intern("Message"),
+      rb_cObject);
+  rb_iv_set(klass, kDescriptorInstanceVar, get_def_obj(desc->msgdef));
+  rb_define_alloc_func(klass, Message_alloc);
+  rb_define_method(klass, "method_missing",
+                   Message_method_missing, -1);
+  rb_define_method(klass, "initialize", Message_initialize, -1);
+  rb_define_method(klass, "dup", Message_dup, 0);
+  // Also define #clone so that we don't inherit Object#clone.
+  rb_define_method(klass, "clone", Message_dup, 0);
+  rb_define_method(klass, "==", Message_eq, 1);
+  rb_define_method(klass, "hash", Message_hash, 0);
+  rb_define_method(klass, "inspect", Message_inspect, 0);
+  rb_define_method(klass, "[]", Message_index, 1);
+  rb_define_method(klass, "[]=", Message_index_set, 2);
+  rb_define_singleton_method(klass, "decode", Message_decode, 1);
+  rb_define_singleton_method(klass, "encode", Message_encode, 1);
+  rb_define_singleton_method(klass, "decode_json", Message_decode_json, 1);
+  rb_define_singleton_method(klass, "encode_json", Message_encode_json, 1);
+  rb_define_singleton_method(klass, "descriptor", Message_descriptor, 0);
+  return klass;
+}
+
+/*
+ * call-seq:
+ *     Enum.lookup(number) => name
+ *
+ * This module method, provided on each generated enum module, looks up an enum
+ * value by number and returns its name as a Ruby symbol, or nil if not found.
+ */
+VALUE enum_lookup(VALUE self, VALUE number) {
+  int32_t num = NUM2INT(number);
+  VALUE desc = rb_iv_get(self, kDescriptorInstanceVar);
+  EnumDescriptor* enumdesc = ruby_to_EnumDescriptor(desc);
+
+  const char* name = upb_enumdef_iton(enumdesc->enumdef, num);
+  if (name == NULL) {
+    return Qnil;
+  } else {
+    return ID2SYM(rb_intern(name));
+  }
+}
+
+/*
+ * call-seq:
+ *     Enum.resolve(name) => number
+ *
+ * This module method, provided on each generated enum module, looks up an enum
+ * value by name (as a Ruby symbol) and returns its name, or nil if not found.
+ */
+VALUE enum_resolve(VALUE self, VALUE sym) {
+  const char* name = rb_id2name(SYM2ID(sym));
+  VALUE desc = rb_iv_get(self, kDescriptorInstanceVar);
+  EnumDescriptor* enumdesc = ruby_to_EnumDescriptor(desc);
+
+  int32_t num = 0;
+  bool found = upb_enumdef_ntoiz(enumdesc->enumdef, name, &num);
+  if (!found) {
+    return Qnil;
+  } else {
+    return INT2NUM(num);
+  }
+}
+
+/*
+ * call-seq:
+ *     Enum.descriptor
+ *
+ * This module method, provided on each generated enum module, returns the
+ * EnumDescriptor corresponding to this enum type.
+ */
+VALUE enum_descriptor(VALUE self) {
+  return rb_iv_get(self, kDescriptorInstanceVar);
+}
+
+VALUE build_module_from_enumdesc(EnumDescriptor* enumdesc) {
+  VALUE mod = rb_define_module_id(
+      rb_intern(upb_enumdef_fullname(enumdesc->enumdef)));
+
+  upb_enum_iter it;
+  for (upb_enum_begin(&it, enumdesc->enumdef);
+       !upb_enum_done(&it);
+       upb_enum_next(&it)) {
+    const char* name = upb_enum_iter_name(&it);
+    int32_t value = upb_enum_iter_number(&it);
+    if (name[0] < 'A' || name[0] > 'Z') {
+      rb_raise(rb_eTypeError,
+               "Enum value '%s' does not start with an uppercase letter "
+               "as is required for Ruby constants.",
+               name);
+    }
+    rb_define_const(mod, name, INT2NUM(value));
+  }
+
+  rb_define_singleton_method(mod, "lookup", enum_lookup, 1);
+  rb_define_singleton_method(mod, "resolve", enum_resolve, 1);
+  rb_define_singleton_method(mod, "descriptor", enum_descriptor, 0);
+  rb_iv_set(mod, kDescriptorInstanceVar, get_def_obj(enumdesc->enumdef));
+
+  return mod;
+}
+
+/*
+ * call-seq:
+ *     Google::Protobuf.deep_copy(obj) => copy_of_obj
+ *
+ * Performs a deep copy of either a RepeatedField instance or a message object,
+ * recursively copying its members.
+ */
+VALUE Google_Protobuf_deep_copy(VALUE self, VALUE obj) {
+  VALUE klass = CLASS_OF(obj);
+  if (klass == cRepeatedField) {
+    return RepeatedField_deep_copy(obj);
+  } else {
+    return Message_deep_copy(obj);
+  }
+}

ruby/ext/protobuf_c/protobuf.c

@@ -0,0 +1,102 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2014 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "protobuf.h"
+
+// -----------------------------------------------------------------------------
+// Global map from upb {msg,enum}defs to wrapper Descriptor/EnumDescriptor
+// instances.
+// -----------------------------------------------------------------------------
+
+// This is a hash table from def objects (encoded by converting pointers to
+// Ruby integers) to MessageDef/EnumDef instances (as Ruby values).
+VALUE upb_def_to_ruby_obj_map;
+
+void add_def_obj(const void* def, VALUE value) {
+  rb_hash_aset(upb_def_to_ruby_obj_map, ULL2NUM((intptr_t)def), value);
+}
+
+VALUE get_def_obj(const void* def) {
+  return rb_hash_aref(upb_def_to_ruby_obj_map, ULL2NUM((intptr_t)def));
+}
+
+// -----------------------------------------------------------------------------
+// Utilities.
+// -----------------------------------------------------------------------------
+
+// Raises a Ruby error if |status| is not OK, using its error message.
+void check_upb_status(const upb_status* status, const char* msg) {
+  if (!upb_ok(status)) {
+    rb_raise(rb_eRuntimeError, "%s: %s\n", msg, upb_status_errmsg(status));
+  }
+}
+
+// String encodings: we look these up once, at load time, and then cache them
+// here.
+rb_encoding* kRubyStringUtf8Encoding;
+rb_encoding* kRubyStringASCIIEncoding;
+rb_encoding* kRubyString8bitEncoding;
+
+// -----------------------------------------------------------------------------
+// Initialization/entry point.
+// -----------------------------------------------------------------------------
+
+// This must be named "Init_protobuf_c" because the Ruby module is named
+// "protobuf_c" -- the VM looks for this symbol in our .so.
+void Init_protobuf_c() {
+  VALUE google = rb_define_module("Google");
+  VALUE protobuf = rb_define_module_under(google, "Protobuf");
+  VALUE internal = rb_define_module_under(protobuf, "Internal");
+  DescriptorPool_register(protobuf);
+  Descriptor_register(protobuf);
+  FieldDescriptor_register(protobuf);
+  EnumDescriptor_register(protobuf);
+  MessageBuilderContext_register(internal);
+  EnumBuilderContext_register(internal);
+  Builder_register(internal);
+  RepeatedField_register(protobuf);
+
+  rb_define_singleton_method(protobuf, "encode", Google_Protobuf_encode, 1);
+  rb_define_singleton_method(protobuf, "decode", Google_Protobuf_decode, 2);
+  rb_define_singleton_method(protobuf, "encode_json",
+                             Google_Protobuf_encode_json, 1);
+  rb_define_singleton_method(protobuf, "decode_json",
+                             Google_Protobuf_decode_json, 2);
+
+  rb_define_singleton_method(protobuf, "deep_copy",
+                             Google_Protobuf_deep_copy, 1);
+
+  kRubyStringUtf8Encoding = rb_utf8_encoding();
+  kRubyStringASCIIEncoding = rb_usascii_encoding();
+  kRubyString8bitEncoding = rb_ascii8bit_encoding();
+
+  upb_def_to_ruby_obj_map = rb_hash_new();
+  rb_gc_register_address(&upb_def_to_ruby_obj_map);
+}

ruby/ext/protobuf_c/protobuf.h

@@ -0,0 +1,404 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2014 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef __GOOGLE_PROTOBUF_RUBY_PROTOBUF_H__
+#define __GOOGLE_PROTOBUF_RUBY_PROTOBUF_H__
+
+#include <ruby/ruby.h>
+#include <ruby/vm.h>
+#include <ruby/encoding.h>
+
+#include "upb/def.h"
+#include "upb/handlers.h"
+#include "upb/pb/decoder.h"
+#include "upb/pb/encoder.h"
+#include "upb/pb/glue.h"
+#include "upb/json/parser.h"
+#include "upb/json/printer.h"
+#include "upb/shim/shim.h"
+#include "upb/symtab.h"
+
+// Forward decls.
+struct DescriptorPool;
+struct Descriptor;
+struct FieldDescriptor;
+struct EnumDescriptor;
+struct MessageLayout;
+struct MessageHeader;
+struct MessageBuilderContext;
+struct EnumBuilderContext;
+struct Builder;
+
+typedef struct DescriptorPool DescriptorPool;
+typedef struct Descriptor Descriptor;
+typedef struct FieldDescriptor FieldDescriptor;
+typedef struct EnumDescriptor EnumDescriptor;
+typedef struct MessageLayout MessageLayout;
+typedef struct MessageHeader MessageHeader;
+typedef struct MessageBuilderContext MessageBuilderContext;
+typedef struct EnumBuilderContext EnumBuilderContext;
+typedef struct Builder Builder;
+
+/*
+ It can be a bit confusing how the C structs defined below and the Ruby
+ objects interact and hold references to each other. First, a few principles:
+
+ - Ruby's "TypedData" abstraction lets a Ruby VALUE hold a pointer to a C
+   struct (or arbitrary memory chunk), own it, and free it when collected.
+   Thus, each struct below will have a corresponding Ruby object
+   wrapping/owning it.
+
+ - To get back from an underlying upb {msg,enum}def to the Ruby object, we
+   keep a global hashmap, accessed by get_def_obj/add_def_obj below.
+
+ The in-memory structure is then something like:
+
+   Ruby                        |      upb
+                               |
+   DescriptorPool  ------------|-----------> upb_symtab____________________
+                               |                | (message types)          \
+                               |                v                           \
+   Descriptor   ---------------|-----------> upb_msgdef         (enum types)|
+    |--> msgclass              |                |   ^                       |
+    |    (dynamically built)   |                |   | (submsg fields)       |
+    |--> MessageLayout         |                |   |                       /
+    |--------------------------|> decoder method|   |                      /
+    \--------------------------|> serialize     |   |                     /
+                               |  handlers      v   |                    /
+   FieldDescriptor  -----------|-----------> upb_fielddef               /
+                               |                    |                  /
+                               |                    v (enum fields)   /
+   EnumDescriptor  ------------|-----------> upb_enumdef  <----------'
+                               |
+                               |
+               ^               |               \___/
+               `---------------|-----------------'    (get_def_obj map)
+ */
+
+// -----------------------------------------------------------------------------
+// Ruby class structure definitions.
+// -----------------------------------------------------------------------------
+
+struct DescriptorPool {
+  upb_symtab* symtab;
+};
+
+struct Descriptor {
+  const upb_msgdef* msgdef;
+  MessageLayout* layout;
+  VALUE klass;  // begins as nil
+  const upb_handlers* fill_handlers;
+  const upb_pbdecodermethod* fill_method;
+  const upb_handlers* pb_serialize_handlers;
+  const upb_handlers* json_serialize_handlers;
+};
+
+struct FieldDescriptor {
+  const upb_fielddef* fielddef;
+};
+
+struct EnumDescriptor {
+  const upb_enumdef* enumdef;
+  VALUE module;  // begins as nil
+};
+
+struct MessageBuilderContext {
+  VALUE descriptor;
+};
+
+struct EnumBuilderContext {
+  VALUE enumdesc;
+};
+
+struct Builder {
+  VALUE pending_list;
+  upb_def** defs;  // used only while finalizing
+};
+
+extern VALUE cDescriptorPool;
+extern VALUE cDescriptor;
+extern VALUE cFieldDescriptor;
+extern VALUE cEnumDescriptor;
+extern VALUE cMessageBuilderContext;
+extern VALUE cEnumBuilderContext;
+extern VALUE cBuilder;
+
+extern const char* kDescriptorInstanceVar;
+
+// We forward-declare all of the Ruby method implementations here because we
+// sometimes call the methods directly across .c files, rather than going
+// through Ruby's method dispatching (e.g. during message parse). It's cleaner
+// to keep the list of object methods together than to split them between
+// static-in-file definitions and header declarations.
+
+void DescriptorPool_mark(void* _self);
+void DescriptorPool_free(void* _self);
+VALUE DescriptorPool_alloc(VALUE klass);
+void DescriptorPool_register(VALUE module);
+DescriptorPool* ruby_to_DescriptorPool(VALUE value);
+VALUE DescriptorPool_add(VALUE _self, VALUE def);
+VALUE DescriptorPool_build(VALUE _self);
+VALUE DescriptorPool_lookup(VALUE _self, VALUE name);
+VALUE DescriptorPool_generated_pool(VALUE _self);
+
+void Descriptor_mark(void* _self);
+void Descriptor_free(void* _self);
+VALUE Descriptor_alloc(VALUE klass);
+void Descriptor_register(VALUE module);
+Descriptor* ruby_to_Descriptor(VALUE value);
+VALUE Descriptor_name(VALUE _self);
+VALUE Descriptor_name_set(VALUE _self, VALUE str);
+VALUE Descriptor_each(VALUE _self);
+VALUE Descriptor_lookup(VALUE _self, VALUE name);
+VALUE Descriptor_add_field(VALUE _self, VALUE obj);
+VALUE Descriptor_msgclass(VALUE _self);
+extern const rb_data_type_t _Descriptor_type;
+
+void FieldDescriptor_mark(void* _self);
+void FieldDescriptor_free(void* _self);
+VALUE FieldDescriptor_alloc(VALUE klass);
+void FieldDescriptor_register(VALUE module);
+FieldDescriptor* ruby_to_FieldDescriptor(VALUE value);
+VALUE FieldDescriptor_name(VALUE _self);
+VALUE FieldDescriptor_name_set(VALUE _self, VALUE str);
+VALUE FieldDescriptor_type(VALUE _self);
+VALUE FieldDescriptor_type_set(VALUE _self, VALUE type);
+VALUE FieldDescriptor_label(VALUE _self);
+VALUE FieldDescriptor_label_set(VALUE _self, VALUE label);
+VALUE FieldDescriptor_number(VALUE _self);
+VALUE FieldDescriptor_number_set(VALUE _self, VALUE number);
+VALUE FieldDescriptor_submsg_name(VALUE _self);
+VALUE FieldDescriptor_submsg_name_set(VALUE _self, VALUE value);
+VALUE FieldDescriptor_subtype(VALUE _self);
+VALUE FieldDescriptor_get(VALUE _self, VALUE msg_rb);
+VALUE FieldDescriptor_set(VALUE _self, VALUE msg_rb, VALUE value);
+upb_fieldtype_t ruby_to_fieldtype(VALUE type);
+VALUE fieldtype_to_ruby(upb_fieldtype_t type);
+
+void EnumDescriptor_mark(void* _self);
+void EnumDescriptor_free(void* _self);
+VALUE EnumDescriptor_alloc(VALUE klass);
+void EnumDescriptor_register(VALUE module);
+EnumDescriptor* ruby_to_EnumDescriptor(VALUE value);
+VALUE EnumDescriptor_name(VALUE _self);
+VALUE EnumDescriptor_name_set(VALUE _self, VALUE str);
+VALUE EnumDescriptor_add_value(VALUE _self, VALUE name, VALUE number);
+VALUE EnumDescriptor_lookup_name(VALUE _self, VALUE name);
+VALUE EnumDescriptor_lookup_value(VALUE _self, VALUE number);
+VALUE EnumDescriptor_each(VALUE _self);
+VALUE EnumDescriptor_enummodule(VALUE _self);
+extern const rb_data_type_t _EnumDescriptor_type;
+
+void MessageBuilderContext_mark(void* _self);
+void MessageBuilderContext_free(void* _self);
+VALUE MessageBuilderContext_alloc(VALUE klass);
+void MessageBuilderContext_register(VALUE module);
+MessageBuilderContext* ruby_to_MessageBuilderContext(VALUE value);
+VALUE MessageBuilderContext_initialize(VALUE _self, VALUE descriptor);
+VALUE MessageBuilderContext_optional(int argc, VALUE* argv, VALUE _self);
+VALUE MessageBuilderContext_required(int argc, VALUE* argv, VALUE _self);
+VALUE MessageBuilderContext_repeated(int argc, VALUE* argv, VALUE _self);
+
+void EnumBuilderContext_mark(void* _self);
+void EnumBuilderContext_free(void* _self);
+VALUE EnumBuilderContext_alloc(VALUE klass);
+void EnumBuilderContext_register(VALUE module);
+EnumBuilderContext* ruby_to_EnumBuilderContext(VALUE value);
+VALUE EnumBuilderContext_initialize(VALUE _self, VALUE enumdesc);
+VALUE EnumBuilderContext_value(VALUE _self, VALUE name, VALUE number);
+
+void Builder_mark(void* _self);
+void Builder_free(void* _self);
+VALUE Builder_alloc(VALUE klass);
+void Builder_register(VALUE module);
+Builder* ruby_to_Builder(VALUE value);
+VALUE Builder_add_message(VALUE _self, VALUE name);
+VALUE Builder_add_enum(VALUE _self, VALUE name);
+VALUE Builder_finalize_to_pool(VALUE _self, VALUE pool_rb);
+
+// -----------------------------------------------------------------------------
+// Native slot storage abstraction.
+// -----------------------------------------------------------------------------
+
+size_t native_slot_size(upb_fieldtype_t type);
+void native_slot_set(upb_fieldtype_t type,
+                     VALUE type_class,
+                     void* memory,
+                     VALUE value);
+VALUE native_slot_get(upb_fieldtype_t type,
+                      VALUE type_class,
+                      void* memory);
+void native_slot_init(upb_fieldtype_t type, void* memory);
+void native_slot_mark(upb_fieldtype_t type, void* memory);
+void native_slot_dup(upb_fieldtype_t type, void* to, void* from);
+void native_slot_deep_copy(upb_fieldtype_t type, void* to, void* from);
+bool native_slot_eq(upb_fieldtype_t type, void* mem1, void* mem2);
+
+void native_slot_validate_string_encoding(upb_fieldtype_t type, VALUE value);
+
+extern rb_encoding* kRubyStringUtf8Encoding;
+extern rb_encoding* kRubyStringASCIIEncoding;
+extern rb_encoding* kRubyString8bitEncoding;
+
+// -----------------------------------------------------------------------------
+// Repeated field container type.
+// -----------------------------------------------------------------------------
+
+typedef struct {
+  upb_fieldtype_t field_type;
+  VALUE field_type_class;
+  void* elements;
+  int size;
+  int capacity;
+} RepeatedField;
+
+void RepeatedField_mark(void* self);
+void RepeatedField_free(void* self);
+VALUE RepeatedField_alloc(VALUE klass);
+VALUE RepeatedField_init(int argc, VALUE* argv, VALUE self);
+void RepeatedField_register(VALUE module);
+
+extern const rb_data_type_t RepeatedField_type;
+extern VALUE cRepeatedField;
+
+RepeatedField* ruby_to_RepeatedField(VALUE value);
+
+void RepeatedField_register(VALUE module);
+VALUE RepeatedField_each(VALUE _self);
+VALUE RepeatedField_index(VALUE _self, VALUE _index);
+void* RepeatedField_index_native(VALUE _self, int index);
+VALUE RepeatedField_index_set(VALUE _self, VALUE _index, VALUE val);
+void RepeatedField_reserve(RepeatedField* self, int new_size);
+VALUE RepeatedField_push(VALUE _self, VALUE val);
+void RepeatedField_push_native(VALUE _self, void* data);
+VALUE RepeatedField_pop(VALUE _self);
+VALUE RepeatedField_insert(int argc, VALUE* argv, VALUE _self);
+VALUE RepeatedField_replace(VALUE _self, VALUE list);
+VALUE RepeatedField_clear(VALUE _self);
+VALUE RepeatedField_length(VALUE _self);
+VALUE RepeatedField_dup(VALUE _self);
+VALUE RepeatedField_deep_copy(VALUE _self);
+VALUE RepeatedField_eq(VALUE _self, VALUE _other);
+VALUE RepeatedField_hash(VALUE _self);
+VALUE RepeatedField_inspect(VALUE _self);
+VALUE RepeatedField_plus(VALUE _self, VALUE list);
+
+// -----------------------------------------------------------------------------
+// Message layout / storage.
+// -----------------------------------------------------------------------------
+
+struct MessageLayout {
+  const upb_msgdef* msgdef;
+  size_t* offsets;
+  size_t size;
+};
+
+MessageLayout* create_layout(const upb_msgdef* msgdef);
+void free_layout(MessageLayout* layout);
+VALUE layout_get(MessageLayout* layout,
+                 void* storage,
+                 const upb_fielddef* field);
+void layout_set(MessageLayout* layout,
+                void* storage,
+                const upb_fielddef* field,
+                VALUE val);
+void layout_init(MessageLayout* layout, void* storage);
+void layout_mark(MessageLayout* layout, void* storage);
+void layout_dup(MessageLayout* layout, void* to, void* from);
+void layout_deep_copy(MessageLayout* layout, void* to, void* from);
+VALUE layout_eq(MessageLayout* layout, void* msg1, void* msg2);
+VALUE layout_hash(MessageLayout* layout, void* storage);
+VALUE layout_inspect(MessageLayout* layout, void* storage);
+
+// -----------------------------------------------------------------------------
+// Message class creation.
+// -----------------------------------------------------------------------------
+
+struct MessageHeader {
+  Descriptor* descriptor;  // kept alive by self.class.descriptor reference.
+  // Data comes after this.
+};
+
+extern rb_data_type_t Message_type;
+
+VALUE build_class_from_descriptor(Descriptor* descriptor);
+void* Message_data(void* msg);
+void Message_mark(void* self);
+void Message_free(void* self);
+VALUE Message_alloc(VALUE klass);
+VALUE Message_method_missing(int argc, VALUE* argv, VALUE _self);
+VALUE Message_initialize(int argc, VALUE* argv, VALUE _self);
+VALUE Message_dup(VALUE _self);
+VALUE Message_deep_copy(VALUE _self);
+VALUE Message_eq(VALUE _self, VALUE _other);
+VALUE Message_hash(VALUE _self);
+VALUE Message_inspect(VALUE _self);
+VALUE Message_index(VALUE _self, VALUE field_name);
+VALUE Message_index_set(VALUE _self, VALUE field_name, VALUE value);
+VALUE Message_descriptor(VALUE klass);
+VALUE Message_decode(VALUE klass, VALUE data);
+VALUE Message_encode(VALUE klass, VALUE msg_rb);
+VALUE Message_decode_json(VALUE klass, VALUE data);
+VALUE Message_encode_json(VALUE klass, VALUE msg_rb);
+
+VALUE Google_Protobuf_encode(VALUE self, VALUE msg_rb);
+VALUE Google_Protobuf_decode(VALUE self, VALUE klass, VALUE msg_rb);
+VALUE Google_Protobuf_encode_json(VALUE self, VALUE msg_rb);
+VALUE Google_Protobuf_decode_json(VALUE self, VALUE klass, VALUE msg_rb);
+
+VALUE Google_Protobuf_deep_copy(VALUE self, VALUE obj);
+
+VALUE build_module_from_enumdesc(EnumDescriptor* enumdef);
+VALUE enum_lookup(VALUE self, VALUE number);
+VALUE enum_resolve(VALUE self, VALUE sym);
+
+const upb_pbdecodermethod *new_fillmsg_decodermethod(
+    Descriptor* descriptor, const void *owner);
+
+// -----------------------------------------------------------------------------
+// Global map from upb {msg,enum}defs to wrapper Descriptor/EnumDescriptor
+// instances.
+// -----------------------------------------------------------------------------
+void add_def_obj(const void* def, VALUE value);
+VALUE get_def_obj(const void* def);
+
+// -----------------------------------------------------------------------------
+// Utilities.
+// -----------------------------------------------------------------------------
+
+void check_upb_status(const upb_status* status, const char* msg);
+
+#define CHECK_UPB(code, msg) do {                                             \
+    upb_status status = UPB_STATUS_INIT;                                      \
+    code;                                                                     \
+    check_upb_status(&status, msg);                                           \
+} while (0)
+
+#endif  // __GOOGLE_PROTOBUF_RUBY_PROTOBUF_H__

ruby/ext/protobuf_c/repeated_field.c

@@ -0,0 +1,597 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2014 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "protobuf.h"
+
+// -----------------------------------------------------------------------------
+// Repeated field container type.
+// -----------------------------------------------------------------------------
+
+const rb_data_type_t RepeatedField_type = {
+  "Google::Protobuf::RepeatedField",
+  { RepeatedField_mark, RepeatedField_free, NULL },
+};
+
+VALUE cRepeatedField;
+
+RepeatedField* ruby_to_RepeatedField(VALUE _self) {
+  RepeatedField* self;
+  TypedData_Get_Struct(_self, RepeatedField, &RepeatedField_type, self);
+  return self;
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.each(&block)
+ *
+ * Invokes the block once for each element of the repeated field. RepeatedField
+ * also includes Enumerable; combined with this method, the repeated field thus
+ * acts like an ordinary Ruby sequence.
+ */
+VALUE RepeatedField_each(VALUE _self) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+  upb_fieldtype_t field_type = self->field_type;
+  VALUE field_type_class = self->field_type_class;
+  int element_size = native_slot_size(field_type);
+
+  size_t off = 0;
+  for (int i = 0; i < self->size; i++, off += element_size) {
+    void* memory = (void *) (((uint8_t *)self->elements) + off);
+    VALUE val = native_slot_get(field_type, field_type_class, memory);
+    rb_yield(val);
+  }
+  return Qnil;
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.[](index) => value
+ *
+ * Accesses the element at the given index. Throws an exception on out-of-bounds
+ * errors.
+ */
+VALUE RepeatedField_index(VALUE _self, VALUE _index) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+  int element_size = native_slot_size(self->field_type);
+  upb_fieldtype_t field_type = self->field_type;
+  VALUE field_type_class = self->field_type_class;
+
+  int index = NUM2INT(_index);
+  if (index < 0 || index >= self->size) {
+    rb_raise(rb_eRangeError, "Index out of range");
+  }
+
+  void* memory = (void *) (((uint8_t *)self->elements) + index * element_size);
+  return native_slot_get(field_type, field_type_class, memory);
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.[]=(index, value)
+ *
+ * Sets the element at the given index. On out-of-bounds assignments, extends
+ * the array and fills the hole (if any) with default values.
+ */
+VALUE RepeatedField_index_set(VALUE _self, VALUE _index, VALUE val) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+  upb_fieldtype_t field_type = self->field_type;
+  VALUE field_type_class = self->field_type_class;
+  int element_size = native_slot_size(field_type);
+
+  int index = NUM2INT(_index);
+  if (index < 0 || index >= (INT_MAX - 1)) {
+    rb_raise(rb_eRangeError, "Index out of range");
+  }
+  if (index >= self->size) {
+    RepeatedField_reserve(self, index + 1);
+    upb_fieldtype_t field_type = self->field_type;
+    int element_size = native_slot_size(field_type);
+    for (int i = self->size; i <= index; i++) {
+      void* elem = (void *)(((uint8_t *)self->elements) + i * element_size);
+      native_slot_init(field_type, elem);
+    }
+    self->size = index + 1;
+  }
+
+  void* memory = (void *) (((uint8_t *)self->elements) + index * element_size);
+  native_slot_set(field_type, field_type_class, memory, val);
+  return Qnil;
+}
+
+static int kInitialSize = 8;
+
+void RepeatedField_reserve(RepeatedField* self, int new_size) {
+  if (new_size <= self->capacity) {
+    return;
+  }
+  if (self->capacity == 0) {
+    self->capacity = kInitialSize;
+  }
+  while (self->capacity < new_size) {
+    self->capacity *= 2;
+  }
+  void* old_elems = self->elements;
+  int elem_size = native_slot_size(self->field_type);
+  self->elements = ALLOC_N(uint8_t, elem_size * self->capacity);
+  if (old_elems != NULL) {
+    memcpy(self->elements, old_elems, self->size * elem_size);
+    xfree(old_elems);
+  }
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.push(value)
+ *
+ * Adds a new element to the repeated field.
+ */
+VALUE RepeatedField_push(VALUE _self, VALUE val) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+  upb_fieldtype_t field_type = self->field_type;
+  int element_size = native_slot_size(field_type);
+  RepeatedField_reserve(self, self->size + 1);
+  int index = self->size;
+  void* memory = (void *) (((uint8_t *)self->elements) + index * element_size);
+  native_slot_set(field_type, self->field_type_class, memory, val);
+  // native_slot_set may raise an error; bump index only after set.
+  self->size++;
+  return _self;
+}
+
+// Used by parsing handlers.
+void RepeatedField_push_native(VALUE _self, void* data) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+  upb_fieldtype_t field_type = self->field_type;
+  int element_size = native_slot_size(field_type);
+  RepeatedField_reserve(self, self->size + 1);
+  int index = self->size;
+  void* memory = (void *) (((uint8_t *)self->elements) + index * element_size);
+  memcpy(memory, data, element_size);
+  self->size++;
+}
+
+void* RepeatedField_index_native(VALUE _self, int index) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+  upb_fieldtype_t field_type = self->field_type;
+  int element_size = native_slot_size(field_type);
+  return ((uint8_t *)self->elements) + index * element_size;
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.pop => value
+ *
+ * Removes the last element and returns it. Throws an exception if the repeated
+ * field is empty.
+ */
+VALUE RepeatedField_pop(VALUE _self) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+  upb_fieldtype_t field_type = self->field_type;
+  VALUE field_type_class = self->field_type_class;
+  int element_size = native_slot_size(field_type);
+  if (self->size == 0) {
+    rb_raise(rb_eRangeError, "Pop from empty repeated field is not allowed.");
+  }
+  int index = self->size - 1;
+  void* memory = (void *) (((uint8_t *)self->elements) + index * element_size);
+  VALUE ret = native_slot_get(field_type, field_type_class, memory);
+  self->size--;
+  return ret;
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.insert(*args)
+ *
+ * Pushes each arg in turn onto the end of the repeated field.
+ */
+VALUE RepeatedField_insert(int argc, VALUE* argv, VALUE _self) {
+  for (int i = 0; i < argc; i++) {
+    RepeatedField_push(_self, argv[i]);
+  }
+  return Qnil;
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.replace(list)
+ *
+ * Replaces the contents of the repeated field with the given list of elements.
+ */
+VALUE RepeatedField_replace(VALUE _self, VALUE list) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+  Check_Type(list, T_ARRAY);
+  self->size = 0;
+  for (int i = 0; i < RARRAY_LEN(list); i++) {
+    RepeatedField_push(_self, rb_ary_entry(list, i));
+  }
+  return Qnil;
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.clear
+ *
+ * Clears (removes all elements from) this repeated field.
+ */
+VALUE RepeatedField_clear(VALUE _self) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+  self->size = 0;
+  return Qnil;
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.length
+ *
+ * Returns the length of this repeated field.
+ */
+VALUE RepeatedField_length(VALUE _self) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+  return INT2NUM(self->size);
+}
+
+static VALUE RepeatedField_new_this_type(VALUE _self) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+  VALUE new_rptfield = Qnil;
+  VALUE element_type = fieldtype_to_ruby(self->field_type);
+  if (self->field_type_class != Qnil) {
+    new_rptfield = rb_funcall(CLASS_OF(_self), rb_intern("new"), 2,
+                              element_type, self->field_type_class);
+  } else {
+    new_rptfield = rb_funcall(CLASS_OF(_self), rb_intern("new"), 1,
+                              element_type);
+  }
+  return new_rptfield;
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.dup => repeated_field
+ *
+ * Duplicates this repeated field with a shallow copy. References to all
+ * non-primitive element objects (e.g., submessages) are shared.
+ */
+VALUE RepeatedField_dup(VALUE _self) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+  VALUE new_rptfield = RepeatedField_new_this_type(_self);
+  RepeatedField* new_rptfield_self = ruby_to_RepeatedField(new_rptfield);
+  RepeatedField_reserve(new_rptfield_self, self->size);
+  upb_fieldtype_t field_type = self->field_type;
+  size_t elem_size = native_slot_size(field_type);
+  size_t off = 0;
+  for (int i = 0; i < self->size; i++, off += elem_size) {
+    void* to_mem = (uint8_t *)new_rptfield_self->elements + off;
+    void* from_mem = (uint8_t *)self->elements + off;
+    native_slot_dup(field_type, to_mem, from_mem);
+    new_rptfield_self->size++;
+  }
+
+  return new_rptfield;
+}
+
+// Internal only: used by Google::Protobuf.deep_copy.
+VALUE RepeatedField_deep_copy(VALUE _self) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+  VALUE new_rptfield = RepeatedField_new_this_type(_self);
+  RepeatedField* new_rptfield_self = ruby_to_RepeatedField(new_rptfield);
+  RepeatedField_reserve(new_rptfield_self, self->size);
+  upb_fieldtype_t field_type = self->field_type;
+  size_t elem_size = native_slot_size(field_type);
+  size_t off = 0;
+  for (int i = 0; i < self->size; i++, off += elem_size) {
+    void* to_mem = (uint8_t *)new_rptfield_self->elements + off;
+    void* from_mem = (uint8_t *)self->elements + off;
+    native_slot_deep_copy(field_type, to_mem, from_mem);
+    new_rptfield_self->size++;
+  }
+
+  return new_rptfield;
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.==(other) => boolean
+ *
+ * Compares this repeated field to another. Repeated fields are equal if their
+ * element types are equal, their lengths are equal, and each element is equal.
+ * Elements are compared as per normal Ruby semantics, by calling their :==
+ * methods (or performing a more efficient comparison for primitive types).
+ */
+VALUE RepeatedField_eq(VALUE _self, VALUE _other) {
+  if (_self == _other) {
+    return Qtrue;
+  }
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+
+  // Inefficient but workable: to support comparison to a generic array, we
+  // build a temporary RepeatedField of our type.
+  if (TYPE(_other) == T_ARRAY) {
+    VALUE new_rptfield = RepeatedField_new_this_type(_self);
+    for (int i = 0; i < RARRAY_LEN(_other); i++) {
+      VALUE elem = rb_ary_entry(_other, i);
+      RepeatedField_push(new_rptfield, elem);
+    }
+    _other = new_rptfield;
+  }
+
+  RepeatedField* other = ruby_to_RepeatedField(_other);
+  if (self->field_type != other->field_type ||
+      self->field_type_class != other->field_type_class ||
+      self->size != other->size) {
+    return Qfalse;
+  }
+
+  upb_fieldtype_t field_type = self->field_type;
+  size_t elem_size = native_slot_size(field_type);
+  size_t off = 0;
+  for (int i = 0; i < self->size; i++, off += elem_size) {
+    void* self_mem = ((uint8_t *)self->elements) + off;
+    void* other_mem = ((uint8_t *)other->elements) + off;
+    if (!native_slot_eq(field_type, self_mem, other_mem)) {
+      return Qfalse;
+    }
+  }
+  return Qtrue;
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.hash => hash_value
+ *
+ * Returns a hash value computed from this repeated field's elements.
+ */
+VALUE RepeatedField_hash(VALUE _self) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+
+  VALUE hash = LL2NUM(0);
+
+  upb_fieldtype_t field_type = self->field_type;
+  VALUE field_type_class = self->field_type_class;
+  size_t elem_size = native_slot_size(field_type);
+  size_t off = 0;
+  for (int i = 0; i < self->size; i++, off += elem_size) {
+    void* mem = ((uint8_t *)self->elements) + off;
+    VALUE elem = native_slot_get(field_type, field_type_class, mem);
+    hash = rb_funcall(hash, rb_intern("<<"), 1, INT2NUM(2));
+    hash = rb_funcall(hash, rb_intern("^"), 1,
+                      rb_funcall(elem, rb_intern("hash"), 0));
+  }
+
+  return hash;
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.inspect => string
+ *
+ * Returns a string representing this repeated field's elements. It will be
+ * formated as "[<element>, <element>, ...]", with each element's string
+ * representation computed by its own #inspect method.
+ */
+VALUE RepeatedField_inspect(VALUE _self) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+
+  VALUE str = rb_str_new2("[");
+
+  bool first = true;
+
+  upb_fieldtype_t field_type = self->field_type;
+  VALUE field_type_class = self->field_type_class;
+  size_t elem_size = native_slot_size(field_type);
+  size_t off = 0;
+  for (int i = 0; i < self->size; i++, off += elem_size) {
+    void* mem = ((uint8_t *)self->elements) + off;
+    VALUE elem = native_slot_get(field_type, field_type_class, mem);
+    if (!first) {
+      str = rb_str_cat2(str, ", ");
+    } else {
+      first = false;
+    }
+    str = rb_str_append(str, rb_funcall(elem, rb_intern("inspect"), 0));
+  }
+
+  str = rb_str_cat2(str, "]");
+  return str;
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.+(other) => repeated field
+ *
+ * Returns a new repeated field that contains the concatenated list of this
+ * repeated field's elements and other's elements. The other (second) list may
+ * be either another repeated field or a Ruby array.
+ */
+VALUE RepeatedField_plus(VALUE _self, VALUE list) {
+  VALUE dupped = RepeatedField_dup(_self);
+
+  if (TYPE(list) == T_ARRAY) {
+    for (int i = 0; i < RARRAY_LEN(list); i++) {
+      VALUE elem = rb_ary_entry(list, i);
+      RepeatedField_push(dupped, elem);
+    }
+  } else if (RB_TYPE_P(list, T_DATA) && RTYPEDDATA_P(list) &&
+             RTYPEDDATA_TYPE(list) == &RepeatedField_type) {
+    RepeatedField* self = ruby_to_RepeatedField(_self);
+    RepeatedField* list_rptfield = ruby_to_RepeatedField(list);
+    if (self->field_type != list_rptfield->field_type ||
+        self->field_type_class != list_rptfield->field_type_class) {
+      rb_raise(rb_eArgError,
+               "Attempt to append RepeatedField with different element type.");
+    }
+    for (int i = 0; i < list_rptfield->size; i++) {
+      void* mem = RepeatedField_index_native(list, i);
+      RepeatedField_push_native(dupped, mem);
+    }
+  } else {
+    rb_raise(rb_eArgError, "Unknown type appending to RepeatedField");
+  }
+
+  return dupped;
+}
+
+static void validate_type_class(upb_fieldtype_t type, VALUE klass) {
+  if (rb_iv_get(klass, kDescriptorInstanceVar) == Qnil) {
+    rb_raise(rb_eArgError,
+             "Type class has no descriptor. Please pass a "
+             "class or enum as returned by the DescriptorPool.");
+  }
+  if (type == UPB_TYPE_MESSAGE) {
+    VALUE desc = rb_iv_get(klass, kDescriptorInstanceVar);
+    if (!RB_TYPE_P(desc, T_DATA) || !RTYPEDDATA_P(desc) ||
+        RTYPEDDATA_TYPE(desc) != &_Descriptor_type) {
+      rb_raise(rb_eArgError, "Descriptor has an incorrect type.");
+    }
+    if (rb_get_alloc_func(klass) != &Message_alloc) {
+      rb_raise(rb_eArgError,
+               "Message class was not returned by the DescriptorPool.");
+    }
+  } else if (type == UPB_TYPE_ENUM) {
+    VALUE enumdesc = rb_iv_get(klass, kDescriptorInstanceVar);
+    if (!RB_TYPE_P(enumdesc, T_DATA) || !RTYPEDDATA_P(enumdesc) ||
+        RTYPEDDATA_TYPE(enumdesc) != &_EnumDescriptor_type) {
+      rb_raise(rb_eArgError, "Descriptor has an incorrect type.");
+    }
+  }
+}
+
+void RepeatedField_init_args(int argc, VALUE* argv,
+                             VALUE _self) {
+  RepeatedField* self = ruby_to_RepeatedField(_self);
+  VALUE ary = Qnil;
+  if (argc < 1) {
+    rb_raise(rb_eArgError, "Expected at least 1 argument.");
+  }
+  self->field_type = ruby_to_fieldtype(argv[0]);
+
+  if (self->field_type == UPB_TYPE_MESSAGE ||
+      self->field_type == UPB_TYPE_ENUM) {
+    if (argc < 2) {
+      rb_raise(rb_eArgError, "Expected at least 2 arguments for message/enum.");
+    }
+    self->field_type_class = argv[1];
+    if (argc > 2) {
+      ary = argv[2];
+    }
+    validate_type_class(self->field_type, self->field_type_class);
+  } else {
+    if (argc > 2) {
+      rb_raise(rb_eArgError, "Too many arguments: expected 1 or 2.");
+    }
+    if (argc > 1) {
+      ary = argv[1];
+    }
+  }
+
+  if (ary != Qnil) {
+    if (!RB_TYPE_P(ary, T_ARRAY)) {
+      rb_raise(rb_eArgError, "Expected array as initialize argument");
+    }
+    for (int i = 0; i < RARRAY_LEN(ary); i++) {
+      RepeatedField_push(_self, rb_ary_entry(ary, i));
+    }
+  }
+}
+
+// Mark, free, alloc, init and class setup functions.
+
+void RepeatedField_mark(void* _self) {
+  RepeatedField* self = (RepeatedField*)_self;
+  rb_gc_mark(self->field_type_class);
+  upb_fieldtype_t field_type = self->field_type;
+  int element_size = native_slot_size(field_type);
+  for (int i = 0; i < self->size; i++) {
+    void* memory = (((uint8_t *)self->elements) + i * element_size);
+    native_slot_mark(self->field_type, memory);
+  }
+}
+
+void RepeatedField_free(void* _self) {
+  RepeatedField* self = (RepeatedField*)_self;
+  xfree(self->elements);
+  xfree(self);
+}
+
+/*
+ * call-seq:
+ *     RepeatedField.new(type, type_class = nil, initial_elems = [])
+ *
+ * Creates a new repeated field. The provided type must be a Ruby symbol, and
+ * can take on the same values as those accepted by FieldDescriptor#type=. If
+ * the type is :message or :enum, type_class must be non-nil, and must be the
+ * Ruby class or module returned by Descriptor#msgclass or
+ * EnumDescriptor#enummodule, respectively. An initial list of elements may also
+ * be provided.
+ */
+VALUE RepeatedField_alloc(VALUE klass) {
+  RepeatedField* self = ALLOC(RepeatedField);
+  self->elements = NULL;
+  self->size = 0;
+  self->capacity = 0;
+  self->field_type = -1;
+  self->field_type_class = Qnil;
+  VALUE ret = TypedData_Wrap_Struct(klass, &RepeatedField_type, self);
+  return ret;
+}
+
+VALUE RepeatedField_init(int argc, VALUE* argv, VALUE self) {
+  RepeatedField_init_args(argc, argv, self);
+  return Qnil;
+}
+
+void RepeatedField_register(VALUE module) {
+  VALUE klass = rb_define_class_under(
+      module, "RepeatedField", rb_cObject);
+  rb_define_alloc_func(klass, RepeatedField_alloc);
+  cRepeatedField = klass;
+  rb_gc_register_address(&cRepeatedField);
+
+  rb_define_method(klass, "initialize",
+                   RepeatedField_init, -1);
+  rb_define_method(klass, "each", RepeatedField_each, 0);
+  rb_define_method(klass, "[]", RepeatedField_index, 1);
+  rb_define_method(klass, "[]=", RepeatedField_index_set, 2);
+  rb_define_method(klass, "push", RepeatedField_push, 1);
+  rb_define_method(klass, "<<", RepeatedField_push, 1);
+  rb_define_method(klass, "pop", RepeatedField_pop, 0);
+  rb_define_method(klass, "insert", RepeatedField_insert, -1);
+  rb_define_method(klass, "replace", RepeatedField_replace, 1);
+  rb_define_method(klass, "clear", RepeatedField_clear, 0);
+  rb_define_method(klass, "length", RepeatedField_length, 0);
+  rb_define_method(klass, "dup", RepeatedField_dup, 0);
+  // Also define #clone so that we don't inherit Object#clone.
+  rb_define_method(klass, "clone", RepeatedField_dup, 0);
+  rb_define_method(klass, "==", RepeatedField_eq, 1);
+  rb_define_method(klass, "hash", RepeatedField_hash, 0);
+  rb_define_method(klass, "inspect", RepeatedField_inspect, 0);
+  rb_define_method(klass, "+", RepeatedField_plus, 1);
+  rb_include_module(klass, rb_mEnumerable);
+}

ruby/ext/protobuf_c/storage.c

@@ -0,0 +1,577 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2014 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "protobuf.h"
+
+#include <math.h>
+
+#include <ruby/encoding.h>
+
+// -----------------------------------------------------------------------------
+// Ruby <-> native slot management.
+// -----------------------------------------------------------------------------
+
+#define DEREF(memory, type) *(type*)(memory)
+
+size_t native_slot_size(upb_fieldtype_t type) {
+  switch (type) {
+    case UPB_TYPE_FLOAT:   return 4;
+    case UPB_TYPE_DOUBLE:  return 8;
+    case UPB_TYPE_BOOL:    return 1;
+    case UPB_TYPE_STRING:  return sizeof(VALUE);
+    case UPB_TYPE_BYTES:   return sizeof(VALUE);
+    case UPB_TYPE_MESSAGE: return sizeof(VALUE);
+    case UPB_TYPE_ENUM:    return 4;
+    case UPB_TYPE_INT32:   return 4;
+    case UPB_TYPE_INT64:   return 8;
+    case UPB_TYPE_UINT32:  return 4;
+    case UPB_TYPE_UINT64:  return 8;
+    default: return 0;
+  }
+}
+
+static void check_int_range_precision(upb_fieldtype_t type, VALUE val) {
+  // NUM2{INT,UINT,LL,ULL} macros do the appropriate range checks on upper
+  // bound; we just need to do precision checks (i.e., disallow rounding) and
+  // check for < 0 on unsigned types.
+  if (TYPE(val) == T_FLOAT) {
+    double dbl_val = NUM2DBL(val);
+    if (floor(dbl_val) != dbl_val) {
+      rb_raise(rb_eRangeError,
+               "Non-integral floating point value assigned to integer field.");
+    }
+  }
+  if (type == UPB_TYPE_UINT32 || type == UPB_TYPE_UINT64) {
+    if (NUM2DBL(val) < 0) {
+      rb_raise(rb_eRangeError,
+               "Assigning negative value to unsigned integer field.");
+    }
+  }
+}
+
+static bool is_ruby_num(VALUE value) {
+  return (TYPE(value) == T_FLOAT ||
+          TYPE(value) == T_FIXNUM ||
+          TYPE(value) == T_BIGNUM);
+}
+
+void native_slot_validate_string_encoding(upb_fieldtype_t type, VALUE value) {
+  bool bad_encoding = false;
+  rb_encoding* string_encoding = rb_enc_from_index(ENCODING_GET(value));
+  if (type == UPB_TYPE_STRING) {
+    bad_encoding =
+        string_encoding != kRubyStringUtf8Encoding &&
+        string_encoding != kRubyStringASCIIEncoding;
+  } else {
+    bad_encoding =
+        string_encoding != kRubyString8bitEncoding;
+  }
+  // Check that encoding is UTF-8 or ASCII (for string fields) or ASCII-8BIT
+  // (for bytes fields).
+  if (bad_encoding) {
+    rb_raise(rb_eTypeError, "Encoding for '%s' fields must be %s (was %s)",
+             (type == UPB_TYPE_STRING) ? "string" : "bytes",
+             (type == UPB_TYPE_STRING) ? "UTF-8 or ASCII" : "ASCII-8BIT",
+             rb_enc_name(string_encoding));
+  }
+}
+
+void native_slot_set(upb_fieldtype_t type, VALUE type_class,
+                     void* memory, VALUE value) {
+  switch (type) {
+    case UPB_TYPE_FLOAT:
+      if (!is_ruby_num(value)) {
+        rb_raise(rb_eTypeError, "Expected number type for float field.");
+      }
+      DEREF(memory, float) = NUM2DBL(value);
+      break;
+    case UPB_TYPE_DOUBLE:
+      if (!is_ruby_num(value)) {
+        rb_raise(rb_eTypeError, "Expected number type for double field.");
+      }
+      DEREF(memory, double) = NUM2DBL(value);
+      break;
+    case UPB_TYPE_BOOL: {
+      int8_t val = -1;
+      if (value == Qtrue) {
+        val = 1;
+      } else if (value == Qfalse) {
+        val = 0;
+      } else {
+        rb_raise(rb_eTypeError, "Invalid argument for boolean field.");
+      }
+      DEREF(memory, int8_t) = val;
+      break;
+    }
+    case UPB_TYPE_STRING:
+    case UPB_TYPE_BYTES: {
+      if (CLASS_OF(value) != rb_cString) {
+        rb_raise(rb_eTypeError, "Invalid argument for string field.");
+      }
+      native_slot_validate_string_encoding(type, value);
+      DEREF(memory, VALUE) = value;
+      break;
+    }
+    case UPB_TYPE_MESSAGE: {
+      if (CLASS_OF(value) != type_class) {
+        rb_raise(rb_eTypeError,
+                 "Invalid type %s to assign to submessage field.",
+                 rb_class2name(CLASS_OF(value)));
+      }
+      DEREF(memory, VALUE) = value;
+      break;
+    }
+    case UPB_TYPE_ENUM: {
+      if (!is_ruby_num(value) && TYPE(value) != T_SYMBOL) {
+        rb_raise(rb_eTypeError,
+                 "Expected number or symbol type for enum field.");
+      }
+      int32_t int_val = 0;
+      if (TYPE(value) == T_SYMBOL) {
+        // Ensure that the given symbol exists in the enum module.
+        VALUE lookup = rb_const_get(type_class, SYM2ID(value));
+        if (lookup == Qnil) {
+          rb_raise(rb_eRangeError, "Unknown symbol value for enum field.");
+        } else {
+          int_val = NUM2INT(lookup);
+        }
+      } else {
+        check_int_range_precision(UPB_TYPE_INT32, value);
+        int_val = NUM2INT(value);
+      }
+      DEREF(memory, int32_t) = int_val;
+      break;
+    }
+    case UPB_TYPE_INT32:
+    case UPB_TYPE_INT64:
+    case UPB_TYPE_UINT32:
+    case UPB_TYPE_UINT64:
+      if (!is_ruby_num(value)) {
+        rb_raise(rb_eTypeError, "Expected number type for integral field.");
+      }
+      check_int_range_precision(type, value);
+      switch (type) {
+      case UPB_TYPE_INT32:
+        DEREF(memory, int32_t) = NUM2INT(value);
+        break;
+      case UPB_TYPE_INT64:
+        DEREF(memory, int64_t) = NUM2LL(value);
+        break;
+      case UPB_TYPE_UINT32:
+        DEREF(memory, uint32_t) = NUM2UINT(value);
+        break;
+      case UPB_TYPE_UINT64:
+        DEREF(memory, uint64_t) = NUM2ULL(value);
+        break;
+      default:
+        break;
+      }
+      break;
+    default:
+      break;
+  }
+}
+
+VALUE native_slot_get(upb_fieldtype_t type, VALUE type_class, void* memory) {
+  switch (type) {
+    case UPB_TYPE_FLOAT:
+      return DBL2NUM(DEREF(memory, float));
+    case UPB_TYPE_DOUBLE:
+      return DBL2NUM(DEREF(memory, double));
+    case UPB_TYPE_BOOL:
+      return DEREF(memory, int8_t) ? Qtrue : Qfalse;
+    case UPB_TYPE_STRING:
+    case UPB_TYPE_BYTES:
+    case UPB_TYPE_MESSAGE:
+      return *((VALUE *)memory);
+    case UPB_TYPE_ENUM: {
+      int32_t val = DEREF(memory, int32_t);
+      VALUE symbol = enum_lookup(type_class, INT2NUM(val));
+      if (symbol == Qnil) {
+        return INT2NUM(val);
+      } else {
+        return symbol;
+      }
+    }
+    case UPB_TYPE_INT32:
+      return INT2NUM(DEREF(memory, int32_t));
+    case UPB_TYPE_INT64:
+      return LL2NUM(DEREF(memory, int64_t));
+    case UPB_TYPE_UINT32:
+      return UINT2NUM(DEREF(memory, uint32_t));
+    case UPB_TYPE_UINT64:
+      return ULL2NUM(DEREF(memory, uint64_t));
+    default:
+      return Qnil;
+  }
+}
+
+void native_slot_init(upb_fieldtype_t type, void* memory) {
+  switch (type) {
+    case UPB_TYPE_FLOAT:
+      DEREF(memory, float) = 0.0;
+      break;
+    case UPB_TYPE_DOUBLE:
+      DEREF(memory, double) = 0.0;
+      break;
+    case UPB_TYPE_BOOL:
+      DEREF(memory, int8_t) = 0;
+      break;
+    case UPB_TYPE_STRING:
+    case UPB_TYPE_BYTES:
+      // TODO(cfallin): set encoding appropriately
+      DEREF(memory, VALUE) = rb_str_new2("");
+      break;
+    case UPB_TYPE_MESSAGE:
+      DEREF(memory, VALUE) = Qnil;
+      break;
+    case UPB_TYPE_ENUM:
+    case UPB_TYPE_INT32:
+      DEREF(memory, int32_t) = 0;
+      break;
+    case UPB_TYPE_INT64:
+      DEREF(memory, int64_t) = 0;
+      break;
+    case UPB_TYPE_UINT32:
+      DEREF(memory, uint32_t) = 0;
+      break;
+    case UPB_TYPE_UINT64:
+      DEREF(memory, uint64_t) = 0;
+      break;
+    default:
+      break;
+  }
+}
+
+void native_slot_mark(upb_fieldtype_t type, void* memory) {
+  switch (type) {
+    case UPB_TYPE_STRING:
+    case UPB_TYPE_BYTES:
+    case UPB_TYPE_MESSAGE:
+      rb_gc_mark(DEREF(memory, VALUE));
+      break;
+    default:
+      break;
+  }
+}
+
+void native_slot_dup(upb_fieldtype_t type, void* to, void* from) {
+  memcpy(to, from, native_slot_size(type));
+}
+
+void native_slot_deep_copy(upb_fieldtype_t type, void* to, void* from) {
+  switch (type) {
+    case UPB_TYPE_STRING:
+    case UPB_TYPE_BYTES: {
+      VALUE from_val = DEREF(from, VALUE);
+      DEREF(to, VALUE) = (from_val != Qnil) ?
+          rb_funcall(from_val, rb_intern("dup"), 0) : Qnil;
+      break;
+    }
+    case UPB_TYPE_MESSAGE: {
+      VALUE from_val = DEREF(from, VALUE);
+      DEREF(to, VALUE) = (from_val != Qnil) ?
+          Message_deep_copy(from_val) : Qnil;
+      break;
+    }
+    default:
+      memcpy(to, from, native_slot_size(type));
+  }
+}
+
+bool native_slot_eq(upb_fieldtype_t type, void* mem1, void* mem2) {
+  switch (type) {
+    case UPB_TYPE_STRING:
+    case UPB_TYPE_BYTES:
+    case UPB_TYPE_MESSAGE: {
+      VALUE val1 = DEREF(mem1, VALUE);
+      VALUE val2 = DEREF(mem2, VALUE);
+      VALUE ret = rb_funcall(val1, rb_intern("=="), 1, val2);
+      return ret == Qtrue;
+    }
+    default:
+      return !memcmp(mem1, mem2, native_slot_size(type));
+  }
+}
+
+// -----------------------------------------------------------------------------
+// Memory layout management.
+// -----------------------------------------------------------------------------
+
+MessageLayout* create_layout(const upb_msgdef* msgdef) {
+  MessageLayout* layout = ALLOC(MessageLayout);
+  int nfields = upb_msgdef_numfields(msgdef);
+  layout->offsets = ALLOC_N(size_t, nfields);
+
+  upb_msg_iter it;
+  size_t off = 0;
+  for (upb_msg_begin(&it, msgdef); !upb_msg_done(&it); upb_msg_next(&it)) {
+    const upb_fielddef* field = upb_msg_iter_field(&it);
+    size_t field_size =
+        (upb_fielddef_label(field) == UPB_LABEL_REPEATED) ?
+        sizeof(VALUE) : native_slot_size(upb_fielddef_type(field));
+    // align current offset
+    off = (off + field_size - 1) & ~(field_size - 1);
+    layout->offsets[upb_fielddef_index(field)] = off;
+    off += field_size;
+  }
+
+  layout->size = off;
+
+  layout->msgdef = msgdef;
+  upb_msgdef_ref(layout->msgdef, &layout->msgdef);
+
+  return layout;
+}
+
+void free_layout(MessageLayout* layout) {
+  xfree(layout->offsets);
+  upb_msgdef_unref(layout->msgdef, &layout->msgdef);
+  xfree(layout);
+}
+
+static VALUE get_type_class(const upb_fielddef* field) {
+  VALUE type_class = Qnil;
+  if (upb_fielddef_type(field) == UPB_TYPE_MESSAGE) {
+    VALUE submsgdesc =
+        get_def_obj(upb_fielddef_subdef(field));
+    type_class = Descriptor_msgclass(submsgdesc);
+  } else if (upb_fielddef_type(field) == UPB_TYPE_ENUM) {
+    VALUE subenumdesc =
+        get_def_obj(upb_fielddef_subdef(field));
+    type_class = EnumDescriptor_enummodule(subenumdesc);
+  }
+  return type_class;
+}
+
+VALUE layout_get(MessageLayout* layout,
+                 void* storage,
+                 const upb_fielddef* field) {
+  void* memory = ((uint8_t *)storage) +
+      layout->offsets[upb_fielddef_index(field)];
+  if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
+    return *((VALUE *)memory);
+  } else {
+    return native_slot_get(upb_fielddef_type(field),
+                           get_type_class(field),
+                           memory);
+  }
+}
+
+static void check_repeated_field_type(VALUE val, const upb_fielddef* field) {
+  assert(upb_fielddef_label(field) == UPB_LABEL_REPEATED);
+
+  if (!RB_TYPE_P(val, T_DATA) || !RTYPEDDATA_P(val) ||
+      RTYPEDDATA_TYPE(val) != &RepeatedField_type) {
+    rb_raise(rb_eTypeError, "Expected repeated field array");
+  }
+
+  RepeatedField* self = ruby_to_RepeatedField(val);
+  if (self->field_type != upb_fielddef_type(field)) {
+    rb_raise(rb_eTypeError, "Repeated field array has wrong element type");
+  }
+
+  if (upb_fielddef_type(field) == UPB_TYPE_MESSAGE ||
+      upb_fielddef_type(field) == UPB_TYPE_ENUM) {
+    RepeatedField* self = ruby_to_RepeatedField(val);
+    if (self->field_type_class !=
+        get_def_obj(upb_fielddef_subdef(field))) {
+      rb_raise(rb_eTypeError,
+               "Repeated field array has wrong message/enum class");
+    }
+  }
+}
+
+void layout_set(MessageLayout* layout,
+                void* storage,
+                const upb_fielddef* field,
+                VALUE val) {
+  void* memory = ((uint8_t *)storage) +
+      layout->offsets[upb_fielddef_index(field)];
+  if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
+    check_repeated_field_type(val, field);
+    *((VALUE *)memory) = val;
+  } else {
+    native_slot_set(upb_fielddef_type(field), get_type_class(field),
+                    memory, val);
+  }
+}
+
+void layout_init(MessageLayout* layout,
+                 void* storage) {
+  upb_msg_iter it;
+  for (upb_msg_begin(&it, layout->msgdef);
+       !upb_msg_done(&it);
+       upb_msg_next(&it)) {
+    const upb_fielddef* field = upb_msg_iter_field(&it);
+    void* memory = ((uint8_t *)storage) +
+        layout->offsets[upb_fielddef_index(field)];
+
+    if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
+      VALUE ary = Qnil;
+      VALUE type_class = get_type_class(field);
+      if (type_class != Qnil) {
+        VALUE args[2] = {
+          fieldtype_to_ruby(upb_fielddef_type(field)),
+          type_class,
+        };
+        ary = rb_class_new_instance(2, args, cRepeatedField);
+      } else {
+        VALUE args[1] = { fieldtype_to_ruby(upb_fielddef_type(field)) };
+        ary = rb_class_new_instance(1, args, cRepeatedField);
+      }
+      *((VALUE *)memory) = ary;
+    } else {
+      native_slot_init(upb_fielddef_type(field), memory);
+    }
+  }
+}
+
+void layout_mark(MessageLayout* layout, void* storage) {
+  upb_msg_iter it;
+  for (upb_msg_begin(&it, layout->msgdef);
+       !upb_msg_done(&it);
+       upb_msg_next(&it)) {
+    const upb_fielddef* field = upb_msg_iter_field(&it);
+    void* memory = ((uint8_t *)storage) +
+        layout->offsets[upb_fielddef_index(field)];
+
+    if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
+      rb_gc_mark(*((VALUE *)memory));
+    } else {
+      native_slot_mark(upb_fielddef_type(field), memory);
+    }
+  }
+}
+
+void layout_dup(MessageLayout* layout, void* to, void* from) {
+  upb_msg_iter it;
+  for (upb_msg_begin(&it, layout->msgdef);
+       !upb_msg_done(&it);
+       upb_msg_next(&it)) {
+    const upb_fielddef* field = upb_msg_iter_field(&it);
+    void* to_memory = ((uint8_t *)to) +
+        layout->offsets[upb_fielddef_index(field)];
+    void* from_memory = ((uint8_t *)from) +
+        layout->offsets[upb_fielddef_index(field)];
+
+    if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
+      *((VALUE *)to_memory) = RepeatedField_dup(*((VALUE *)from_memory));
+    } else {
+      native_slot_dup(upb_fielddef_type(field), to_memory, from_memory);
+    }
+  }
+}
+
+void layout_deep_copy(MessageLayout* layout, void* to, void* from) {
+  upb_msg_iter it;
+  for (upb_msg_begin(&it, layout->msgdef);
+       !upb_msg_done(&it);
+       upb_msg_next(&it)) {
+    const upb_fielddef* field = upb_msg_iter_field(&it);
+    void* to_memory = ((uint8_t *)to) +
+        layout->offsets[upb_fielddef_index(field)];
+    void* from_memory = ((uint8_t *)from) +
+        layout->offsets[upb_fielddef_index(field)];
+
+    if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
+      *((VALUE *)to_memory) = RepeatedField_deep_copy(*((VALUE *)from_memory));
+    } else {
+      native_slot_deep_copy(upb_fielddef_type(field), to_memory, from_memory);
+    }
+  }
+}
+
+VALUE layout_eq(MessageLayout* layout, void* msg1, void* msg2) {
+  upb_msg_iter it;
+  for (upb_msg_begin(&it, layout->msgdef);
+       !upb_msg_done(&it);
+       upb_msg_next(&it)) {
+    const upb_fielddef* field = upb_msg_iter_field(&it);
+    void* msg1_memory = ((uint8_t *)msg1) +
+        layout->offsets[upb_fielddef_index(field)];
+    void* msg2_memory = ((uint8_t *)msg2) +
+        layout->offsets[upb_fielddef_index(field)];
+
+    if (upb_fielddef_label(field) == UPB_LABEL_REPEATED) {
+      if (RepeatedField_eq(*((VALUE *)msg1_memory),
+                           *((VALUE *)msg2_memory)) == Qfalse) {
+        return Qfalse;
+      }
+    } else {
+      if (!native_slot_eq(upb_fielddef_type(field),
+                          msg1_memory, msg2_memory)) {
+        return Qfalse;
+      }
+    }
+  }
+  return Qtrue;
+}
+
+VALUE layout_hash(MessageLayout* layout, void* storage) {
+  upb_msg_iter it;
+  st_index_t h = rb_hash_start(0);
+  VALUE hash_sym = rb_intern("hash");
+  for (upb_msg_begin(&it, layout->msgdef);
+       !upb_msg_done(&it);
+       upb_msg_next(&it)) {
+    const upb_fielddef* field = upb_msg_iter_field(&it);
+    VALUE field_val = layout_get(layout, storage, field);
+    h = rb_hash_uint(h, NUM2LONG(rb_funcall(field_val, hash_sym, 0)));
+  }
+  h = rb_hash_end(h);
+
+  return INT2FIX(h);
+}
+
+VALUE layout_inspect(MessageLayout* layout, void* storage) {
+  VALUE str = rb_str_new2("");
+
+  upb_msg_iter it;
+  bool first = true;
+  for (upb_msg_begin(&it, layout->msgdef);
+       !upb_msg_done(&it);
+       upb_msg_next(&it)) {
+    const upb_fielddef* field = upb_msg_iter_field(&it);
+    VALUE field_val = layout_get(layout, storage, field);
+
+    if (!first) {
+      str = rb_str_cat2(str, ", ");
+    } else {
+      first = false;
+    }
+    str = rb_str_cat2(str, upb_fielddef_name(field));
+    str = rb_str_cat2(str, ": ");
+
+    str = rb_str_append(str, rb_funcall(field_val, rb_intern("inspect"), 0));
+  }
+
+  return str;
+}

ruby/lib/protobuf.rb

@@ -0,0 +1,31 @@
+# Protocol Buffers - Google's data interchange format
+# Copyright 2008 Google Inc.  All rights reserved.
+# https://developers.google.com/protocol-buffers/
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+require 'protobuf_c'

ruby/tests/basic.rb

@@ -0,0 +1,633 @@
+#!/usr/bin/ruby
+
+require 'protobuf'
+require 'test/unit'
+
+# ------------- generated code --------------
+
+module BasicTest
+  pool = Google::Protobuf::DescriptorPool.new
+  pool.build do
+    add_message "TestMessage" do
+      optional :optional_int32,  :int32,        1
+      optional :optional_int64,  :int64,        2
+      optional :optional_uint32, :uint32,       3
+      optional :optional_uint64, :uint64,       4
+      optional :optional_bool,   :bool,         5
+      optional :optional_float,  :float,        6
+      optional :optional_double, :double,       7
+      optional :optional_string, :string,       8
+      optional :optional_bytes,  :bytes,        9
+      optional :optional_msg,    :message,      10, "TestMessage2"
+      optional :optional_enum,   :enum,         11, "TestEnum"
+
+      repeated :repeated_int32,  :int32,        12
+      repeated :repeated_int64,  :int64,        13
+      repeated :repeated_uint32, :uint32,       14
+      repeated :repeated_uint64, :uint64,       15
+      repeated :repeated_bool,   :bool,         16
+      repeated :repeated_float,  :float,        17
+      repeated :repeated_double, :double,       18
+      repeated :repeated_string, :string,       19
+      repeated :repeated_bytes,  :bytes,        20
+      repeated :repeated_msg,    :message,      21, "TestMessage2"
+      repeated :repeated_enum,   :enum,         22, "TestEnum"
+    end
+    add_message "TestMessage2" do
+      optional :foo, :int32, 1
+    end
+    add_message "Recursive1" do
+      optional :foo, :message, 1, "Recursive2"
+    end
+    add_message "Recursive2" do
+      optional :foo, :message, 1, "Recursive1"
+    end
+    add_enum "TestEnum" do
+      value :Default, 0
+      value :A, 1
+      value :B, 2
+      value :C, 3
+    end
+    add_message "BadFieldNames" do
+      optional :dup, :int32, 1
+      optional :class, :int32, 2
+      optional :"a.b", :int32, 3
+    end
+  end
+
+  TestMessage = pool.lookup("TestMessage").msgclass
+  TestMessage2 = pool.lookup("TestMessage2").msgclass
+  Recursive1 = pool.lookup("Recursive1").msgclass
+  Recursive2 = pool.lookup("Recursive2").msgclass
+  TestEnum = pool.lookup("TestEnum").enummodule
+  BadFieldNames = pool.lookup("BadFieldNames").msgclass
+
+# ------------ test cases ---------------
+
+  class MessageContainerTest < Test::Unit::TestCase
+
+    def test_defaults
+      m = TestMessage.new
+      assert m.optional_int32 == 0
+      assert m.optional_int64 == 0
+      assert m.optional_uint32 == 0
+      assert m.optional_uint64 == 0
+      assert m.optional_bool == false
+      assert m.optional_float == 0.0
+      assert m.optional_double == 0.0
+      assert m.optional_string == ""
+      assert m.optional_bytes == ""
+      assert m.optional_msg == nil
+      assert m.optional_enum == :Default
+    end
+
+    def test_setters
+      m = TestMessage.new
+      m.optional_int32 = -42
+      assert m.optional_int32 == -42
+      m.optional_int64 = -0x1_0000_0000
+      assert m.optional_int64 == -0x1_0000_0000
+      m.optional_uint32 = 0x9000_0000
+      assert m.optional_uint32 == 0x9000_0000
+      m.optional_uint64 = 0x9000_0000_0000_0000
+      assert m.optional_uint64 == 0x9000_0000_0000_0000
+      m.optional_bool = true
+      assert m.optional_bool == true
+      m.optional_float = 0.5
+      assert m.optional_float == 0.5
+      m.optional_double = 0.5
+      m.optional_string = "hello"
+      assert m.optional_string == "hello"
+      m.optional_bytes = "world".encode!('ASCII-8BIT')
+      assert m.optional_bytes == "world"
+      m.optional_msg = TestMessage2.new(:foo => 42)
+      assert m.optional_msg == TestMessage2.new(:foo => 42)
+    end
+
+    def test_ctor_args
+      m = TestMessage.new(:optional_int32 => -42,
+                          :optional_msg => TestMessage2.new,
+                          :optional_enum => :C,
+                          :repeated_string => ["hello", "there", "world"])
+      assert m.optional_int32 == -42
+      assert m.optional_msg.class == TestMessage2
+      assert m.repeated_string.length == 3
+      assert m.optional_enum == :C
+      assert m.repeated_string[0] == "hello"
+      assert m.repeated_string[1] == "there"
+      assert m.repeated_string[2] == "world"
+    end
+
+    def test_inspect
+      m = TestMessage.new(:optional_int32 => -42,
+                          :optional_enum => :A,
+                          :optional_msg => TestMessage2.new,
+                          :repeated_string => ["hello", "there", "world"])
+      expected = '<BasicTest::TestMessage: optional_int32: -42, optional_int64: 0, optional_uint32: 0, optional_uint64: 0, optional_bool: false, optional_float: 0.0, optional_double: 0.0, optional_string: "", optional_bytes: "", optional_msg: <BasicTest::TestMessage2: foo: 0>, optional_enum: :A, repeated_int32: [], repeated_int64: [], repeated_uint32: [], repeated_uint64: [], repeated_bool: [], repeated_float: [], repeated_double: [], repeated_string: ["hello", "there", "world"], repeated_bytes: [], repeated_msg: [], repeated_enum: []>'
+      assert m.inspect == expected
+    end
+
+    def test_hash
+      m1 = TestMessage.new(:optional_int32 => 42)
+      m2 = TestMessage.new(:optional_int32 => 102)
+      assert m1.hash != 0
+      assert m2.hash != 0
+      # relying on the randomness here -- if hash function changes and we are
+      # unlucky enough to get a collision, then change the values above.
+      assert m1.hash != m2.hash
+    end
+
+    def test_type_errors
+      m = TestMessage.new
+      assert_raise TypeError do
+        m.optional_int32 = "hello"
+      end
+      assert_raise TypeError do
+        m.optional_string = 42
+      end
+      assert_raise TypeError do
+        m.optional_string = nil
+      end
+      assert_raise TypeError do
+        m.optional_bool = 42
+      end
+      assert_raise TypeError do
+        m.optional_msg = TestMessage.new  # expects TestMessage2
+      end
+
+      assert_raise TypeError do
+        m.repeated_int32 = []  # needs RepeatedField
+      end
+
+      assert_raise TypeError do
+        m.repeated_int32.push "hello"
+      end
+
+      assert_raise TypeError do
+        m.repeated_msg.push TestMessage.new
+      end
+    end
+
+    def test_string_encoding
+      m = TestMessage.new
+
+      # Assigning a normal (ASCII or UTF8) string to a bytes field, or
+      # ASCII-8BIT to a string field, raises an error.
+      assert_raise TypeError do
+        m.optional_bytes = "Test string ASCII".encode!('ASCII')
+      end
+      assert_raise TypeError do
+        m.optional_bytes = "Test string UTF-8 \u0100".encode!('UTF-8')
+      end
+      assert_raise TypeError do
+        m.optional_string = ["FFFF"].pack('H*')
+      end
+
+      # "Ordinary" use case.
+      m.optional_bytes = ["FFFF"].pack('H*')
+      m.optional_string = "\u0100"
+
+      # strings are mutable so we can do this, but serialize should catch it.
+      m.optional_string = "asdf".encode!('UTF-8')
+      m.optional_string.encode!('ASCII-8BIT')
+      assert_raise TypeError do
+        data = TestMessage.encode(m)
+      end
+    end
+
+    def test_rptfield_int32
+      l = Google::Protobuf::RepeatedField.new(:int32)
+      assert l.count == 0
+      l = Google::Protobuf::RepeatedField.new(:int32, [1, 2, 3])
+      assert l.count == 3
+      assert l == [1, 2, 3]
+      l.push 4
+      assert l == [1, 2, 3, 4]
+      dst_list = []
+      l.each { |val| dst_list.push val }
+      assert dst_list == [1, 2, 3, 4]
+      assert l.to_a == [1, 2, 3, 4]
+      assert l[0] == 1
+      assert l[3] == 4
+      l[0] = 5
+      assert l == [5, 2, 3, 4]
+
+      l2 = l.dup
+      assert l == l2
+      assert l.object_id != l2.object_id
+      l2.push 6
+      assert l.count == 4
+      assert l2.count == 5
+
+      assert l.inspect == '[5, 2, 3, 4]'
+
+      l.insert(7, 8, 9)
+      assert l == [5, 2, 3, 4, 7, 8, 9]
+      assert l.pop == 9
+      assert l == [5, 2, 3, 4, 7, 8]
+
+      assert_raise TypeError do
+        m = TestMessage.new
+        l.push m
+      end
+
+      m = TestMessage.new
+      m.repeated_int32 = l
+      assert m.repeated_int32 == [5, 2, 3, 4, 7, 8]
+      assert m.repeated_int32.object_id == l.object_id
+      l.push 42
+      assert m.repeated_int32.pop == 42
+
+      l3 = l + l.dup
+      assert l3.count == l.count * 2
+      l.count.times do |i|
+        assert l3[i] == l[i]
+        assert l3[l.count + i] == l[i]
+      end
+
+      l.clear
+      assert l.count == 0
+      l += [1, 2, 3, 4]
+      l.replace([5, 6, 7, 8])
+      assert l == [5, 6, 7, 8]
+
+      l4 = Google::Protobuf::RepeatedField.new(:int32)
+      l4[5] = 42
+      assert l4 == [0, 0, 0, 0, 0, 42]
+
+      l4 << 100
+      assert l4 == [0, 0, 0, 0, 0, 42, 100]
+      l4 << 101 << 102
+      assert l4 == [0, 0, 0, 0, 0, 42, 100, 101, 102]
+    end
+
+    def test_rptfield_msg
+      l = Google::Protobuf::RepeatedField.new(:message, TestMessage)
+      l.push TestMessage.new
+      assert l.count == 1
+      assert_raise TypeError do
+        l.push TestMessage2.new
+      end
+      assert_raise TypeError do
+        l.push 42
+      end
+
+      l2 = l.dup
+      assert l2[0] == l[0]
+      assert l2[0].object_id == l[0].object_id
+
+      l2 = Google::Protobuf.deep_copy(l)
+      assert l2[0] == l[0]
+      assert l2[0].object_id != l[0].object_id
+
+      l3 = l + l2
+      assert l3.count == 2
+      assert l3[0] == l[0]
+      assert l3[1] == l2[0]
+      l3[0].optional_int32 = 1000
+      assert l[0].optional_int32 == 1000
+
+      new_msg = TestMessage.new(:optional_int32 => 200)
+      l4 = l + [new_msg]
+      assert l4.count == 2
+      new_msg.optional_int32 = 1000
+      assert l4[1].optional_int32 == 1000
+    end
+
+    def test_rptfield_enum
+      l = Google::Protobuf::RepeatedField.new(:enum, TestEnum)
+      l.push :A
+      l.push :B
+      l.push :C
+      assert l.count == 3
+      assert_raise NameError do
+        l.push :D
+      end
+      assert l[0] == :A
+
+      l.push 4
+      assert l[3] == 4
+    end
+
+    def test_rptfield_initialize
+      assert_raise ArgumentError do
+        l = Google::Protobuf::RepeatedField.new
+      end
+      assert_raise ArgumentError do
+        l = Google::Protobuf::RepeatedField.new(:message)
+      end
+      assert_raise ArgumentError do
+        l = Google::Protobuf::RepeatedField.new([1, 2, 3])
+      end
+      assert_raise ArgumentError do
+        l = Google::Protobuf::RepeatedField.new(:message, [TestMessage2.new])
+      end
+    end
+
+    def test_enum_field
+      m = TestMessage.new
+      assert m.optional_enum == :Default
+      m.optional_enum = :A
+      assert m.optional_enum == :A
+      assert_raise NameError do
+        m.optional_enum = :ASDF
+      end
+      m.optional_enum = 1
+      assert m.optional_enum == :A
+      m.optional_enum = 100
+      assert m.optional_enum == 100
+    end
+
+    def test_dup
+      m = TestMessage.new
+      m.optional_string = "hello"
+      m.optional_int32 = 42
+      m.repeated_msg.push TestMessage2.new(:foo => 100)
+      m.repeated_msg.push TestMessage2.new(:foo => 200)
+
+      m2 = m.dup
+      assert m == m2
+      m.optional_int32 += 1
+      assert m != m2
+      assert m.repeated_msg[0] == m2.repeated_msg[0]
+      assert m.repeated_msg[0].object_id == m2.repeated_msg[0].object_id
+    end
+
+    def test_deep_copy
+      m = TestMessage.new(:optional_int32 => 42,
+                          :repeated_msg => [TestMessage2.new(:foo => 100)])
+      m2 = Google::Protobuf.deep_copy(m)
+      assert m == m2
+      assert m.repeated_msg == m2.repeated_msg
+      assert m.repeated_msg.object_id != m2.repeated_msg.object_id
+      assert m.repeated_msg[0].object_id != m2.repeated_msg[0].object_id
+    end
+
+    def test_enum_lookup
+      assert TestEnum::A == 1
+      assert TestEnum::B == 2
+      assert TestEnum::C == 3
+
+      assert TestEnum::lookup(1) == :A
+      assert TestEnum::lookup(2) == :B
+      assert TestEnum::lookup(3) == :C
+
+      assert TestEnum::resolve(:A) == 1
+      assert TestEnum::resolve(:B) == 2
+      assert TestEnum::resolve(:C) == 3
+    end
+
+    def test_parse_serialize
+      m = TestMessage.new(:optional_int32 => 42,
+                          :optional_string => "hello world",
+                          :optional_enum => :B,
+                          :repeated_string => ["a", "b", "c"],
+                          :repeated_int32 => [42, 43, 44],
+                          :repeated_enum => [:A, :B, :C, 100],
+                          :repeated_msg => [TestMessage2.new(:foo => 1), TestMessage2.new(:foo => 2)])
+      data = TestMessage.encode m
+      m2 = TestMessage.decode data
+      assert m == m2
+
+      data = Google::Protobuf.encode m
+      m2 = Google::Protobuf.decode(TestMessage, data)
+      assert m == m2
+    end
+
+    def test_def_errors
+      s = Google::Protobuf::DescriptorPool.new
+      assert_raise TypeError do
+        s.build do
+          # enum with no default (integer value 0)
+          add_enum "MyEnum" do
+            value :A, 1
+          end
+        end
+      end
+      assert_raise TypeError do
+        s.build do
+          # message with required field (unsupported in proto3)
+          add_message "MyMessage" do
+            required :foo, :int32, 1
+          end
+        end
+      end
+    end
+
+    def test_corecursive
+      # just be sure that we can instantiate types with corecursive field-type
+      # references.
+      m = Recursive1.new(:foo => Recursive2.new(:foo => Recursive1.new))
+      assert Recursive1.descriptor.lookup("foo").subtype ==
+        Recursive2.descriptor
+      assert Recursive2.descriptor.lookup("foo").subtype ==
+        Recursive1.descriptor
+
+      serialized = Recursive1.encode(m)
+      m2 = Recursive1.decode(serialized)
+      assert m == m2
+    end
+
+    def test_serialize_cycle
+      m = Recursive1.new(:foo => Recursive2.new)
+      m.foo.foo = m
+      assert_raise RuntimeError do
+        serialized = Recursive1.encode(m)
+      end
+    end
+
+    def test_bad_field_names
+      m = BadFieldNames.new(:dup => 1, :class => 2)
+      m2 = m.dup
+      assert m == m2
+      assert m['dup'] == 1
+      assert m['class'] == 2
+      m['dup'] = 3
+      assert m['dup'] == 3
+      m['a.b'] = 4
+      assert m['a.b'] == 4
+    end
+
+
+    def test_int_ranges
+      m = TestMessage.new
+
+      m.optional_int32 = 0
+      m.optional_int32 = -0x8000_0000
+      m.optional_int32 = +0x7fff_ffff
+      m.optional_int32 = 1.0
+      m.optional_int32 = -1.0
+      m.optional_int32 = 2e9
+      assert_raise RangeError do
+        m.optional_int32 = -0x8000_0001
+      end
+      assert_raise RangeError do
+        m.optional_int32 = +0x8000_0000
+      end
+      assert_raise RangeError do
+        m.optional_int32 = +0x1000_0000_0000_0000_0000_0000 # force Bignum
+      end
+      assert_raise RangeError do
+        m.optional_int32 = 1e12
+      end
+      assert_raise RangeError do
+        m.optional_int32 = 1.5
+      end
+
+      m.optional_uint32 = 0
+      m.optional_uint32 = +0xffff_ffff
+      m.optional_uint32 = 1.0
+      m.optional_uint32 = 4e9
+      assert_raise RangeError do
+        m.optional_uint32 = -1
+      end
+      assert_raise RangeError do
+        m.optional_uint32 = -1.5
+      end
+      assert_raise RangeError do
+        m.optional_uint32 = -1.5e12
+      end
+      assert_raise RangeError do
+        m.optional_uint32 = -0x1000_0000_0000_0000
+      end
+      assert_raise RangeError do
+        m.optional_uint32 = +0x1_0000_0000
+      end
+      assert_raise RangeError do
+        m.optional_uint32 = +0x1000_0000_0000_0000_0000_0000 # force Bignum
+      end
+      assert_raise RangeError do
+        m.optional_uint32 = 1e12
+      end
+      assert_raise RangeError do
+        m.optional_uint32 = 1.5
+      end
+
+      m.optional_int64 = 0
+      m.optional_int64 = -0x8000_0000_0000_0000
+      m.optional_int64 = +0x7fff_ffff_ffff_ffff
+      m.optional_int64 = 1.0
+      m.optional_int64 = -1.0
+      m.optional_int64 = 8e18
+      m.optional_int64 = -8e18
+      assert_raise RangeError do
+        m.optional_int64 = -0x8000_0000_0000_0001
+      end
+      assert_raise RangeError do
+        m.optional_int64 = +0x8000_0000_0000_0000
+      end
+      assert_raise RangeError do
+        m.optional_int64 = +0x1000_0000_0000_0000_0000_0000 # force Bignum
+      end
+      assert_raise RangeError do
+        m.optional_int64 = 1e50
+      end
+      assert_raise RangeError do
+        m.optional_int64 = 1.5
+      end
+
+      m.optional_uint64 = 0
+      m.optional_uint64 = +0xffff_ffff_ffff_ffff
+      m.optional_uint64 = 1.0
+      m.optional_uint64 = 16e18
+      assert_raise RangeError do
+        m.optional_uint64 = -1
+      end
+      assert_raise RangeError do
+        m.optional_uint64 = -1.5
+      end
+      assert_raise RangeError do
+        m.optional_uint64 = -1.5e12
+      end
+      assert_raise RangeError do
+        m.optional_uint64 = -0x1_0000_0000_0000_0000
+      end
+      assert_raise RangeError do
+        m.optional_uint64 = +0x1_0000_0000_0000_0000
+      end
+      assert_raise RangeError do
+        m.optional_uint64 = +0x1000_0000_0000_0000_0000_0000 # force Bignum
+      end
+      assert_raise RangeError do
+        m.optional_uint64 = 1e50
+      end
+      assert_raise RangeError do
+        m.optional_uint64 = 1.5
+      end
+
+    end
+
+    def test_stress_test
+      m = TestMessage.new
+      m.optional_int32 = 42
+      m.optional_int64 = 0x100000000
+      m.optional_string = "hello world"
+      10.times do m.repeated_msg.push TestMessage2.new(:foo => 42) end
+      10.times do m.repeated_string.push "hello world" end
+
+      data = TestMessage.encode(m)
+
+      l = 0
+      10_000.times do
+        m = TestMessage.decode(data)
+        data_new = TestMessage.encode(m)
+        assert data_new == data
+        data = data_new
+      end
+    end
+
+    def test_reflection
+      m = TestMessage.new(:optional_int32 => 1234)
+      msgdef = m.class.descriptor
+      assert msgdef.class == Google::Protobuf::Descriptor
+      assert msgdef.any? {|field| field.name == "optional_int32"}
+      optional_int32 = msgdef.lookup "optional_int32"
+      assert optional_int32.class == Google::Protobuf::FieldDescriptor
+      assert optional_int32 != nil
+      assert optional_int32.name == "optional_int32"
+      assert optional_int32.type == :int32
+      optional_int32.set(m, 5678)
+      assert m.optional_int32 == 5678
+      m.optional_int32 = 1000
+      assert optional_int32.get(m) == 1000
+
+      optional_msg = msgdef.lookup "optional_msg"
+      assert optional_msg.subtype == TestMessage2.descriptor
+
+      optional_msg.set(m, optional_msg.subtype.msgclass.new)
+
+      assert msgdef.msgclass == TestMessage
+
+      optional_enum = msgdef.lookup "optional_enum"
+      assert optional_enum.subtype == TestEnum.descriptor
+      assert optional_enum.subtype.class == Google::Protobuf::EnumDescriptor
+      optional_enum.subtype.each do |k, v|
+        # set with integer, check resolution to symbolic name
+        optional_enum.set(m, v)
+        assert optional_enum.get(m) == k
+      end
+    end
+
+    def test_json
+      m = TestMessage.new(:optional_int32 => 1234,
+                          :optional_int64 => -0x1_0000_0000,
+                          :optional_uint32 => 0x8000_0000,
+                          :optional_uint64 => 0xffff_ffff_ffff_ffff,
+                          :optional_bool => true,
+                          :optional_float => 1.0,
+                          :optional_double => -1e100,
+                          :optional_string => "Test string",
+                          :optional_bytes => ["FFFFFFFF"].pack('H*'),
+                          :optional_msg => TestMessage2.new(:foo => 42),
+                          :repeated_int32 => [1, 2, 3, 4],
+                          :repeated_string => ["a", "b", "c"],
+                          :repeated_bool => [true, false, true, false],
+                          :repeated_msg => [TestMessage2.new(:foo => 1),
+                                            TestMessage2.new(:foo => 2)])
+
+      json_text = TestMessage.encode_json(m)
+      m2 = TestMessage.decode_json(json_text)
+      assert m == m2
+    end
+  end
+end

ruby/tests/stress.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/ruby
+
+require 'protobuf'
+require 'test/unit'
+
+module StressTest
+  pool = Google::Protobuf::DescriptorPool.new
+  pool.build do
+    add_message "TestMessage" do
+      optional :a,  :int32,        1
+      repeated :b,  :message,      2, "M"
+    end
+    add_message "M" do
+      optional :foo, :string, 1
+    end
+  end
+
+  TestMessage = pool.lookup("TestMessage").msgclass
+  M = pool.lookup("M").msgclass
+
+  class StressTest < Test::Unit::TestCase
+    def get_msg
+      TestMessage.new(:a => 1000,
+                      :b => [M.new(:foo => "hello"),
+                             M.new(:foo => "world")])
+    end
+    def test_stress
+      m = get_msg
+      data = TestMessage.encode(m)
+      100_000.times do
+        mnew = TestMessage.decode(data)
+        mnew = mnew.dup
+        assert mnew.inspect == m.inspect
+        assert TestMessage.encode(mnew) == data
+      end
+    end
+  end
+end

src/Makefile.am

@@ -260,7 +260,8 @@ libprotoc_la_SOURCES =                                         \
   google/protobuf/compiler/javanano/javanano_message_field.cc  \
   google/protobuf/compiler/javanano/javanano_message.h         \
   google/protobuf/compiler/javanano/javanano_primitive_field.cc \
-  google/protobuf/compiler/python/python_generator.cc
+  google/protobuf/compiler/python/python_generator.cc          \
+  google/protobuf/compiler/ruby/ruby_generator.cc
 
 bin_PROGRAMS = protoc
 protoc_LDADD = $(PTHREAD_LIBS) libprotobuf.la libprotoc.la

src/google/protobuf/compiler/main.cc

@@ -35,7 +35,7 @@
 #include <google/protobuf/compiler/python/python_generator.h>
 #include <google/protobuf/compiler/java/java_generator.h>
 #include <google/protobuf/compiler/javanano/javanano_generator.h>
-
+#include <google/protobuf/compiler/ruby/ruby_generator.h>
 
 int main(int argc, char* argv[]) {
 
@@ -63,5 +63,10 @@ int main(int argc, char* argv[]) {
   cli.RegisterGenerator("--javanano_out", &javanano_generator,
                         "Generate Java Nano source file.");
 
+  // Ruby
+  google::protobuf::compiler::ruby::Generator rb_generator;
+  cli.RegisterGenerator("--ruby_out", &rb_generator,
+                        "Generate Ruby source file.");
+
   return cli.Run(argc, argv);
 }

src/google/protobuf/compiler/ruby/ruby_generator.cc

@@ -0,0 +1,313 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <sstream>
+
+#include <google/protobuf/compiler/code_generator.h>
+#include <google/protobuf/compiler/plugin.h>
+#include <google/protobuf/descriptor.h>
+#include <google/protobuf/descriptor.pb.h>
+#include <google/protobuf/io/printer.h>
+#include <google/protobuf/io/zero_copy_stream.h>
+
+#include <google/protobuf/compiler/ruby/ruby_generator.h>
+
+using google::protobuf::internal::scoped_ptr;
+
+namespace google {
+namespace protobuf {
+namespace compiler {
+namespace ruby {
+
+// Forward decls.
+std::string IntToString(uint32_t value);
+std::string StripDotProto(const std::string& proto_file);
+std::string LabelForField(google::protobuf::FieldDescriptor* field);
+std::string TypeName(google::protobuf::FieldDescriptor* field);
+void GenerateMessage(const google::protobuf::Descriptor* message,
+                     google::protobuf::io::Printer* printer);
+void GenerateEnum(const google::protobuf::EnumDescriptor* en,
+                  google::protobuf::io::Printer* printer);
+void GenerateMessageAssignment(
+    const std::string& prefix,
+    const google::protobuf::Descriptor* message,
+    google::protobuf::io::Printer* printer);
+void GenerateEnumAssignment(
+    const std::string& prefix,
+    const google::protobuf::EnumDescriptor* en,
+    google::protobuf::io::Printer* printer);
+
+std::string IntToString(uint32_t value) {
+  std::ostringstream os;
+  os << value;
+  return os.str();
+}
+
+std::string StripDotProto(const std::string& proto_file) {
+  int lastindex = proto_file.find_last_of(".");
+  return proto_file.substr(0, lastindex);
+}
+
+std::string LabelForField(const google::protobuf::FieldDescriptor* field) {
+  switch (field->label()) {
+    case FieldDescriptor::LABEL_OPTIONAL: return "optional";
+    case FieldDescriptor::LABEL_REQUIRED: return "required";
+    case FieldDescriptor::LABEL_REPEATED: return "repeated";
+    default: assert(false); return "";
+  }
+}
+
+std::string TypeName(const google::protobuf::FieldDescriptor* field) {
+  switch (field->cpp_type()) {
+    case FieldDescriptor::CPPTYPE_INT32: return "int32";
+    case FieldDescriptor::CPPTYPE_INT64: return "int64";
+    case FieldDescriptor::CPPTYPE_UINT32: return "uint32";
+    case FieldDescriptor::CPPTYPE_UINT64: return "uint64";
+    case FieldDescriptor::CPPTYPE_DOUBLE: return "double";
+    case FieldDescriptor::CPPTYPE_FLOAT: return "float";
+    case FieldDescriptor::CPPTYPE_BOOL: return "bool";
+    case FieldDescriptor::CPPTYPE_ENUM: return "enum";
+    case FieldDescriptor::CPPTYPE_STRING: return "string";
+    case FieldDescriptor::CPPTYPE_MESSAGE: return "message";
+    default: assert(false); return "";
+  }
+}
+
+void GenerateMessage(const google::protobuf::Descriptor* message,
+                     google::protobuf::io::Printer* printer) {
+  printer->Print(
+    "add_message \"$name$\" do\n",
+    "name", message->full_name());
+  printer->Indent();
+
+  for (int i = 0; i < message->field_count(); i++) {
+    const FieldDescriptor* field = message->field(i);
+    printer->Print(
+      "$label$ :$name$, ",
+      "label", LabelForField(field),
+      "name", field->name());
+    printer->Print(
+      ":$type$, $number$",
+      "type", TypeName(field),
+      "number", IntToString(field->number()));
+    if (field->cpp_type() == FieldDescriptor::CPPTYPE_MESSAGE) {
+      printer->Print(
+        ", \"$subtype$\"\n",
+       "subtype", field->message_type()->full_name());
+    } else if (field->cpp_type() == FieldDescriptor::CPPTYPE_ENUM) {
+      printer->Print(
+        ", \"$subtype$\"\n",
+        "subtype", field->enum_type()->full_name());
+    } else {
+      printer->Print("\n");
+    }
+  }
+
+  printer->Outdent();
+  printer->Print("end\n");
+
+  for (int i = 0; i < message->nested_type_count(); i++) {
+    GenerateMessage(message->nested_type(i), printer);
+  }
+  for (int i = 0; i < message->enum_type_count(); i++) {
+    GenerateEnum(message->enum_type(i), printer);
+  }
+}
+
+void GenerateEnum(const google::protobuf::EnumDescriptor* en,
+                  google::protobuf::io::Printer* printer) {
+  printer->Print(
+    "add_enum \"$name$\" do\n",
+    "name", en->full_name());
+  printer->Indent();
+
+  for (int i = 0; i < en->value_count(); i++) {
+    const EnumValueDescriptor* value = en->value(i);
+    printer->Print(
+      "value :$name$, $number$\n",
+      "name", value->name(),
+      "number", IntToString(value->number()));
+  }
+
+  printer->Outdent();
+  printer->Print(
+    "end\n");
+}
+
+// Module names, class names, and enum value names need to be Ruby constants,
+// which must start with a capital letter.
+std::string RubifyConstant(const std::string& name) {
+  std::string ret = name;
+  if (!ret.empty()) {
+    if (ret[0] >= 'a' && ret[0] <= 'z') {
+      // If it starts with a lowercase letter, capitalize it.
+      ret[0] = ret[0] - 'a' + 'A';
+    } else if (ret[0] < 'A' || ret[0] > 'Z') {
+      // Otherwise (e.g. if it begins with an underscore), we need to come up
+      // with some prefix that starts with a capital letter. We could be smarter
+      // here, e.g. try to strip leading underscores, but this may cause other
+      // problems if the user really intended the name. So let's just prepend a
+      // well-known suffix.
+      ret = "PB_" + ret;
+    }
+  }
+  return ret;
+}
+
+void GenerateMessageAssignment(
+    const std::string& prefix,
+    const google::protobuf::Descriptor* message,
+    google::protobuf::io::Printer* printer) {
+  printer->Print(
+    "$prefix$$name$ = ",
+    "prefix", prefix,
+    "name", RubifyConstant(message->name()));
+  printer->Print(
+    "Google::Protobuf::DescriptorPool.generated_pool."
+    "lookup(\"$full_name$\").msgclass\n",
+    "full_name", message->full_name());
+
+  std::string nested_prefix = prefix + message->name() + "::";
+  for (int i = 0; i < message->nested_type_count(); i++) {
+    GenerateMessageAssignment(nested_prefix, message->nested_type(i), printer);
+  }
+  for (int i = 0; i < message->enum_type_count(); i++) {
+    GenerateEnumAssignment(nested_prefix, message->enum_type(i), printer);
+  }
+}
+
+void GenerateEnumAssignment(
+    const std::string& prefix,
+    const google::protobuf::EnumDescriptor* en,
+    google::protobuf::io::Printer* printer) {
+  printer->Print(
+    "$prefix$$name$ = ",
+    "prefix", prefix,
+    "name", RubifyConstant(en->name()));
+  printer->Print(
+    "Google::Protobuf::DescriptorPool.generated_pool."
+    "lookup(\"$full_name$\").enummodule\n",
+    "full_name", en->full_name());
+}
+
+int GeneratePackageModules(
+    std::string package_name,
+    google::protobuf::io::Printer* printer) {
+  int levels = 0;
+  while (!package_name.empty()) {
+    size_t dot_index = package_name.find(".");
+    string component;
+    if (dot_index == string::npos) {
+      component = package_name;
+      package_name = "";
+    } else {
+      component = package_name.substr(0, dot_index);
+      package_name = package_name.substr(dot_index + 1);
+    }
+    component = RubifyConstant(component);
+    printer->Print(
+      "module $name$\n",
+      "name", component);
+    printer->Indent();
+    levels++;
+  }
+  return levels;
+}
+
+void EndPackageModules(
+    int levels,
+    google::protobuf::io::Printer* printer) {
+  while (levels > 0) {
+    levels--;
+    printer->Outdent();
+    printer->Print(
+      "end\n");
+  }
+}
+
+void GenerateFile(const google::protobuf::FileDescriptor* file,
+                  google::protobuf::io::Printer* printer) {
+  printer->Print(
+    "# Generated by the protocol buffer compiler.  DO NOT EDIT!\n"
+    "# source: $filename$\n"
+    "\n",
+    "filename", file->name());
+
+  printer->Print(
+    "require 'protobuf'\n\n");
+
+  for (int i = 0; i < file->dependency_count(); i++) {
+    const std::string& name = file->dependency(i)->name();
+    printer->Print(
+      "require '$name$'\n", "name", StripDotProto(name));
+  }
+
+  printer->Print(
+    "Google::Protobuf::DescriptorPool.generated_pool.build do\n");
+  printer->Indent();
+  for (int i = 0; i < file->message_type_count(); i++) {
+    GenerateMessage(file->message_type(i), printer);
+  }
+  for (int i = 0; i < file->enum_type_count(); i++) {
+    GenerateEnum(file->enum_type(i), printer);
+  }
+  printer->Outdent();
+  printer->Print(
+    "end\n\n");
+
+  int levels = GeneratePackageModules(file->package(), printer);
+  for (int i = 0; i < file->message_type_count(); i++) {
+    GenerateMessageAssignment("", file->message_type(i), printer);
+  }
+  for (int i = 0; i < file->enum_type_count(); i++) {
+    GenerateEnumAssignment("", file->enum_type(i), printer);
+  }
+  EndPackageModules(levels, printer);
+}
+
+bool Generator::Generate(
+    const FileDescriptor* file,
+    const string& parameter,
+    GeneratorContext* generator_context,
+    string* error) const {
+  std::string filename =
+      StripDotProto(file->name()) + ".rb";
+  scoped_ptr<io::ZeroCopyOutputStream> output(generator_context->Open(filename));
+  io::Printer printer(output.get(), '$');
+
+  GenerateFile(file, &printer);
+
+  return true;
+}
+
+}  // namespace ruby
+}  // namespace compiler
+}  // namespace protobuf
+}  // namespace google

src/google/protobuf/compiler/ruby/ruby_generator.h

@@ -0,0 +1,57 @@
+// Protocol Buffers - Google's data interchange format
+// Copyright 2008 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef GOOGLE_PROTOBUF_COMPILER_RUBY_GENERATOR_H__
+#define GOOGLE_PROTOBUF_COMPILER_RUBY_GENERATOR_H__
+
+#include <string>
+
+#include <google/protobuf/compiler/code_generator.h>
+
+namespace google {
+namespace protobuf {
+namespace compiler {
+namespace ruby {
+
+class Generator : public google::protobuf::compiler::CodeGenerator {
+  virtual bool Generate(
+      const FileDescriptor* file,
+      const string& parameter,
+      GeneratorContext* generator_context,
+      string* error) const;
+};
+
+}  // namespace ruby
+}  // namespace compiler
+}  // namespace protobuf
+}  // namespace google
+
+#endif  // GOOGLE_PROTOBUF_COMPILER_RUBY_GENERATOR_H__
+

upb

@@ -0,0 +1 @@
+Subproject commit 56913be6bb57f81dbbf7baf9cc9a0a2cd1a36493