serialize.lisp

   1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
   2 ;;;                                                                  ;;;
   3 ;;; Confidential and proprietary information of ITA Software, Inc.   ;;;
   4 ;;;                                                                  ;;;
   5 ;;; Copyright (c) 2012 ITA Software, Inc.  All rights reserved.      ;;;
   6 ;;;                                                                  ;;;
   7 ;;; Original author: Scott McKay                                     ;;;
   8 ;;;                                                                  ;;;
   9 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  10
  11 (in-package "PROTO-IMPL")
  12
  13
  14 ;;; Protobuf serialization from Lisp objects
  15
  16 ;;; Serialization
  17
  18 ;; Serialize the object using the given protobuf type
  19 (defun serialize-object-to-stream (object type &key (stream *standard-output*) visited)
  20   "Serializes the object 'object' of type 'type' onto the stream 'stream'
  21    using the wire format.
  22    'type' is the Lisp name of a Protobufs message (usually the name of a
  23    Lisp class) or a 'protobuf-message'.
  24    'visited' is a hash table used to cache object sizes. If it is supplied, it will be
  25    cleared before it is used; otherwise, a fresh table will be created.
  26    The return value is the buffer containing the serialized object. If the stream is
  27    nil, the buffer is not actually written to anywhere."
  28   (let* ((visited (let ((v (or visited (make-hash-table))))
  29                     (clrhash v)
  30                     v))
  31          (size    (object-size object type visited))
  32          (buffer  (make-array size :element-type '(unsigned-byte 8))))
  33     (serialize-object object type buffer 0 visited)
  34     (when stream
  35       (write-sequence buffer stream))
  36     buffer))
  37
  38 ;; Allow clients to add their own methods
  39 ;; This is how we address the problem of cycles, e.g. -- if you have an object
  40 ;; that may contain cycles, serialize the cyclic object using a "handle"
  41 (defgeneric serialize-object (object type buffer &optional start visited)
  42   (:documentation
  43    "Serializes the object 'object' of type 'type' into the byte array 'buffer'
  44     using the wire format.
  45     'type' is the Lisp name of a Protobufs message (usually the name of a
  46     Lisp class) or a 'protobuf-message'.
  47     The object is serialized into the byte array given by 'buffer' starting
  48     at the fixnum index 'index' using the wire format.
  49     'visited' is a hash table used to cache object sizes.
  50     The return value is the buffer containing the serialized object."))
  51
  52 (defmethod serialize-object (object (type symbol) buffer &optional start visited)
  53   (let ((message (find-message-for-class type)))
  54     (assert message ()
  55             "There is no Protobuf message having the type ~S" type)
  56     (serialize-object object message buffer start visited)))
  57
  58 ;; 'visited' is used to cache object sizes
  59 ;; If it's passed in explicitly, it is assumed to already have the sizes within it
  60 ;; The default method uses metadata from the protobuf "schema" for the message
  61 (defmethod serialize-object (object (message protobuf-message) buffer &optional start visited)
  62   (declare (type (simple-array (unsigned-byte 8)) buffer))
  63   (let ((visited (or visited (make-hash-table)))
  64         (index   (or start 0)))
  65     (declare (type fixnum index))
  66     (macrolet ((read-slot (object slot reader)
  67                  ;; Don't do a boundp check, we assume the object is fully populated
  68                  ;; Unpopulated slots should be "nullable" and should contain nil
  69                  `(if ,reader
  70                     (funcall ,reader ,object)
  71                     (slot-value ,object ,slot))))
  72       (labels ((do-field (object trace field)
  73                  ;; We don't do cycle detection here
  74                  ;; If the client needs it, he can define his own 'serialize-object'
  75                  ;; method to clean things up first
  76                  (let* ((type   (if (eq (proto-class field) 'boolean) :bool (proto-class field)))
  77                         (slot   (proto-value field))
  78                         (reader (proto-reader field))
  79                         msg)
  80                    (when (or slot reader)
  81                      (cond ((eq (proto-required field) :repeated)
  82                             (cond ((and (proto-packed field) (packed-type-p type))
  83                                    (let ((tag (make-tag type (proto-index field))))
  84                                      (setq index (serialize-packed (read-slot object slot reader)
  85                                                                    type tag buffer index))))
  86                                   ((keywordp type)
  87                                    (let ((tag (make-tag type (proto-index field))))
  88                                      (map () #'(lambda (v)
  89                                                  (setq index (serialize-prim v type tag buffer index)))
  90                                              (read-slot object slot reader))))
  91                                   ((typep (setq msg (and type (or (find-message trace type)
  92                                                                   (find-enum trace type))))
  93                                           'protobuf-message)
  94                                    (dolist (v (if slot (read-slot object slot reader) (list object)))
  95                                      ;; To serialize an embedded message, first say that it's
  96                                      ;; a string, then encode its size, then serialize its fields
  97                                      (let ((tag (make-tag $wire-type-string (proto-index field)))
  98                                            (len (object-size v msg visited)))
  99                                        (setq index (encode-uint32 tag buffer index))
 100                                        (setq index (encode-uint32 len buffer index)))
 101                                      (map () (curry #'do-field v msg)
 102                                              (proto-fields msg))))
 103                                   ((typep msg 'protobuf-enum)
 104                                    (let ((tag (make-tag $wire-type-varint (proto-index field))))
 105                                      (map () #'(lambda (v)
 106                                                  (setq index (serialize-enum v (proto-values msg) tag buffer index)))
 107                                              (read-slot object slot reader))))))
 108                            (t
 109                             (cond ((keywordp type)
 110                                    (let ((v (read-slot object slot reader)))
 111                                      (when (or v (eq type :bool))
 112                                        (let ((tag (make-tag type (proto-index field))))
 113                                          (setq index (serialize-prim v type tag buffer index))))))
 114                                   ((typep (setq msg (and type (or (find-message trace type)
 115                                                                   (find-enum trace type))))
 116                                           'protobuf-message)
 117                                    (let ((v (if slot (read-slot object slot reader) object)))
 118                                      (when v
 119                                        (let ((tag (make-tag $wire-type-string (proto-index field)))
 120                                              (len (object-size v msg visited)))
 121                                          (setq index (encode-uint32 tag buffer index))
 122                                          (setq index (encode-uint32 len buffer index))
 123                                          (map () (curry #'do-field v msg)
 124                                                  (proto-fields msg))))))
 125                                   ((typep msg 'protobuf-enum)
 126                                    (let ((v (read-slot object slot reader)))
 127                                      (when v
 128                                        (let ((tag (make-tag $wire-type-varint (proto-index field))))
 129                                          (setq index (serialize-enum v (proto-values msg) tag buffer index)))))))))))))
 130         (declare (dynamic-extent #'do-field))
 131         (map () (curry #'do-field object message) (proto-fields message))))
 132     (values buffer index)))
 133
 134
 135 ;;; Deserialization
 136
 137 (defun deserialize-object-from-stream (type &key (stream *standard-input*))
 138   "Deserializes an object of the given type 'type' as a Protobuf object.
 139    'type' is the Lisp name of a Protobufs message (usually the name of a
 140    Lisp class) or a 'protobuf-message'.
 141    The return value is the object."
 142   (let* ((size    (file-length stream))
 143          (buffer  (make-array size :element-type '(unsigned-byte 8))))
 144     (read-sequence buffer stream)
 145     (deserialize-object type buffer 0 size)))
 146
 147 ;; Allow clients to add their own methods
 148 ;; This is you might preserve object identity, e.g.
 149 (defgeneric deserialize-object (type buffer &optional start end)
 150   (:documentation
 151    "Deserializes an object of the given type 'type' as a Protobufs object.
 152     'type' is the Lisp name of a Protobufs message (usually the name of a
 153     Lisp class) or a 'protobuf-message'.
 154     The encoded bytes are in the byte array given by 'buffer' starting at
 155     the fixnum index 'index' up to the length of the buffer, given by 'length'.
 156     The return values are the object and the index at which deserialization stopped.."))
 157
 158 (defmethod deserialize-object ((type symbol) buffer &optional start end)
 159   (let ((message (find-message-for-class type)))
 160     (assert message ()
 161             "There is no Protobuf message having the type ~S" type)
 162     (deserialize-object message buffer start end)))
 163
 164 ;; The default method uses metadata from the protobuf "schema" for the message
 165 (defmethod deserialize-object ((message protobuf-message) buffer &optional start end)
 166   (declare (type (simple-array (unsigned-byte 8)) buffer))
 167   (let ((index   (or start 0))
 168         (length  (or end (length buffer))))
 169     (declare (type fixnum index length))
 170     (macrolet ((read-slot (object slot reader)
 171                  `(if ,reader
 172                     (funcall ,reader ,object)
 173                     (slot-value ,object ,slot)))
 174                (write-slot (object slot writer value)
 175                  `(if ,writer
 176                     (funcall ,writer ,object ,value)
 177                     (setf (slot-value ,object ,slot) ,value))))
 178       (labels ((deserialize (type trace end)
 179                  (declare (type fixnum end))
 180                  (let* ((message (find-message trace type))
 181                         (object  (and message
 182                                       (make-instance (or (proto-alias-for message) (proto-class message)))))
 183                         ;; All the slots into which we store a repeated element
 184                         (rslots ()))
 185                    (loop
 186                      (multiple-value-bind (tag idx)
 187                          (if (i< index end) (decode-uint32 buffer index) (values 0 index))
 188                        ;; We're done if we've gotten to the end index or
 189                        ;; we see a null byte (there can never be null tags
 190                        ;; because field indices start at 1)
 191                        (when (i= tag 0)
 192                          ;; Now set the repeated slots
 193                          ;; If we do this element by element, we get killed by type checking
 194                          ;; in the slot setters
 195                          (dolist (field rslots)
 196                            (let ((slot   (proto-value field))
 197                                  (reader (proto-reader field))
 198                                  (writer (proto-writer field)))
 199                              (write-slot object slot writer
 200                                          (nreverse (read-slot object slot reader)))))
 201                          (return-from deserialize
 202                            (values object index)))
 203                        (setq index idx)
 204                        (let* ((wtype (ilogand tag #x7))
 205                               (fidx  (ilogand (iash tag -3) #x1FFFFFFF))
 206                               (field (find fidx (proto-fields message) :key #'proto-index))
 207                               (type  (and field (if (eq (proto-class field) 'boolean) :bool (proto-class field))))
 208                               ;; It's OK for this to be null
 209                               ;; That means we're parsing some version of a message
 210                               ;; that has the field, but our current message does not
 211                               ;; We still have to deserialize everything, though
 212                               (slot   (and field (proto-value field)))
 213                               (reader (and field (proto-reader field)))
 214                               (writer (and field (proto-writer field)))
 215                               msg)
 216                          (if (null field)
 217                            ;; If there's no field descriptor for this index, just skip
 218                            ;; the next element in the buffer having the given wire type
 219                            (setq index (skip-element buffer index wtype))
 220                            ;;--- Check for mismatched wire type, running past end of buffer, etc
 221                            (cond ((and field (eq (proto-required field) :repeated))
 222                                   (cond ((and (proto-packed field) (packed-type-p type))
 223                                          (multiple-value-bind (values idx)
 224                                              (deserialize-packed type buffer index)
 225                                            (setq index idx)
 226                                            (when slot
 227                                              (write-slot object slot writer values))))
 228                                         ((keywordp type)
 229                                          (multiple-value-bind (val idx)
 230                                              (deserialize-prim type buffer index)
 231                                            (setq index idx)
 232                                            (when slot
 233                                              (pushnew field rslots)
 234                                              ;; This "push" will type-check the entire list for
 235                                              ;; 'quux:list-of', so avoid using that type in classes
 236                                              ;; in Protobufs if performance is an issue
 237                                              (write-slot object slot writer
 238                                                          (cons val (read-slot object slot reader))))))
 239                                         ((typep (setq msg (and type (or (find-message trace type)
 240                                                                         (find-enum trace type))))
 241                                                 'protobuf-message)
 242                                          (multiple-value-bind (len idx)
 243                                              (decode-uint32 buffer index)
 244                                            (setq index idx)
 245                                            (let ((obj (deserialize type msg (+ index len))))
 246                                              (when slot
 247                                                (pushnew field rslots)
 248                                                (write-slot object slot writer
 249                                                            (cons obj (read-slot object slot reader)))))))
 250                                         ((typep msg 'protobuf-enum)
 251                                          (multiple-value-bind (val idx)
 252                                              (deserialize-enum (proto-values msg) buffer index)
 253                                            (setq index idx)
 254                                            (when slot
 255                                              (pushnew field rslots)
 256                                              (write-slot object slot writer
 257                                                          (cons val (read-slot object slot reader))))))))
 258                                  (t
 259                                   (cond ((keywordp type)
 260                                          (multiple-value-bind (val idx)
 261                                              (deserialize-prim type buffer index)
 262                                            (setq index idx)
 263                                            (when slot
 264                                              (write-slot object slot writer val))))
 265                                         ((typep (setq msg (and type (or (find-message trace type)
 266                                                                         (find-enum trace type))))
 267                                                 'protobuf-message)
 268                                          (multiple-value-bind (len idx)
 269                                              (decode-uint32 buffer index)
 270                                            (setq index idx)
 271                                            (let ((obj (deserialize type msg (+ index len))))
 272                                              (when slot
 273                                                (write-slot object slot writer obj)))))
 274                                         ((typep msg 'protobuf-enum)
 275                                          (multiple-value-bind (val idx)
 276                                              (deserialize-enum (proto-values msg) buffer index)
 277                                            (setq index idx)
 278                                            (when slot
 279                                              (write-slot object slot writer val))))))))))))))
 280         (declare (dynamic-extent #'deserialize))
 281         (deserialize (proto-class message) message length)))))
 282
 283 ;;; Object sizes
 284
 285 ;; Allow clients to add their own methods
 286 ;; This is how we address the problem of cycles, e.g. -- if you have an object
 287 ;; that may contain cycles, return the size of the "handle" to the object
 288 (defgeneric object-size (object type &optional visited)
 289   (:documentation
 290    "Computes the size in bytes of the object 'object' of type 'type'.
 291     'type' is the Lisp name of a Protobufs message (usually the name of a
 292     Lisp class) or a 'protobuf-message'.
 293     'visited' is a hash table used to cache object sizes.
 294     The return value is the size of the object in bytes."))
 295
 296 (defmethod object-size (object (type symbol) &optional visited)
 297   (let ((message (find-message-for-class type)))
 298     (assert message ()
 299             "There is no Protobuf message having the type ~S" type)
 300     (object-size object message visited)))
 301
 302 ;; 'visited' is used to cache object sizes
 303 ;; The default method uses metadata from the protobuf "schema" for the message
 304 (defmethod object-size (object (message protobuf-message) &optional visited)
 305   (let ((size (and visited (gethash object visited))))
 306     (when size
 307       (return-from object-size size)))
 308   (let ((size 0))
 309     (declare (type fixnum size))
 310     (macrolet ((read-slot (object slot reader)
 311                  ;; Don't do a boundp check, we assume the object is fully populated
 312                  ;; Unpopulated slots should be "nullable" and should contain nil
 313                  `(if ,reader
 314                     (funcall ,reader ,object)
 315                     (slot-value ,object ,slot))))
 316       (labels ((do-field (object trace field)
 317                  ;; We don't do cycle detection here
 318                  ;; If the client needs it, he can define his own 'object-size'
 319                  ;; method to clean things up first
 320                  (let* ((type   (if (eq (proto-class field) 'boolean) :bool (proto-class field)))
 321                         (slot   (proto-value field))
 322                         (reader (proto-reader field))
 323                         msg)
 324                    (when (or slot reader)
 325                      (cond ((eq (proto-required field) :repeated)
 326                             (cond ((and (proto-packed field) (packed-type-p type))
 327                                    (let ((tag (make-tag type (proto-index field))))
 328                                      (iincf size (packed-size (read-slot object slot reader) type tag))))
 329                                   ((keywordp type)
 330                                    (let ((tag (make-tag type (proto-index field))))
 331                                      (map () #'(lambda (v)
 332                                                  (iincf size (prim-size v type tag)))
 333                                              (read-slot object slot reader))))
 334                                   ((typep (setq msg (and type (or (find-message trace type)
 335                                                                   (find-enum trace type))))
 336                                           'protobuf-message)
 337                                    (dolist (v (if slot (read-slot object slot reader) (list object)))
 338                                      (let ((tag (make-tag $wire-type-string (proto-index field)))
 339                                            (len (object-size v msg visited)))
 340                                        (iincf size (length32 tag))
 341                                        (iincf size (length32 len)))
 342                                      (map () (curry #'do-field v msg)
 343                                              (proto-fields msg))))
 344                                   ((typep msg 'protobuf-enum)
 345                                    (let ((tag (make-tag $wire-type-varint (proto-index field))))
 346                                      (map () #'(lambda (v)
 347                                                  (iincf size (enum-size v (proto-values msg) tag)))
 348                                              (read-slot object slot reader))))))
 349                            (t
 350                             (cond ((keywordp type)
 351                                    (let ((v (read-slot object slot reader)))
 352                                      (when (or v (eq type :bool))
 353                                        (let ((tag (make-tag type (proto-index field))))
 354                                          (iincf size (prim-size v type tag))))))
 355                                   ((typep (setq msg (and type (or (find-message trace type)
 356                                                                   (find-enum trace type))))
 357                                           'protobuf-message)
 358                                    (let ((v (if slot (read-slot object slot reader) object)))
 359                                      (when v
 360                                        (let ((tag (make-tag $wire-type-string (proto-index field)))
 361                                              (len (object-size v msg visited)))
 362                                          (iincf size (length32 tag))
 363                                          (iincf size (length32 len)))
 364                                        (map () (curry #'do-field v msg)
 365                                                (proto-fields msg)))))
 366                                   ((typep msg 'protobuf-enum)
 367                                    (let ((v (read-slot object slot reader)))
 368                                      (when v
 369                                        (let ((tag (make-tag $wire-type-varint (proto-index field))))
 370                                          (iincf size (enum-size (read-slot object slot reader) (proto-values msg) tag)))))))))))))
 371         (declare (dynamic-extent #'do-field))
 372         (map () (curry #'do-field object message) (proto-fields message))
 373         (when visited
 374           (setf (gethash object visited) size))   ;cache the size
 375         size))))
 376
 377 \f
 378 ;;; Compile-time generation of serializers
 379
 380 ;; Note well: keep this in sync with the main 'serialize-object' method above
 381 (defun generate-serializer (message)
 382   "Generate a 'serialize-object' method for the given message."
 383   (with-gensyms (vobj vbuf vidx vval vclass)
 384     (with-collectors ((serializers collect-serializer))
 385       (dolist (field (proto-fields message))
 386         (let* ((class  (if (eq (proto-class field) 'boolean) :bool (proto-class field)))
 387                (msg    (and class (not (keywordp class))
 388                             (or (find-message message class)
 389                                 (find-enum message class))))
 390                (reader (cond ((proto-reader field)
 391                               `(,(proto-reader field) ,vobj))
 392                              ((proto-value field)
 393                               `(slot-value ,vobj ',(proto-value field)))))
 394                (index  (proto-index field)))
 395           (when reader
 396             (cond ((eq (proto-required field) :repeated)
 397                    (cond ((and (proto-packed field) (packed-type-p class))
 398                           (collect-serializer
 399                            (let ((tag (make-tag class index)))
 400                              `(setq ,vidx (serialize-packed ,reader ,class ,tag ,vbuf ,vidx)))))
 401                          ((keywordp class)
 402                           (collect-serializer
 403                            (let ((tag (make-tag class index)))
 404                              `(dolist (,vval ,reader)
 405                                 (setq ,vidx (serialize-prim ,vval ,class ,tag ,vbuf ,vidx))))))
 406                          ((typep msg 'protobuf-message)
 407                           (collect-serializer
 408                            (let ((tag (make-tag $wire-type-string index)))
 409                              `(dolist (,vval ,reader)
 410                                 (let ((len (or (and visited (gethash ,vval visited))
 411                                                (object-size ,vval ,msg visited))))
 412                                   (setq ,vidx (encode-uint32 ,tag ,vbuf ,vidx))
 413                                   (setq ,vidx (encode-uint32 len ,vbuf ,vidx))
 414                                   (serialize-object ,vval ,msg ,vbuf ,vidx visited)
 415                                   (iincf ,vidx len))))))
 416                          ((typep msg 'protobuf-enum)
 417                           (collect-serializer
 418                            (let ((tag (make-tag $wire-type-varint index)))
 419                              `(dolist (,vval ,reader)
 420                                 (setq ,vidx (serialize-enum ,vval '(,@(proto-values msg)) ,tag ,vbuf ,vidx))))))))
 421                   (t
 422                    (cond ((keywordp class)
 423                           (collect-serializer
 424                            (let ((tag (make-tag class index)))
 425                              (if (eq class :bool)
 426                                `(let ((,vval ,reader))
 427                                   (setq ,vidx (serialize-prim ,vval ,class ,tag ,vbuf ,vidx)))
 428                                `(let ((,vval ,reader))
 429                                   (when ,vval
 430                                     (setq ,vidx (serialize-prim ,vval ,class ,tag ,vbuf ,vidx))))))))
 431                          ((typep msg 'protobuf-message)
 432                           (collect-serializer
 433                            (let ((tag (make-tag $wire-type-string index)))
 434                              `(let ((,vval ,reader))
 435                                 (when ,vval
 436                                   (let ((len (or (and visited (gethash ,vval visited))
 437                                                  (object-size ,vval ,msg visited))))
 438                                     (setq ,vidx (encode-uint32 ,tag ,vbuf ,vidx))
 439                                     (setq ,vidx (encode-uint32 len ,vbuf ,vidx))
 440                                     (serialize-object ,vval ,msg ,vbuf ,vidx visited)
 441                                     (iincf ,vidx len)))))))
 442                          ((typep msg 'protobuf-enum)
 443                           (collect-serializer
 444                            (let ((tag (make-tag $wire-type-varint index)))
 445                              `(let ((,vval ,reader))
 446                                 (when ,vval
 447                                   (setq ,vidx (serialize-enum ,vval '(,@(proto-values msg)) ,tag ,vbuf ,vidx)))))))))))))
 448       `(defmethod serialize-object
 449            (,vobj (,vclass (eql ,message)) ,vbuf &optional (,vidx 0) visited)
 450          (declare (ignorable visited)
 451                   (type (simple-array (unsigned-byte 8)) ,vbuf)
 452                   (type fixnum ,vidx))
 453          (locally (declare (optimize (speed 3) (safety 0) (debug 0)))
 454           ,@serializers
 455            (values ,vbuf ,vidx))))))
 456
 457 ;; Note well: keep this in sync with the main 'deserialize-object' method above
 458 (defun generate-deserializer (message)
 459   "Generate a 'deserialize-object' method for the given message."
 460   (with-gensyms (vclass vbuf vidx vlen vobj vval)
 461     (with-collectors ((deserializers collect-deserializer)
 462                       (rslots collect-rslot))
 463       (flet ((read-slot (object field)
 464                (cond ((proto-reader field)
 465                       `(,(proto-reader field) ,object))
 466                      ((proto-value field)
 467                       `(slot-value ,object ',(proto-value field)))))
 468              (write-slot (object field value)
 469                (cond ((proto-writer field)
 470                       `(,(proto-writer field) ,object ,value))
 471                      ((proto-value field)
 472                       `(setf (slot-value ,object ',(proto-value field)) ,value)))))
 473         (dolist (field (proto-fields message))
 474           (let* ((class  (if (eq (proto-class field) 'boolean) :bool (proto-class field)))
 475                  (msg    (and class (not (keywordp class))
 476                               (or (find-message message class)
 477                                   (find-enum message class))))
 478                  (index  (proto-index field)))
 479             (cond ((eq (proto-required field) :repeated)
 480                    (cond ((and (proto-packed field) (packed-type-p class))
 481                           (collect-deserializer
 482                            `((,(make-tag class index))
 483                              (multiple-value-bind (,vval idx)
 484                                  (deserialize-packed ,class ,vbuf ,vidx)
 485                                (setq ,vidx idx)
 486                                ,(write-slot vobj field vval)))))
 487                          ((keywordp class)
 488                           (collect-rslot field)
 489                           (collect-deserializer
 490                            `((,(make-tag class index))
 491                              (multiple-value-bind (,vval idx)
 492                                  (deserialize-prim ,class ,vbuf ,vidx)
 493                                (setq ,vidx idx)
 494                                (let ((val ,(read-slot vobj field)))
 495                                  ,(write-slot vobj field `(cons ,vval val)))))))
 496                          ((typep msg 'protobuf-message)
 497                           (collect-rslot field)
 498                           (collect-deserializer
 499                            `((,(make-tag $wire-type-string index))
 500                              ;; Call 'deserialize-object' with the name of the message
 501                              ;; class so that we preferentially get any optimized version
 502                              ;; of the method
 503                              (multiple-value-bind (len idx)
 504                                  (decode-uint32 ,vbuf ,vidx)
 505                                (setq ,vidx idx)
 506                                (multiple-value-bind (,vval idx)
 507                                    (deserialize-object ',class ,vbuf ,vidx (i+ ,vidx len))
 508                                  (setq ,vidx idx)
 509                                  (let ((val ,(read-slot vobj field)))
 510                                    ,(write-slot vobj field `(cons ,vval val))))))))
 511                          ((typep msg 'protobuf-enum)
 512                           (collect-rslot field)
 513                           (collect-deserializer
 514                            `((,(make-tag $wire-type-varint index))
 515                              (multiple-value-bind (,vval idx)
 516                                  (deserialize-enum '(,@(proto-values msg)) ,vbuf ,vidx)
 517                                (setq ,vidx idx)
 518                                (let ((val ,(read-slot vobj field)))
 519                                  ,(write-slot vobj field `(cons ,vval val)))))))))
 520                   (t
 521                    (cond ((keywordp class)
 522                           (collect-deserializer
 523                            `((,(make-tag class index))
 524                              (multiple-value-bind (,vval idx)
 525                                  (deserialize-prim ,class ,vbuf ,vidx)
 526                                (setq ,vidx idx)
 527                                ,(write-slot vobj field vval)))))
 528                          ((typep msg 'protobuf-message)
 529                           (collect-deserializer
 530                            `((,(make-tag $wire-type-string index))
 531                              (multiple-value-bind (len idx)
 532                                  (decode-uint32 ,vbuf ,vidx)
 533                                (setq ,vidx idx)
 534                                (multiple-value-bind (,vval idx)
 535                                    (deserialize-object ',class ,vbuf ,vidx (i+ ,vidx len))
 536                                  (setq ,vidx idx)
 537                                  ,(write-slot vobj field vval))))))
 538                          ((typep msg 'protobuf-enum)
 539                           (collect-deserializer
 540                            `((,(make-tag $wire-type-varint index))
 541                              (multiple-value-bind (,vval idx)
 542                                  (deserialize-enum '(,@(proto-values msg)) ,vbuf ,vidx)
 543                                (setq ,vidx idx)
 544                                ,(write-slot vobj field vval)))))))))))
 545     `(defmethod deserialize-object
 546          ((,vclass (eql ,message)) ,vbuf &optional ,vidx ,vlen)
 547        (declare (type (simple-array (unsigned-byte 8)) ,vbuf))
 548        (locally (declare (optimize (speed 3) (safety 0) (debug 0)))
 549          (let ((,vidx (or ,vidx 0))
 550                (,vlen (or ,vlen (length ,vbuf))))
 551            (declare (type fixnum ,vidx ,vlen))
 552            (let ((,vobj (make-instance ',(or (proto-alias-for message) (proto-class message)))))
 553              (loop
 554                (multiple-value-bind (tag idx)
 555                    (if (i< ,vidx ,vlen) (decode-uint32 ,vbuf ,vidx) (values 0 ,vidx))
 556                  (when (i= tag 0)
 557                    (dolist (field ',(delete-duplicates rslots))
 558                      (let* ((slot   (proto-value field))
 559                             (reader (proto-reader field))
 560                             (writer (proto-writer field))
 561                             (value  (nreverse (if reader
 562                                                 (funcall reader ,vobj)
 563                                                 (slot-value ,vobj slot)))))
 564                        (if writer
 565                          (funcall writer ,vobj value)
 566                          (setf (slot-value ,vobj slot) value))))
 567                    (return-from deserialize-object
 568                                 (values ,vobj ,vidx)))
 569                  (setq ,vidx idx)
 570                  (case tag
 571                    ,@deserializers
 572                    (otherwise
 573                     (setq ,vidx (skip-element ,vbuf ,vidx (ilogand tag #x7))))))))))))))
 574
 575 ;; Note well: keep this in sync with the main 'object-size' method above
 576 (defun generate-object-size (message)
 577   "Generate an 'object-size' method for the given message."
 578   (with-gensyms (vobj vsize vval vclass)
 579     (with-collectors ((sizers collect-sizer))
 580       (dolist (field (proto-fields message))
 581         (let* ((class  (if (eq (proto-class field) 'boolean) :bool (proto-class field)))
 582                (msg    (and class (not (keywordp class))
 583                             (or (find-message message class)
 584                                 (find-enum message class))))
 585                (reader (cond ((proto-reader field)
 586                               `(,(proto-reader field) ,vobj))
 587                              ((proto-value field)
 588                               `(slot-value ,vobj ',(proto-value field)))))
 589                (index  (proto-index field)))
 590           (when reader
 591             (cond ((eq (proto-required field) :repeated)
 592                    (cond ((and (proto-packed field) (packed-type-p class))
 593                           (collect-sizer
 594                            (let ((tag (make-tag class index)))
 595                              `(iincf ,vsize (packed-size ,reader ,class ,tag)))))
 596                          ((keywordp class)
 597                           (collect-sizer
 598                            (let ((tag (make-tag class index)))
 599                              `(dolist (,vval ,reader)
 600                                 (iincf ,vsize (prim-size ,vval ,class ,tag))))))
 601                          ((typep msg 'protobuf-message)
 602                           (collect-sizer
 603                            (let ((tag (make-tag $wire-type-string index)))
 604                              `(dolist (,vval ,reader)
 605                                 ;; Call 'object-size' with the name of the message
 606                                 ;; class so that we preferentially get any optimized version
 607                                 ;; of the method
 608                                 (let ((len (or (and visited (gethash ,vval visited))
 609                                                (object-size ,vval ,msg visited))))
 610                                   (iincf ,vsize (length32 ,tag))
 611                                   (iincf ,vsize (length32 len))
 612                                   (iincf ,vsize len))))))
 613                          ((typep msg 'protobuf-enum)
 614                           (let ((tag (make-tag $wire-type-varint index)))
 615                             (collect-sizer
 616                              `(dolist (,vval ,reader)
 617                                 (iincf ,vsize (enum-size ,vval '(,@(proto-values msg)) ,tag))))))))
 618                   (t
 619                    (cond ((keywordp class)
 620                           (let ((tag (make-tag class index)))
 621                             (collect-sizer
 622                              (if (eq class :bool)
 623                                `(let ((,vval ,reader))
 624                                   (iincf ,vsize (prim-size ,vval ,class ,tag)))
 625                                `(let ((,vval ,reader))
 626                                   (when ,vval
 627                                     (iincf ,vsize (prim-size ,vval ,class ,tag))))))))
 628                          ((typep msg 'protobuf-message)
 629                           (collect-sizer
 630                            (let ((tag (make-tag $wire-type-string index)))
 631                              `(let ((,vval ,reader))
 632                                 (when ,vval
 633                                   (let ((len (or (and visited (gethash ,vval visited))
 634                                                  (object-size ,vval ,msg visited))))
 635                                     (iincf ,vsize (length32 ,tag))
 636                                     (iincf ,vsize (length32 len))
 637                                     (iincf ,vsize len)))))))
 638                          ((typep msg 'protobuf-enum)
 639                           (let ((tag (make-tag $wire-type-varint index)))
 640                             (collect-sizer
 641                              `(let ((,vval ,reader))
 642                                 (when ,vval
 643                                   (iincf ,vsize (enum-size ,vval '(,@(proto-values msg)) ,tag)))))))))))))
 644       `(defmethod object-size
 645            (,vobj (,vclass (eql ,message)) &optional visited)
 646          (declare (ignorable visited))
 647          (locally (declare (optimize (speed 3) (safety 0) (debug 0)))
 648            (let ((,vsize (and visited (gethash ,vobj visited))))
 649              (when ,vsize
 650                (return-from object-size ,vsize)))
 651            (let ((,vsize 0))
 652              (declare (type fixnum ,vsize))
 653              ,@sizers
 654              (when visited
 655                (setf (gethash ,vobj visited) ,vsize))
 656              ,vsize))))))