1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
3 ;;; Free Software published under an MIT-like license. See LICENSE ;;;
5 ;;; Copyright (c) 2012 Google, Inc. All rights reserved. ;;;
7 ;;; Original author: Scott McKay ;;;
9 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
11 (in-package "PROTO-IMPL")
14 ;;; Optimized fixnum arithmetic
16 (eval-when (:compile-toplevel :load-toplevel :execute)
18 (defparameter $optimize-default '(optimize (speed 1) (safety 3) (debug 3))
19 "Compiler optimization settings for safe, debuggable code.")
20 (defparameter $optimize-fast-unsafe '(optimize (speed 3) (safety 0) (debug 0))
21 "Compiler optimization settings for fast, unsafe, hard-to-debug code.")
26 (defmacro i+ (&rest fixnums)
27 `(the fixnum (+ ,@(loop for n in fixnums collect `(the fixnum ,n)))))
29 (defmacro i- (number &rest fixnums)
30 `(the fixnum (- (the fixnum ,number) ,@(loop for n in fixnums collect `(the fixnum ,n)))))
32 (defmacro i* (&rest fixnums)
33 `(the fixnum (* ,@(loop for n in fixnums collect `(the fixnum ,n)))))
35 (defmacro i= (&rest fixnums)
36 `(= ,@(loop for n in fixnums collect `(the fixnum ,n))))
38 (defmacro i< (&rest fixnums)
39 `(< ,@(loop for n in fixnums collect `(the fixnum ,n))))
41 (defmacro i<= (&rest fixnums)
42 `(<= ,@(loop for n in fixnums collect `(the fixnum ,n))))
44 (defmacro i> (&rest fixnums)
45 `(> ,@(loop for n in fixnums collect `(the fixnum ,n))))
47 (defmacro i>= (&rest fixnums)
48 `(>= ,@(loop for n in fixnums collect `(the fixnum ,n))))
50 (defmacro iash (value count)
51 `(the fixnum (ash (the fixnum ,value) (the fixnum ,count))))
53 (defmacro ilogior (&rest fixnums)
55 `(the fixnum (logior (the fixnum ,(car fixnums))
57 `(ilogior ,@(cdr fixnums))
58 `(the fixnum ,(cadr fixnums)))))
59 `(the fixnum ,(car fixnums))))
61 (defmacro ilogand (&rest fixnums)
63 `(the fixnum (logand (the fixnum ,(car fixnums))
65 `(ilogand ,@(cdr fixnums))
66 `(the fixnum ,(cadr fixnums)))))
67 `(the fixnum ,(car fixnums))))
69 (define-modify-macro iincf (&optional (delta 1)) i+)
70 (define-modify-macro idecf (&optional (delta 1)) i-)
72 (defmacro ildb (bytespec value)
73 `(the fixnum (ldb ,bytespec (the fixnum ,value))))
78 (defun starts-with (string prefix &key (start 0))
79 "Returns true if 'string' starts with the prefix 'prefix' (case insensitive)."
80 (and (i>= (length string) (i+ start (length prefix)))
81 (string-equal string prefix :start1 start :end1 (i+ start (length prefix)))
84 (defun ends-with (string suffix &key (end (length string)))
85 "Returns true if 'string' ends with the prefix 'prefix' (case insensitive)."
86 (and (i>= end (length suffix))
87 (string-equal string suffix :start1 (i- end (length suffix)) :end1 end)
90 (defun strcat (&rest strings)
91 "Concatenate a bunch of strings."
92 (declare (dynamic-extent strings))
93 (apply #'concatenate 'string strings))
96 ;; (camel-case "camel-case") => "CamelCase"
97 (defun camel-case (string &optional (separators '(#\-)))
98 (let ((words (split-string string :separators separators)))
99 (format nil "~{~@(~A~)~}" words)))
101 ;; (camel-case-but-one "camel-case") => "camelCase"
102 (defun camel-case-but-one (string &optional (separators '(#\-)))
103 (let ((words (split-string string :separators separators)))
104 (format nil "~(~A~)~{~@(~A~)~}" (car words) (cdr words))))
107 ;; (uncamel-case "CamelCase") => "CAMEL-CASE"
108 ;; (uncamel-case "TCPConnection") => "TCP-CONNECTION"
109 ;; (uncamel-case "NewTCPConnection") => "NEW-TCP-CONNECTION"
110 ;; (uncamel-case "new_RPC_LispService") => "NEW-RPC-LISP-SERVICE"
111 ;; (uncamel-case "RPC_LispServiceRequest_get_request") => "RPC-LISP-SERVICE-REQUEST-GET-REQUEST"
112 ;; (uncamel-case "TCP2Name3") => "TCP2-NAME3"
113 (defun uncamel-case (name)
114 ;; We need a whole state machine to get this right
115 (labels ((uncamel (chars state result)
116 (let ((ch (first chars)))
120 (uncamel (rest chars) 'upper
123 ;; "TCPConnection" => "TCP-CONNECTION"
124 (if (and (second chars) (lower-case-p (second chars)))
125 (list* ch #\- result)
127 ((lower digit) (list* ch #\- result))
128 (otherwise (cons ch result)))))
130 (uncamel (rest chars) 'lower
131 (cons (char-upcase ch) result)))
133 (uncamel (rest chars) 'digit
135 ((or (eql ch #\-) (eql ch #\_))
136 (uncamel (rest chars) 'dash
139 (uncamel (rest chars) 'dot
142 (error "Invalid name character: ~A" ch))))))
143 (strcat (nreverse (uncamel (concatenate 'list name) nil ())))))
146 (defun split-string (line &key (start 0) (end (length line)) (separators '(#\-)))
147 "Given a string 'string', splits it at each of the separators.
148 Returns a list of the string pieces, with empty pieces removed."
149 (unless (i= start end)
150 (loop for this fixnum = start then (i+ next 1)
151 for next fixnum = (or (position-if #'(lambda (ch) (member ch separators)) line
152 :start this :end end)
154 for piece = (string-right-trim '(#\space) (subseq line this next))
155 when (not (i= (length piece) 0))
157 until (i>= next end))))
162 (defmacro with-gensyms ((&rest bindings) &body body)
163 `(let ,(mapcar #'(lambda (b) `(,b (gensym ,(string b)))) bindings)
166 (defun make-lisp-symbol (string)
167 "Intern a string of the 'package:string' and return the symbol."
168 (let* ((string (string string))
169 (colon (position #\: string))
170 (pkg (if colon (subseq string 0 colon) "KEYWORD"))
171 (sym (if colon (subseq string (+ colon 1)) string)))
174 (defun fintern (format-string &rest format-args)
175 "Interns a new symbol in the current package."
176 (declare (dynamic-extent format-args))
177 (intern (nstring-upcase (apply #'format nil format-string format-args))))
179 (defun kintern (format-string &rest format-args)
180 "Interns a new symbol in the keyword package."
181 (declare (dynamic-extent format-args))
182 (intern (nstring-upcase (apply #'format nil format-string format-args)) "KEYWORD"))
184 (defun keywordify (x)
185 "Given a symbol designator 'x', return a keyword whose name is 'x'.
186 If 'x' is nil, this returns nil."
187 (check-type x (or string symbol null))
190 ((symbolp x) (keywordify (symbol-name x)))
191 ((zerop (length x)) nil)
192 ((string-not-equal x "nil")
193 (intern (string-upcase x) (find-package "KEYWORD")))
199 (defmacro with-collectors ((&rest collection-descriptions) &body body)
200 (let ((let-bindings ())
204 (dolist (description collection-descriptions)
205 (destructuring-bind (place name) description
206 (let ((vtail (make-symbol (format nil "~A-TAIL" place))))
207 (setq dynamic-extents
208 (nconc dynamic-extents `(#',name)))
216 (setq ,vtail (if ,vtail
217 (setf (cdr ,vtail) (list ,vobj))
218 (setf ,place (list ,vobj)))))))))))
219 `(let (,@let-bindings)
220 (flet (,@flet-bindings)
221 ,@(and dynamic-extents
222 `((declare (dynamic-extent ,@dynamic-extents))))
225 (defmacro with-prefixed-accessors (names (prefix object) &body body)
226 `(with-accessors (,@(loop for name in names
227 collect `(,name ,(fintern "~A~A" prefix name))))
231 (defmacro dovector ((var vector &optional value) &body body)
232 (with-gensyms (vidx vlen vvec)
233 `(let* ((,vvec ,vector)
234 (,vlen (length ,vvec)))
235 (loop for ,vidx fixnum from 0 below ,vlen
236 as ,var = (aref ,vvec ,vidx)
238 finally (return ,value)))))
241 ;;; Functional programming, please
243 (defun curry (function &rest args)
244 (if (and args (null (cdr args))) ;fast test for length = 1
245 (let ((arg (car args)))
246 #'(lambda (&rest more-args)
247 (apply function arg more-args)))
248 #'(lambda (&rest more-args)
249 (apply function (append args more-args)))))
251 (define-compiler-macro curry (&whole form function &rest args &environment env)
252 (declare (ignore env))
253 (if (and (listp function)
254 (eq (first function) 'function)
255 (symbolp (second function))
256 (and args (null (cdr args))))
257 `#'(lambda (&rest more-args)
258 (apply ,function ,(car args) more-args))
264 ;; A parameterized list type for repeated fields
265 ;; The elements aren't type-checked
266 (deftype list-of (type)
271 ;; The same, but use a (stretchy) vector
272 (deftype vector-of (type)
275 '(array * (*)))) ;an 1-dimensional array of any type
277 ;; This corresponds to the :bytes Protobufs type
278 (deftype byte-vector () '(array (unsigned-byte 8) (*)))
280 (defun make-byte-vector (size)
281 (make-array size :element-type '(unsigned-byte 8)))
283 ;; The Protobufs integer types
284 (deftype int32 () '(signed-byte 32))
285 (deftype int64 () '(signed-byte 64))
286 (deftype uint32 () '(unsigned-byte 32))
287 (deftype uint64 () '(unsigned-byte 64))
288 (deftype sint32 () '(signed-byte 32))
289 (deftype sint64 () '(signed-byte 64))
290 (deftype fixed32 () '(signed-byte 32))
291 (deftype fixed64 () '(signed-byte 64))
292 (deftype sfixed32 () '(signed-byte 32))
293 (deftype sfixed64 () '(signed-byte 64))
296 ;;; Code generation utilities
298 (defvar *proto-name-separators* '(#\- #\_ #\/ #\space))
299 (defvar *camel-case-field-names* nil)
301 (defun find-proto-package (name)
302 "A very fuzzy definition of 'find-package'."
305 ;; Try looking under the given name and the all-uppercase name
306 (or (find-package (string name))
307 (find-package (string-upcase (string name)))))
309 ;; If 'name' is a list, it's actually a fully-qualified path
310 (or (find-proto-package (first name))
311 (find-proto-package (format nil "~{~A~^.~}" name))))))
313 ;; "class-name" -> "ClassName", ("ClassName")
314 ;; "outer-class.inner-class" -> "InnerClass", ("OuterClass" "InnerClass")
315 (defun class-name->proto (x)
316 "Given a Lisp class name, returns a Protobufs message or enum name.
317 The second value is the fully qualified name, as a list."
318 (let* ((xs (split-string (string x) :separators '(#\.)))
319 (ns (loop for x in (butlast xs)
320 collect (remove-if-not #'alphanumericp
321 (camel-case (format nil "~A" x) *proto-name-separators*))))
323 (name (remove-if-not #'alphanumericp (camel-case nx *proto-name-separators*))))
324 (values name (append ns (list name))
325 ;; This might be the name of a package, too
326 (format nil "~{~A~^.~}" (butlast xs)))))
328 ;; "enum-value" -> "ENUM_VALUE", ("ENUM_VALUE")
329 ;; "class-name.enum-value" -> "ENUM_VALUE", ("ClassName" "ENUM_VALUE")
330 (defun enum-name->proto (x &optional prefix)
331 "Given a Lisp enum value name, returns a Protobufs enum value name.
332 The second value is the fully qualified name, as a list."
333 (let* ((xs (split-string (string x) :separators '(#\.)))
334 (ns (loop for x in (butlast xs)
335 collect (remove-if-not #'alphanumericp
336 (camel-case (format nil "~A" x) *proto-name-separators*))))
337 (nx (string-upcase (car (last xs))))
338 (nx (if (and prefix (starts-with nx prefix)) (subseq nx (length prefix)) nx))
339 ;; Keep underscores, they are standards separators in Protobufs enum names
340 (name (remove-if-not #'(lambda (x) (or (alphanumericp x) (eql x #\_)))
341 (format nil "~{~A~^_~}"
342 (split-string nx :separators *proto-name-separators*)))))
343 (values name (append ns (list name))
344 (format nil "~{~A~^.~}" (butlast xs)))))
346 ;; "slot-name" -> "slot_name", ("slot_name") or "slotName", ("slotName")
347 ;; "class-name.slot-name" -> "Class.slot_name", ("ClassName" "slot_name")
348 (defun slot-name->proto (x)
349 "Given a Lisp slot name, returns a Protobufs field name.
350 The second value is the fully qualified name, as a list."
351 (let* ((xs (split-string (string x) :separators '(#\.)))
352 (ns (loop for x in (butlast xs)
353 collect (remove-if-not #'alphanumericp
354 (camel-case (format nil "~A" x) *proto-name-separators*))))
355 (nx (string-downcase (car (last xs))))
356 (name (if *camel-case-field-names*
357 (remove-if-not #'alphanumericp
358 (camel-case-but-one (format nil "~A" nx) *proto-name-separators*))
359 ;; Keep underscores, they are standards separators in Protobufs field names
360 (remove-if-not #'(lambda (x) (or (alphanumericp x) (eql x #\_)))
361 (format nil "~{~A~^_~}"
362 (split-string nx :separators *proto-name-separators*))))))
363 (values name (append ns (list name))
364 (format nil "~{~A~^.~}" (butlast xs)))))
367 ;; "ClassName" -> 'class-name
368 ;; "cl-user.ClassName" -> 'cl-user::class-name
369 ;; "cl-user.OuterClass.InnerClass" -> 'cl-user::outer-class.inner-class
370 (defun proto->class-name (x &optional package)
371 "Given a Protobufs message or enum type name, returns a Lisp class or type name.
372 This resolves Protobufs qualified names as best as it can."
373 (let* ((xs (split-string (substitute #\- #\_ (uncamel-case x))
375 (pkg1 (and (cdr xs) (find-proto-package (first xs))))
376 (pkgn (and (cdr xs) (find-proto-package (butlast xs))))
377 (package (or pkg1 pkgn package))
378 (name (format nil "~{~A~^.~}" (if pkg1 (cdr xs) (if pkgn (last xs) xs)))))
379 (values (if package (intern name package) (make-symbol name)) package xs
380 ;; This might be the name of a package, too
381 (format nil "~{~A~^.~}" (butlast xs)))))
383 ;; "ENUM_VALUE" -> :enum-value
384 ;; "cl-user.ENUM_VALUE" -> :enum-value
385 ;; "cl-user.OuterClass.ENUM_VALUE" -> :enum-value
386 (defun proto->enum-name (x &optional package)
387 "Given a Protobufs enum value name, returns a Lisp enum value name.
388 This resolves Protobufs qualified names as best as it can."
389 (let* ((xs (split-string (substitute #\- #\_ (uncamel-case x))
391 (pkg1 (and (cdr xs) (find-proto-package (first xs))))
392 (pkgn (and (cdr xs) (find-proto-package (butlast xs))))
393 (package (or pkg1 pkgn package))
394 (name (format nil "~{~A~^.~}" (if pkg1 (cdr xs) (if pkgn (last xs) xs)))))
395 (values (kintern name) package xs
396 (format nil "~{~A~^.~}" (butlast xs)))))
398 ;; "slot_name" or "slotName" -> 'slot-name
399 ;; "cl-user.slot_name" or "cl-user.slotName" -> 'cl-user::slot-name
400 ;; "cl-user.OuterClass.slot_name" -> 'cl-user::outer-class.slot-name
401 (defun proto->slot-name (x &optional package)
402 "Given a Protobufs field value name, returns a Lisp slot name.
403 This resolves Protobufs qualified names as best as it can."
404 (let* ((xs (split-string (substitute #\- #\_ (uncamel-case x))
406 (pkg1 (and (cdr xs) (find-proto-package (first xs))))
407 (pkgn (and (cdr xs) (find-proto-package (butlast xs))))
408 (package (or pkg1 pkgn package))
409 (name (format nil "~{~A~^.~}" (if pkg1 (cdr xs) (if pkgn (last xs) xs)))))
410 (values (if package (intern name package) (make-symbol name)) package xs
411 (format nil "~{~A~^.~}" (butlast xs)))))
416 (define-condition protobufs-warning (warning simple-condition) ())
418 (defun protobufs-warn (format-control &rest format-arguments)
419 (warn 'protobufs-warning
420 :format-control format-control
421 :format-arguments format-arguments))
424 #-(or allegro lispworks)
425 (defmacro without-redefinition-warnings (() &body body)
429 (defmacro without-redefinition-warnings (() &body body)
430 `(excl:without-redefinition-warnings ,@body))
433 (defmacro without-redefinition-warnings (() &body body)
434 `(let ((dspec:*redefinition-action* :quiet)) ,@body))
437 ;;; Portable floating point utilities
439 #+(or abcl allegro cmu sbcl lispworks)
440 (defun single-float-bits (x)
441 (declare (type single-float x))
442 #+abcl (system:single-float-bits x)
443 #+allegro (multiple-value-bind (high low)
444 (excl:single-float-to-shorts x)
445 (declare (type (unsigned-byte 16) high low))
446 (logior (ash high 16) low))
447 #+cmu (kernel:single-float-bits x)
448 #+sbcl (sb-kernel:single-float-bits x)
449 #+lispworks (lispworks-float:single-float-bits x))
451 #-(or abcl allegro cmu sbcl lispworks)
452 (defun single-float-bits (x)
453 (declare (type single-float x))
454 (assert (= (float-radix x) 2))
456 (if (eql x 0.0f0) 0 #x-80000000)
457 (multiple-value-bind (lisp-significand lisp-exponent lisp-sign)
458 (integer-decode-float x)
459 (assert (plusp lisp-significand))
460 (let* ((significand lisp-significand)
461 (exponent (+ lisp-exponent 23 127))
463 (if (plusp exponent) ;if not obviously denormalized
466 ;; Special termination case for denormalized float number
468 ;; Denormalized numbers have exponent one greater than
469 ;; in the exponent field
470 (return (ash significand -1)))
471 ;; Ordinary termination case
472 ((>= significand (expt 2 23))
473 (assert (< 0 significand (expt 2 24)))
474 ;; Exponent 0 is reserved for denormalized numbers,
475 ;; and 255 is reserved for specials like NaN
476 (assert (< 0 exponent 255))
477 (return (logior (ash exponent 23)
478 (logand significand (1- (ash 1 23))))))
480 ;; Shift as necessary to set bit 24 of significand
481 (setq significand (ash significand 1)
482 exponent (1- exponent)))))
483 (do () ((zerop exponent)
484 ;; Denormalized numbers have exponent one greater than
485 ;; the exponent field
486 (ash significand -1))
487 (unless (zerop (logand significand 1))
488 (warn "Denormalized '~S' losing bits in ~D" 'single-float-bits x))
489 (setq significand (ash significand -1)
490 exponent (1+ exponent))))))
492 ((1) unsigned-result)
493 ((-1) (logior unsigned-result (- (expt 2 31)))))))))
496 #+(or abcl allegro cmu sbcl lispworks)
497 (defun double-float-bits (x)
498 (declare (type double-float x))
499 #+abcl (values (system:double-float-low-bits x)
500 (system:double-float-high-bits x))
501 #+allegro (multiple-value-bind (us3 us2 us1 us0)
502 (excl:double-float-to-shorts x)
503 (logior (ash us1 16) us0)
504 (logior (ash us3 16) us2))
505 #+cmu (values (kernel:double-float-low-bits x)
506 (kernel:double-float-high-bits x))
507 #+sbcl (values (sb-kernel:double-float-low-bits x)
508 (sb-kernel:double-float-high-bits x))
509 #+lispworks (let ((bits (lispworks-float:double-float-bits x)))
510 (values (logand #xffffffff bits)
513 #-(or abcl allegro cmu sbcl lispworks)
514 (defun double-float-bits (x)
515 (declare (type double-float x))
516 (assert (= (float-radix x) 2))
518 (if (eql x 0.0d0) 0 #x-8000000000000000)
519 (multiple-value-bind (lisp-significand lisp-exponent lisp-sign)
520 (integer-decode-float x)
521 (assert (plusp lisp-significand))
522 (let* ((significand lisp-significand)
523 (exponent (+ lisp-exponent 52 1023))
525 (if (plusp exponent) ;if not obviously denormalized
528 ;; Special termination case for denormalized float number
530 ;; Denormalized numbers have exponent one greater than
531 ;; in the exponent field
532 (return (ash significand -1)))
533 ;; Ordinary termination case
534 ((>= significand (expt 2 52))
535 (assert (< 0 significand (expt 2 53)))
536 ;; Exponent 0 is reserved for denormalized numbers,
537 ;; and 2047 is reserved for specials like NaN
538 (assert (< 0 exponent 2047))
539 (return (logior (ash exponent 52)
540 (logand significand (1- (ash 1 52))))))
542 ;; Shift as necessary to set bit 53 of significand
543 (setq significand (ash significand 1)
544 exponent (1- exponent)))))
545 (do () ((zerop exponent)
546 ;; Denormalized numbers have exponent one greater than
547 ;; the exponent field
548 (ash significand -1))
549 (unless (zerop (logand significand 1))
550 (warn "Denormalized '~S' losing bits in ~D" 'double-float-bits x))
551 (setq significand (ash significand -1)
552 exponent (1+ exponent))))))
555 ((1) unsigned-result)
556 ((-1) (logior unsigned-result (- (expt 2 63)))))))
557 ;; Return the low bits and the high bits
558 (values (logand #xffffffff result) (ash result -32)))))))
561 #+(or abcl allegro cmu sbcl lispworks)
562 (defun make-single-float (bits)
563 (declare (type (signed-byte 32) bits))
564 #+abcl (system:make-single-float bits)
565 #+allegro (excl:shorts-to-single-float (ldb (byte 16 16) bits)
566 (ldb (byte 16 0) bits))
567 #+cmu (kernel:make-single-float bits)
568 #+sbcl (sb-kernel:make-single-float bits)
569 #+lispworks (lispworks-float:make-single-float bits))
571 #-(or abcl allegro cmu sbcl lispworks)
572 (defun make-single-float (bits)
573 (declare (type (signed-byte 32) bits))
575 ;; IEEE float special cases
577 ((= bits #x-80000000) -0.0)
579 (let* ((sign (ecase (ldb (byte 1 31) bits)
582 (iexpt (ldb (byte 8 23) bits))
583 (exponent (if (zerop iexpt) ;denormalized
586 (mantissa (* (logior (ldb (byte 23 0) bits)
587 (if (zerop iexpt) 0 (ash 1 23)))
589 (* sign (expt 2.0 exponent) mantissa)))))
592 #+(or abcl allegro cmu sbcl lispworks)
593 (defun make-double-float (low high)
594 (declare (type (unsigned-byte 32) low)
595 (type (signed-byte 32) high))
596 #+abcl (system:make-double-float (logior (ash high 32) low))
597 #+allegro (excl:shorts-to-double-float (ldb (byte 16 16) high)
598 (ldb (byte 16 0) high)
599 (ldb (byte 16 16) low)
600 (ldb (byte 16 0) low))
601 #+cmu (kernel:make-double-float high low)
602 #+sbcl (sb-kernel:make-double-float high low)
603 #+lispworks (lispworks-float:make-double-float high low))
605 #-(or abcl allegro cmu sbcl lispworks)
606 (defun make-double-float (low high)
607 (declare (type (unsigned-byte 32) low)
608 (type (signed-byte 32) high))
610 ;; IEEE float special cases
611 ((and (zerop high) (zerop low)) 0.0d0)
612 ((and (= high #x-80000000)
615 (let* ((bits (logior (ash high 32) low))
616 (sign (ecase (ldb (byte 1 63) bits)
619 (iexpt (ldb (byte 11 52) bits))
620 (exponent (if (zerop iexpt) ;denormalized
623 (mantissa (* (logior (ldb (byte 52 0) bits)
624 (if (zerop iexpt) 0 (ash 1 52)))
626 (* sign (expt 2.0d0 exponent) mantissa)))))