| Location | Tool | Test ID | Function | Issue |
|---|---|---|---|---|
| /builddir/build/BUILD/Python-2.7.3/Python/marshal.c:114:5 | clang-analyzer | The left operand of '!=' is a garbage value | ||
| /builddir/build/BUILD/Python-2.7.3/Python/marshal.c:114:5 | clang-analyzer | The left operand of '!=' is a garbage value | ||
| /builddir/build/BUILD/Python-2.7.3/Python/marshal.c:608:24 | clang-analyzer | The result of the '<<' expression is undefined | ||
| /builddir/build/BUILD/Python-2.7.3/Python/marshal.c:608:24 | clang-analyzer | The result of the '<<' expression is undefined |
1 /* Write Python objects to files and read them back.
2 This is intended for writing and reading compiled Python code only;
3 a true persistent storage facility would be much harder, since
4 it would have to take circular links and sharing into account. */
5
6 #define PY_SSIZE_T_CLEAN
7
8 #include "Python.h"
9 #include "longintrepr.h"
10 #include "code.h"
11 #include "marshal.h"
12
13 #define ABS(x) ((x) < 0 ? -(x) : (x))
14
15 /* High water mark to determine when the marshalled object is dangerously deep
16 * and risks coring the interpreter. When the object stack gets this deep,
17 * raise an exception instead of continuing.
18 */
19 #define MAX_MARSHAL_STACK_DEPTH 2000
20
21 #define TYPE_NULL '0'
22 #define TYPE_NONE 'N'
23 #define TYPE_FALSE 'F'
24 #define TYPE_TRUE 'T'
25 #define TYPE_STOPITER 'S'
26 #define TYPE_ELLIPSIS '.'
27 #define TYPE_INT 'i'
28 #define TYPE_INT64 'I'
29 #define TYPE_FLOAT 'f'
30 #define TYPE_BINARY_FLOAT 'g'
31 #define TYPE_COMPLEX 'x'
32 #define TYPE_BINARY_COMPLEX 'y'
33 #define TYPE_LONG 'l'
34 #define TYPE_STRING 's'
35 #define TYPE_INTERNED 't'
36 #define TYPE_STRINGREF 'R'
37 #define TYPE_TUPLE '('
38 #define TYPE_LIST '['
39 #define TYPE_DICT '{'
40 #define TYPE_CODE 'c'
41 #define TYPE_UNICODE 'u'
42 #define TYPE_UNKNOWN '?'
43 #define TYPE_SET '<'
44 #define TYPE_FROZENSET '>'
45
46 #define WFERR_OK 0
47 #define WFERR_UNMARSHALLABLE 1
48 #define WFERR_NESTEDTOODEEP 2
49 #define WFERR_NOMEMORY 3
50
51 typedef struct {
52 FILE *fp;
53 int error; /* see WFERR_* values */
54 int depth;
55 /* If fp == NULL, the following are valid: */
56 PyObject *str;
57 char *ptr;
58 char *end;
59 PyObject *strings; /* dict on marshal, list on unmarshal */
60 int version;
61 } WFILE;
62
63 #define w_byte(c, p) if (((p)->fp)) putc((c), (p)->fp); \
64 else if ((p)->ptr != (p)->end) *(p)->ptr++ = (c); \
65 else w_more(c, p)
66
67 static void
68 w_more(int c, WFILE *p)
69 {
70 Py_ssize_t size, newsize;
71 if (p->str == NULL)
72 return; /* An error already occurred */
73 size = PyString_Size(p->str);
74 newsize = size + size + 1024;
75 if (newsize > 32*1024*1024) {
76 newsize = size + (size >> 3); /* 12.5% overallocation */
77 }
78 if (_PyString_Resize(&p->str, newsize) != 0) {
79 p->ptr = p->end = NULL;
80 }
81 else {
82 p->ptr = PyString_AS_STRING((PyStringObject *)p->str) + size;
83 p->end =
84 PyString_AS_STRING((PyStringObject *)p->str) + newsize;
85 *p->ptr++ = Py_SAFE_DOWNCAST(c, int, char);
86 }
87 }
88
89 static void
90 w_string(char *s, int n, WFILE *p)
91 {
92 if (p->fp != NULL) {
93 fwrite(s, 1, n, p->fp);
94 }
95 else {
96 while (--n >= 0) {
97 w_byte(*s, p);
98 s++;
99 }
100 }
101 }
102
103 static void
104 w_short(int x, WFILE *p)
105 {
106 w_byte((char)( x & 0xff), p);
107 w_byte((char)((x>> 8) & 0xff), p);
108 }
109
110 static void
111 w_long(long x, WFILE *p)
112 {
113 w_byte((char)( x & 0xff), p);
114 w_byte((char)((x>> 8) & 0xff), p);
(emitted by clang-analyzer)TODO: a detailed trace is available in the data model (not yet rendered in this report)
(emitted by clang-analyzer)TODO: a detailed trace is available in the data model (not yet rendered in this report)
115 w_byte((char)((x>>16) & 0xff), p);
116 w_byte((char)((x>>24) & 0xff), p);
117 }
118
119 #if SIZEOF_LONG > 4
120 static void
121 w_long64(long x, WFILE *p)
122 {
123 w_long(x, p);
124 w_long(x>>32, p);
125 }
126 #endif
127
128 /* We assume that Python longs are stored internally in base some power of
129 2**15; for the sake of portability we'll always read and write them in base
130 exactly 2**15. */
131
132 #define PyLong_MARSHAL_SHIFT 15
133 #define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)
134 #define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)
135 #if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0
136 #error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"
137 #endif
138 #define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)
139
140 static void
141 w_PyLong(const PyLongObject *ob, WFILE *p)
142 {
143 Py_ssize_t i, j, n, l;
144 digit d;
145
146 w_byte(TYPE_LONG, p);
147 if (Py_SIZE(ob) == 0) {
148 w_long((long)0, p);
149 return;
150 }
151
152 /* set l to number of base PyLong_MARSHAL_BASE digits */
153 n = ABS(Py_SIZE(ob));
154 l = (n-1) * PyLong_MARSHAL_RATIO;
155 d = ob->ob_digit[n-1];
156 assert(d != 0); /* a PyLong is always normalized */
157 do {
158 d >>= PyLong_MARSHAL_SHIFT;
159 l++;
160 } while (d != 0);
161 w_long((long)(Py_SIZE(ob) > 0 ? l : -l), p);
162
163 for (i=0; i < n-1; i++) {
164 d = ob->ob_digit[i];
165 for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
166 w_short(d & PyLong_MARSHAL_MASK, p);
167 d >>= PyLong_MARSHAL_SHIFT;
168 }
169 assert (d == 0);
170 }
171 d = ob->ob_digit[n-1];
172 do {
173 w_short(d & PyLong_MARSHAL_MASK, p);
174 d >>= PyLong_MARSHAL_SHIFT;
175 } while (d != 0);
176 }
177
178 static void
179 w_object(PyObject *v, WFILE *p)
180 {
181 Py_ssize_t i, n;
182
183 p->depth++;
184
185 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
186 p->error = WFERR_NESTEDTOODEEP;
187 }
188 else if (v == NULL) {
189 w_byte(TYPE_NULL, p);
190 }
191 else if (v == Py_None) {
192 w_byte(TYPE_NONE, p);
193 }
194 else if (v == PyExc_StopIteration) {
195 w_byte(TYPE_STOPITER, p);
196 }
197 else if (v == Py_Ellipsis) {
198 w_byte(TYPE_ELLIPSIS, p);
199 }
200 else if (v == Py_False) {
201 w_byte(TYPE_FALSE, p);
202 }
203 else if (v == Py_True) {
204 w_byte(TYPE_TRUE, p);
205 }
206 else if (PyInt_CheckExact(v)) {
207 long x = PyInt_AS_LONG((PyIntObject *)v);
208 #if SIZEOF_LONG > 4
209 long y = Py_ARITHMETIC_RIGHT_SHIFT(long, x, 31);
210 if (y && y != -1) {
211 w_byte(TYPE_INT64, p);
212 w_long64(x, p);
213 }
214 else
215 #endif
216 {
217 w_byte(TYPE_INT, p);
218 w_long(x, p);
219 }
220 }
221 else if (PyLong_CheckExact(v)) {
222 PyLongObject *ob = (PyLongObject *)v;
223 w_PyLong(ob, p);
224 }
225 else if (PyFloat_CheckExact(v)) {
226 if (p->version > 1) {
227 unsigned char buf[8];
228 if (_PyFloat_Pack8(PyFloat_AsDouble(v),
229 buf, 1) < 0) {
230 p->error = WFERR_UNMARSHALLABLE;
231 return;
232 }
233 w_byte(TYPE_BINARY_FLOAT, p);
234 w_string((char*)buf, 8, p);
235 }
236 else {
237 char *buf = PyOS_double_to_string(PyFloat_AS_DOUBLE(v),
238 'g', 17, 0, NULL);
239 if (!buf) {
240 p->error = WFERR_NOMEMORY;
241 return;
242 }
243 n = strlen(buf);
244 w_byte(TYPE_FLOAT, p);
245 w_byte((int)n, p);
246 w_string(buf, (int)n, p);
247 PyMem_Free(buf);
248 }
249 }
250 #ifndef WITHOUT_COMPLEX
251 else if (PyComplex_CheckExact(v)) {
252 if (p->version > 1) {
253 unsigned char buf[8];
254 if (_PyFloat_Pack8(PyComplex_RealAsDouble(v),
255 buf, 1) < 0) {
256 p->error = WFERR_UNMARSHALLABLE;
257 return;
258 }
259 w_byte(TYPE_BINARY_COMPLEX, p);
260 w_string((char*)buf, 8, p);
261 if (_PyFloat_Pack8(PyComplex_ImagAsDouble(v),
262 buf, 1) < 0) {
263 p->error = WFERR_UNMARSHALLABLE;
264 return;
265 }
266 w_string((char*)buf, 8, p);
267 }
268 else {
269 char *buf;
270 w_byte(TYPE_COMPLEX, p);
271 buf = PyOS_double_to_string(PyComplex_RealAsDouble(v),
272 'g', 17, 0, NULL);
273 if (!buf) {
274 p->error = WFERR_NOMEMORY;
275 return;
276 }
277 n = strlen(buf);
278 w_byte((int)n, p);
279 w_string(buf, (int)n, p);
280 PyMem_Free(buf);
281 buf = PyOS_double_to_string(PyComplex_ImagAsDouble(v),
282 'g', 17, 0, NULL);
283 if (!buf) {
284 p->error = WFERR_NOMEMORY;
285 return;
286 }
287 n = strlen(buf);
288 w_byte((int)n, p);
289 w_string(buf, (int)n, p);
290 PyMem_Free(buf);
291 }
292 }
293 #endif
294 else if (PyString_CheckExact(v)) {
295 if (p->strings && PyString_CHECK_INTERNED(v)) {
296 PyObject *o = PyDict_GetItem(p->strings, v);
297 if (o) {
298 long w = PyInt_AsLong(o);
299 w_byte(TYPE_STRINGREF, p);
300 w_long(w, p);
301 goto exit;
302 }
303 else {
304 int ok;
305 o = PyInt_FromSsize_t(PyDict_Size(p->strings));
306 ok = o &&
307 PyDict_SetItem(p->strings, v, o) >= 0;
308 Py_XDECREF(o);
309 if (!ok) {
310 p->depth--;
311 p->error = WFERR_UNMARSHALLABLE;
312 return;
313 }
314 w_byte(TYPE_INTERNED, p);
315 }
316 }
317 else {
318 w_byte(TYPE_STRING, p);
319 }
320 n = PyString_GET_SIZE(v);
321 if (n > INT_MAX) {
322 /* huge strings are not supported */
323 p->depth--;
324 p->error = WFERR_UNMARSHALLABLE;
325 return;
326 }
327 w_long((long)n, p);
328 w_string(PyString_AS_STRING(v), (int)n, p);
329 }
330 #ifdef Py_USING_UNICODE
331 else if (PyUnicode_CheckExact(v)) {
332 PyObject *utf8;
333 utf8 = PyUnicode_AsUTF8String(v);
334 if (utf8 == NULL) {
335 p->depth--;
336 p->error = WFERR_UNMARSHALLABLE;
337 return;
338 }
339 w_byte(TYPE_UNICODE, p);
340 n = PyString_GET_SIZE(utf8);
341 if (n > INT_MAX) {
342 p->depth--;
343 p->error = WFERR_UNMARSHALLABLE;
344 return;
345 }
346 w_long((long)n, p);
347 w_string(PyString_AS_STRING(utf8), (int)n, p);
348 Py_DECREF(utf8);
349 }
350 #endif
351 else if (PyTuple_CheckExact(v)) {
352 w_byte(TYPE_TUPLE, p);
353 n = PyTuple_Size(v);
354 w_long((long)n, p);
355 for (i = 0; i < n; i++) {
356 w_object(PyTuple_GET_ITEM(v, i), p);
357 }
358 }
359 else if (PyList_CheckExact(v)) {
360 w_byte(TYPE_LIST, p);
361 n = PyList_GET_SIZE(v);
362 w_long((long)n, p);
363 for (i = 0; i < n; i++) {
364 w_object(PyList_GET_ITEM(v, i), p);
365 }
366 }
367 else if (PyDict_CheckExact(v)) {
368 Py_ssize_t pos;
369 PyObject *key, *value;
370 w_byte(TYPE_DICT, p);
371 /* This one is NULL object terminated! */
372 pos = 0;
373 while (PyDict_Next(v, &pos, &key, &value)) {
374 w_object(key, p);
375 w_object(value, p);
376 }
377 w_object((PyObject *)NULL, p);
378 }
379 else if (PyAnySet_CheckExact(v)) {
380 PyObject *value, *it;
381
382 if (PyObject_TypeCheck(v, &PySet_Type))
383 w_byte(TYPE_SET, p);
384 else
385 w_byte(TYPE_FROZENSET, p);
386 n = PyObject_Size(v);
387 if (n == -1) {
388 p->depth--;
389 p->error = WFERR_UNMARSHALLABLE;
390 return;
391 }
392 w_long((long)n, p);
393 it = PyObject_GetIter(v);
394 if (it == NULL) {
395 p->depth--;
396 p->error = WFERR_UNMARSHALLABLE;
397 return;
398 }
399 while ((value = PyIter_Next(it)) != NULL) {
400 w_object(value, p);
401 Py_DECREF(value);
402 }
403 Py_DECREF(it);
404 if (PyErr_Occurred()) {
405 p->depth--;
406 p->error = WFERR_UNMARSHALLABLE;
407 return;
408 }
409 }
410 else if (PyCode_Check(v)) {
411 PyCodeObject *co = (PyCodeObject *)v;
412 w_byte(TYPE_CODE, p);
413 w_long(co->co_argcount, p);
414 w_long(co->co_nlocals, p);
415 w_long(co->co_stacksize, p);
416 w_long(co->co_flags, p);
417 w_object(co->co_code, p);
418 w_object(co->co_consts, p);
419 w_object(co->co_names, p);
420 w_object(co->co_varnames, p);
421 w_object(co->co_freevars, p);
422 w_object(co->co_cellvars, p);
423 w_object(co->co_filename, p);
424 w_object(co->co_name, p);
425 w_long(co->co_firstlineno, p);
426 w_object(co->co_lnotab, p);
427 }
428 else if (PyObject_CheckReadBuffer(v)) {
429 /* Write unknown buffer-style objects as a string */
430 char *s;
431 PyBufferProcs *pb = v->ob_type->tp_as_buffer;
432 w_byte(TYPE_STRING, p);
433 n = (*pb->bf_getreadbuffer)(v, 0, (void **)&s);
434 if (n > INT_MAX) {
435 p->depth--;
436 p->error = WFERR_UNMARSHALLABLE;
437 return;
438 }
439 w_long((long)n, p);
440 w_string(s, (int)n, p);
441 }
442 else {
443 w_byte(TYPE_UNKNOWN, p);
444 p->error = WFERR_UNMARSHALLABLE;
445 }
446 exit:
447 p->depth--;
448 }
449
450 /* version currently has no effect for writing longs. */
451 void
452 PyMarshal_WriteLongToFile(long x, FILE *fp, int version)
453 {
454 WFILE wf;
455 wf.fp = fp;
456 wf.error = WFERR_OK;
457 wf.depth = 0;
458 wf.strings = NULL;
459 wf.version = version;
460 w_long(x, &wf);
461 }
462
463 void
464 PyMarshal_WriteObjectToFile(PyObject *x, FILE *fp, int version)
465 {
466 WFILE wf;
467 wf.fp = fp;
468 wf.error = WFERR_OK;
469 wf.depth = 0;
470 wf.strings = (version > 0) ? PyDict_New() : NULL;
471 wf.version = version;
472 w_object(x, &wf);
473 Py_XDECREF(wf.strings);
474 }
475
476 typedef WFILE RFILE; /* Same struct with different invariants */
477
478 #define rs_byte(p) (((p)->ptr < (p)->end) ? (unsigned char)*(p)->ptr++ : EOF)
479
480 #define r_byte(p) ((p)->fp ? getc((p)->fp) : rs_byte(p))
481
482 static int
483 r_string(char *s, int n, RFILE *p)
484 {
485 if (p->fp != NULL)
486 /* The result fits into int because it must be <=n. */
487 return (int)fread(s, 1, n, p->fp);
488 if (p->end - p->ptr < n)
489 n = (int)(p->end - p->ptr);
490 memcpy(s, p->ptr, n);
491 p->ptr += n;
492 return n;
493 }
494
495 static int
496 r_short(RFILE *p)
497 {
498 register short x;
499 x = r_byte(p);
500 x |= r_byte(p) << 8;
501 /* Sign-extension, in case short greater than 16 bits */
502 x |= -(x & 0x8000);
503 return x;
504 }
505
506 static long
507 r_long(RFILE *p)
508 {
509 register long x;
510 register FILE *fp = p->fp;
511 if (fp) {
512 x = getc(fp);
513 x |= (long)getc(fp) << 8;
514 x |= (long)getc(fp) << 16;
515 x |= (long)getc(fp) << 24;
516 }
517 else {
518 x = rs_byte(p);
519 x |= (long)rs_byte(p) << 8;
520 x |= (long)rs_byte(p) << 16;
521 x |= (long)rs_byte(p) << 24;
522 }
523 #if SIZEOF_LONG > 4
524 /* Sign extension for 64-bit machines */
525 x |= -(x & 0x80000000L);
526 #endif
527 return x;
528 }
529
530 /* r_long64 deals with the TYPE_INT64 code. On a machine with
531 sizeof(long) > 4, it returns a Python int object, else a Python long
532 object. Note that w_long64 writes out TYPE_INT if 32 bits is enough,
533 so there's no inefficiency here in returning a PyLong on 32-bit boxes
534 for everything written via TYPE_INT64 (i.e., if an int is written via
535 TYPE_INT64, it *needs* more than 32 bits).
536 */
537 static PyObject *
538 r_long64(RFILE *p)
539 {
540 long lo4 = r_long(p);
541 long hi4 = r_long(p);
542 #if SIZEOF_LONG > 4
543 long x = (hi4 << 32) | (lo4 & 0xFFFFFFFFL);
544 return PyInt_FromLong(x);
545 #else
546 unsigned char buf[8];
547 int one = 1;
548 int is_little_endian = (int)*(char*)&one;
549 if (is_little_endian) {
550 memcpy(buf, &lo4, 4);
551 memcpy(buf+4, &hi4, 4);
552 }
553 else {
554 memcpy(buf, &hi4, 4);
555 memcpy(buf+4, &lo4, 4);
556 }
557 return _PyLong_FromByteArray(buf, 8, is_little_endian, 1);
558 #endif
559 }
560
561 static PyObject *
562 r_PyLong(RFILE *p)
563 {
564 PyLongObject *ob;
565 int size, i, j, md, shorts_in_top_digit;
566 long n;
567 digit d;
568
569 n = r_long(p);
570 if (n == 0)
571 return (PyObject *)_PyLong_New(0);
572 if (n < -INT_MAX || n > INT_MAX) {
573 PyErr_SetString(PyExc_ValueError,
574 "bad marshal data (long size out of range)");
575 return NULL;
576 }
577
578 size = 1 + (ABS(n) - 1) / PyLong_MARSHAL_RATIO;
579 shorts_in_top_digit = 1 + (ABS(n) - 1) % PyLong_MARSHAL_RATIO;
580 ob = _PyLong_New(size);
581 if (ob == NULL)
582 return NULL;
583 Py_SIZE(ob) = n > 0 ? size : -size;
584
585 for (i = 0; i < size-1; i++) {
586 d = 0;
587 for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
588 md = r_short(p);
589 if (md < 0 || md > PyLong_MARSHAL_BASE)
590 goto bad_digit;
591 d += (digit)md << j*PyLong_MARSHAL_SHIFT;
592 }
593 ob->ob_digit[i] = d;
594 }
595 d = 0;
596 for (j=0; j < shorts_in_top_digit; j++) {
597 md = r_short(p);
598 if (md < 0 || md > PyLong_MARSHAL_BASE)
599 goto bad_digit;
600 /* topmost marshal digit should be nonzero */
601 if (md == 0 && j == shorts_in_top_digit - 1) {
602 Py_DECREF(ob);
603 PyErr_SetString(PyExc_ValueError,
604 "bad marshal data (unnormalized long data)");
605 return NULL;
606 }
607 d += (digit)md << j*PyLong_MARSHAL_SHIFT;
608 }
(emitted by clang-analyzer)TODO: a detailed trace is available in the data model (not yet rendered in this report)
(emitted by clang-analyzer)TODO: a detailed trace is available in the data model (not yet rendered in this report)
609 /* top digit should be nonzero, else the resulting PyLong won't be
610 normalized */
611 ob->ob_digit[size-1] = d;
612 return (PyObject *)ob;
613 bad_digit:
614 Py_DECREF(ob);
615 PyErr_SetString(PyExc_ValueError,
616 "bad marshal data (digit out of range in long)");
617 return NULL;
618 }
619
620
621 static PyObject *
622 r_object(RFILE *p)
623 {
624 /* NULL is a valid return value, it does not necessarily means that
625 an exception is set. */
626 PyObject *v, *v2;
627 long i, n;
628 int type = r_byte(p);
629 PyObject *retval;
630
631 p->depth++;
632
633 if (p->depth > MAX_MARSHAL_STACK_DEPTH) {
634 p->depth--;
635 PyErr_SetString(PyExc_ValueError, "recursion limit exceeded");
636 return NULL;
637 }
638
639 switch (type) {
640
641 case EOF:
642 PyErr_SetString(PyExc_EOFError,
643 "EOF read where object expected");
644 retval = NULL;
645 break;
646
647 case TYPE_NULL:
648 retval = NULL;
649 break;
650
651 case TYPE_NONE:
652 Py_INCREF(Py_None);
653 retval = Py_None;
654 break;
655
656 case TYPE_STOPITER:
657 Py_INCREF(PyExc_StopIteration);
658 retval = PyExc_StopIteration;
659 break;
660
661 case TYPE_ELLIPSIS:
662 Py_INCREF(Py_Ellipsis);
663 retval = Py_Ellipsis;
664 break;
665
666 case TYPE_FALSE:
667 Py_INCREF(Py_False);
668 retval = Py_False;
669 break;
670
671 case TYPE_TRUE:
672 Py_INCREF(Py_True);
673 retval = Py_True;
674 break;
675
676 case TYPE_INT:
677 retval = PyInt_FromLong(r_long(p));
678 break;
679
680 case TYPE_INT64:
681 retval = r_long64(p);
682 break;
683
684 case TYPE_LONG:
685 retval = r_PyLong(p);
686 break;
687
688 case TYPE_FLOAT:
689 {
690 char buf[256];
691 double dx;
692 n = r_byte(p);
693 if (n == EOF || r_string(buf, (int)n, p) != n) {
694 PyErr_SetString(PyExc_EOFError,
695 "EOF read where object expected");
696 retval = NULL;
697 break;
698 }
699 buf[n] = '\0';
700 dx = PyOS_string_to_double(buf, NULL, NULL);
701 if (dx == -1.0 && PyErr_Occurred()) {
702 retval = NULL;
703 break;
704 }
705 retval = PyFloat_FromDouble(dx);
706 break;
707 }
708
709 case TYPE_BINARY_FLOAT:
710 {
711 unsigned char buf[8];
712 double x;
713 if (r_string((char*)buf, 8, p) != 8) {
714 PyErr_SetString(PyExc_EOFError,
715 "EOF read where object expected");
716 retval = NULL;
717 break;
718 }
719 x = _PyFloat_Unpack8(buf, 1);
720 if (x == -1.0 && PyErr_Occurred()) {
721 retval = NULL;
722 break;
723 }
724 retval = PyFloat_FromDouble(x);
725 break;
726 }
727
728 #ifndef WITHOUT_COMPLEX
729 case TYPE_COMPLEX:
730 {
731 char buf[256];
732 Py_complex c;
733 n = r_byte(p);
734 if (n == EOF || r_string(buf, (int)n, p) != n) {
735 PyErr_SetString(PyExc_EOFError,
736 "EOF read where object expected");
737 retval = NULL;
738 break;
739 }
740 buf[n] = '\0';
741 c.real = PyOS_string_to_double(buf, NULL, NULL);
742 if (c.real == -1.0 && PyErr_Occurred()) {
743 retval = NULL;
744 break;
745 }
746 n = r_byte(p);
747 if (n == EOF || r_string(buf, (int)n, p) != n) {
748 PyErr_SetString(PyExc_EOFError,
749 "EOF read where object expected");
750 retval = NULL;
751 break;
752 }
753 buf[n] = '\0';
754 c.imag = PyOS_string_to_double(buf, NULL, NULL);
755 if (c.imag == -1.0 && PyErr_Occurred()) {
756 retval = NULL;
757 break;
758 }
759 retval = PyComplex_FromCComplex(c);
760 break;
761 }
762
763 case TYPE_BINARY_COMPLEX:
764 {
765 unsigned char buf[8];
766 Py_complex c;
767 if (r_string((char*)buf, 8, p) != 8) {
768 PyErr_SetString(PyExc_EOFError,
769 "EOF read where object expected");
770 retval = NULL;
771 break;
772 }
773 c.real = _PyFloat_Unpack8(buf, 1);
774 if (c.real == -1.0 && PyErr_Occurred()) {
775 retval = NULL;
776 break;
777 }
778 if (r_string((char*)buf, 8, p) != 8) {
779 PyErr_SetString(PyExc_EOFError,
780 "EOF read where object expected");
781 retval = NULL;
782 break;
783 }
784 c.imag = _PyFloat_Unpack8(buf, 1);
785 if (c.imag == -1.0 && PyErr_Occurred()) {
786 retval = NULL;
787 break;
788 }
789 retval = PyComplex_FromCComplex(c);
790 break;
791 }
792 #endif
793
794 case TYPE_INTERNED:
795 case TYPE_STRING:
796 n = r_long(p);
797 if (n < 0 || n > INT_MAX) {
798 PyErr_SetString(PyExc_ValueError, "bad marshal data (string size out of range)");
799 retval = NULL;
800 break;
801 }
802 v = PyString_FromStringAndSize((char *)NULL, n);
803 if (v == NULL) {
804 retval = NULL;
805 break;
806 }
807 if (r_string(PyString_AS_STRING(v), (int)n, p) != n) {
808 Py_DECREF(v);
809 PyErr_SetString(PyExc_EOFError,
810 "EOF read where object expected");
811 retval = NULL;
812 break;
813 }
814 if (type == TYPE_INTERNED) {
815 PyString_InternInPlace(&v);
816 if (PyList_Append(p->strings, v) < 0) {
817 retval = NULL;
818 break;
819 }
820 }
821 retval = v;
822 break;
823
824 case TYPE_STRINGREF:
825 n = r_long(p);
826 if (n < 0 || n >= PyList_GET_SIZE(p->strings)) {
827 PyErr_SetString(PyExc_ValueError, "bad marshal data (string ref out of range)");
828 retval = NULL;
829 break;
830 }
831 v = PyList_GET_ITEM(p->strings, n);
832 Py_INCREF(v);
833 retval = v;
834 break;
835
836 #ifdef Py_USING_UNICODE
837 case TYPE_UNICODE:
838 {
839 char *buffer;
840
841 n = r_long(p);
842 if (n < 0 || n > INT_MAX) {
843 PyErr_SetString(PyExc_ValueError, "bad marshal data (unicode size out of range)");
844 retval = NULL;
845 break;
846 }
847 buffer = PyMem_NEW(char, n);
848 if (buffer == NULL) {
849 retval = PyErr_NoMemory();
850 break;
851 }
852 if (r_string(buffer, (int)n, p) != n) {
853 PyMem_DEL(buffer);
854 PyErr_SetString(PyExc_EOFError,
855 "EOF read where object expected");
856 retval = NULL;
857 break;
858 }
859 v = PyUnicode_DecodeUTF8(buffer, n, NULL);
860 PyMem_DEL(buffer);
861 retval = v;
862 break;
863 }
864 #endif
865
866 case TYPE_TUPLE:
867 n = r_long(p);
868 if (n < 0 || n > INT_MAX) {
869 PyErr_SetString(PyExc_ValueError, "bad marshal data (tuple size out of range)");
870 retval = NULL;
871 break;
872 }
873 v = PyTuple_New((int)n);
874 if (v == NULL) {
875 retval = NULL;
876 break;
877 }
878 for (i = 0; i < n; i++) {
879 v2 = r_object(p);
880 if ( v2 == NULL ) {
881 if (!PyErr_Occurred())
882 PyErr_SetString(PyExc_TypeError,
883 "NULL object in marshal data for tuple");
884 Py_DECREF(v);
885 v = NULL;
886 break;
887 }
888 PyTuple_SET_ITEM(v, (int)i, v2);
889 }
890 retval = v;
891 break;
892
893 case TYPE_LIST:
894 n = r_long(p);
895 if (n < 0 || n > INT_MAX) {
896 PyErr_SetString(PyExc_ValueError, "bad marshal data (list size out of range)");
897 retval = NULL;
898 break;
899 }
900 v = PyList_New((int)n);
901 if (v == NULL) {
902 retval = NULL;
903 break;
904 }
905 for (i = 0; i < n; i++) {
906 v2 = r_object(p);
907 if ( v2 == NULL ) {
908 if (!PyErr_Occurred())
909 PyErr_SetString(PyExc_TypeError,
910 "NULL object in marshal data for list");
911 Py_DECREF(v);
912 v = NULL;
913 break;
914 }
915 PyList_SET_ITEM(v, (int)i, v2);
916 }
917 retval = v;
918 break;
919
920 case TYPE_DICT:
921 v = PyDict_New();
922 if (v == NULL) {
923 retval = NULL;
924 break;
925 }
926 for (;;) {
927 PyObject *key, *val;
928 key = r_object(p);
929 if (key == NULL)
930 break;
931 val = r_object(p);
932 if (val != NULL)
933 PyDict_SetItem(v, key, val);
934 Py_DECREF(key);
935 Py_XDECREF(val);
936 }
937 if (PyErr_Occurred()) {
938 Py_DECREF(v);
939 v = NULL;
940 }
941 retval = v;
942 break;
943
944 case TYPE_SET:
945 case TYPE_FROZENSET:
946 n = r_long(p);
947 if (n < 0 || n > INT_MAX) {
948 PyErr_SetString(PyExc_ValueError, "bad marshal data (set size out of range)");
949 retval = NULL;
950 break;
951 }
952 v = (type == TYPE_SET) ? PySet_New(NULL) : PyFrozenSet_New(NULL);
953 if (v == NULL) {
954 retval = NULL;
955 break;
956 }
957 for (i = 0; i < n; i++) {
958 v2 = r_object(p);
959 if ( v2 == NULL ) {
960 if (!PyErr_Occurred())
961 PyErr_SetString(PyExc_TypeError,
962 "NULL object in marshal data for set");
963 Py_DECREF(v);
964 v = NULL;
965 break;
966 }
967 if (PySet_Add(v, v2) == -1) {
968 Py_DECREF(v);
969 Py_DECREF(v2);
970 v = NULL;
971 break;
972 }
973 Py_DECREF(v2);
974 }
975 retval = v;
976 break;
977
978 case TYPE_CODE:
979 if (PyEval_GetRestricted()) {
980 PyErr_SetString(PyExc_RuntimeError,
981 "cannot unmarshal code objects in "
982 "restricted execution mode");
983 retval = NULL;
984 break;
985 }
986 else {
987 int argcount;
988 int nlocals;
989 int stacksize;
990 int flags;
991 PyObject *code = NULL;
992 PyObject *consts = NULL;
993 PyObject *names = NULL;
994 PyObject *varnames = NULL;
995 PyObject *freevars = NULL;
996 PyObject *cellvars = NULL;
997 PyObject *filename = NULL;
998 PyObject *name = NULL;
999 int firstlineno;
1000 PyObject *lnotab = NULL;
1001
1002 v = NULL;
1003
1004 /* XXX ignore long->int overflows for now */
1005 argcount = (int)r_long(p);
1006 nlocals = (int)r_long(p);
1007 stacksize = (int)r_long(p);
1008 flags = (int)r_long(p);
1009 code = r_object(p);
1010 if (code == NULL)
1011 goto code_error;
1012 consts = r_object(p);
1013 if (consts == NULL)
1014 goto code_error;
1015 names = r_object(p);
1016 if (names == NULL)
1017 goto code_error;
1018 varnames = r_object(p);
1019 if (varnames == NULL)
1020 goto code_error;
1021 freevars = r_object(p);
1022 if (freevars == NULL)
1023 goto code_error;
1024 cellvars = r_object(p);
1025 if (cellvars == NULL)
1026 goto code_error;
1027 filename = r_object(p);
1028 if (filename == NULL)
1029 goto code_error;
1030 name = r_object(p);
1031 if (name == NULL)
1032 goto code_error;
1033 firstlineno = (int)r_long(p);
1034 lnotab = r_object(p);
1035 if (lnotab == NULL)
1036 goto code_error;
1037
1038 v = (PyObject *) PyCode_New(
1039 argcount, nlocals, stacksize, flags,
1040 code, consts, names, varnames,
1041 freevars, cellvars, filename, name,
1042 firstlineno, lnotab);
1043
1044 code_error:
1045 Py_XDECREF(code);
1046 Py_XDECREF(consts);
1047 Py_XDECREF(names);
1048 Py_XDECREF(varnames);
1049 Py_XDECREF(freevars);
1050 Py_XDECREF(cellvars);
1051 Py_XDECREF(filename);
1052 Py_XDECREF(name);
1053 Py_XDECREF(lnotab);
1054
1055 }
1056 retval = v;
1057 break;
1058
1059 default:
1060 /* Bogus data got written, which isn't ideal.
1061 This will let you keep working and recover. */
1062 PyErr_SetString(PyExc_ValueError, "bad marshal data (unknown type code)");
1063 retval = NULL;
1064 break;
1065
1066 }
1067 p->depth--;
1068 return retval;
1069 }
1070
1071 static PyObject *
1072 read_object(RFILE *p)
1073 {
1074 PyObject *v;
1075 if (PyErr_Occurred()) {
1076 fprintf(stderr, "XXX readobject called with exception set\n");
1077 return NULL;
1078 }
1079 v = r_object(p);
1080 if (v == NULL && !PyErr_Occurred())
1081 PyErr_SetString(PyExc_TypeError, "NULL object in marshal data for object");
1082 return v;
1083 }
1084
1085 int
1086 PyMarshal_ReadShortFromFile(FILE *fp)
1087 {
1088 RFILE rf;
1089 assert(fp);
1090 rf.fp = fp;
1091 rf.strings = NULL;
1092 rf.end = rf.ptr = NULL;
1093 return r_short(&rf);
1094 }
1095
1096 long
1097 PyMarshal_ReadLongFromFile(FILE *fp)
1098 {
1099 RFILE rf;
1100 rf.fp = fp;
1101 rf.strings = NULL;
1102 rf.ptr = rf.end = NULL;
1103 return r_long(&rf);
1104 }
1105
1106 #ifdef HAVE_FSTAT
1107 /* Return size of file in bytes; < 0 if unknown. */
1108 static off_t
1109 getfilesize(FILE *fp)
1110 {
1111 struct stat st;
1112 if (fstat(fileno(fp), &st) != 0)
1113 return -1;
1114 else
1115 return st.st_size;
1116 }
1117 #endif
1118
1119 /* If we can get the size of the file up-front, and it's reasonably small,
1120 * read it in one gulp and delegate to ...FromString() instead. Much quicker
1121 * than reading a byte at a time from file; speeds .pyc imports.
1122 * CAUTION: since this may read the entire remainder of the file, don't
1123 * call it unless you know you're done with the file.
1124 */
1125 PyObject *
1126 PyMarshal_ReadLastObjectFromFile(FILE *fp)
1127 {
1128 /* REASONABLE_FILE_LIMIT is by defn something big enough for Tkinter.pyc. */
1129 #define REASONABLE_FILE_LIMIT (1L << 18)
1130 #ifdef HAVE_FSTAT
1131 off_t filesize;
1132 filesize = getfilesize(fp);
1133 if (filesize > 0 && filesize <= REASONABLE_FILE_LIMIT) {
1134 char* pBuf = (char *)PyMem_MALLOC(filesize);
1135 if (pBuf != NULL) {
1136 PyObject* v;
1137 size_t n;
1138 /* filesize must fit into an int, because it
1139 is smaller than REASONABLE_FILE_LIMIT */
1140 n = fread(pBuf, 1, (int)filesize, fp);
1141 v = PyMarshal_ReadObjectFromString(pBuf, n);
1142 PyMem_FREE(pBuf);
1143 return v;
1144 }
1145
1146 }
1147 #endif
1148 /* We don't have fstat, or we do but the file is larger than
1149 * REASONABLE_FILE_LIMIT or malloc failed -- read a byte at a time.
1150 */
1151 return PyMarshal_ReadObjectFromFile(fp);
1152
1153 #undef REASONABLE_FILE_LIMIT
1154 }
1155
1156 PyObject *
1157 PyMarshal_ReadObjectFromFile(FILE *fp)
1158 {
1159 RFILE rf;
1160 PyObject *result;
1161 rf.fp = fp;
1162 rf.strings = PyList_New(0);
1163 rf.depth = 0;
1164 rf.ptr = rf.end = NULL;
1165 result = r_object(&rf);
1166 Py_DECREF(rf.strings);
1167 return result;
1168 }
1169
1170 PyObject *
1171 PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
1172 {
1173 RFILE rf;
1174 PyObject *result;
1175 rf.fp = NULL;
1176 rf.ptr = str;
1177 rf.end = str + len;
1178 rf.strings = PyList_New(0);
1179 rf.depth = 0;
1180 result = r_object(&rf);
1181 Py_DECREF(rf.strings);
1182 return result;
1183 }
1184
1185 static void
1186 set_error(int error)
1187 {
1188 switch (error) {
1189 case WFERR_NOMEMORY:
1190 PyErr_NoMemory();
1191 break;
1192 case WFERR_UNMARSHALLABLE:
1193 PyErr_SetString(PyExc_ValueError, "unmarshallable object");
1194 break;
1195 case WFERR_NESTEDTOODEEP:
1196 default:
1197 PyErr_SetString(PyExc_ValueError,
1198 "object too deeply nested to marshal");
1199 break;
1200 }
1201 }
1202
1203 PyObject *
1204 PyMarshal_WriteObjectToString(PyObject *x, int version)
1205 {
1206 WFILE wf;
1207 wf.fp = NULL;
1208 wf.str = PyString_FromStringAndSize((char *)NULL, 50);
1209 if (wf.str == NULL)
1210 return NULL;
1211 wf.ptr = PyString_AS_STRING((PyStringObject *)wf.str);
1212 wf.end = wf.ptr + PyString_Size(wf.str);
1213 wf.error = WFERR_OK;
1214 wf.depth = 0;
1215 wf.version = version;
1216 wf.strings = (version > 0) ? PyDict_New() : NULL;
1217 w_object(x, &wf);
1218 Py_XDECREF(wf.strings);
1219 if (wf.str != NULL) {
1220 char *base = PyString_AS_STRING((PyStringObject *)wf.str);
1221 if (wf.ptr - base > PY_SSIZE_T_MAX) {
1222 Py_DECREF(wf.str);
1223 PyErr_SetString(PyExc_OverflowError,
1224 "too much marshall data for a string");
1225 return NULL;
1226 }
1227 if (_PyString_Resize(&wf.str, (Py_ssize_t)(wf.ptr - base)))
1228 return NULL;
1229 }
1230 if (wf.error != WFERR_OK) {
1231 Py_XDECREF(wf.str);
1232 set_error(wf.error);
1233 return NULL;
1234 }
1235 return wf.str;
1236 }
1237
1238 /* And an interface for Python programs... */
1239
1240 static PyObject *
1241 marshal_dump(PyObject *self, PyObject *args)
1242 {
1243 WFILE wf;
1244 PyObject *x;
1245 PyObject *f;
1246 int version = Py_MARSHAL_VERSION;
1247 if (!PyArg_ParseTuple(args, "OO|i:dump", &x, &f, &version))
1248 return NULL;
1249 if (!PyFile_Check(f)) {
1250 PyErr_SetString(PyExc_TypeError,
1251 "marshal.dump() 2nd arg must be file");
1252 return NULL;
1253 }
1254 wf.fp = PyFile_AsFile(f);
1255 wf.str = NULL;
1256 wf.ptr = wf.end = NULL;
1257 wf.error = WFERR_OK;
1258 wf.depth = 0;
1259 wf.strings = (version > 0) ? PyDict_New() : 0;
1260 wf.version = version;
1261 w_object(x, &wf);
1262 Py_XDECREF(wf.strings);
1263 if (wf.error != WFERR_OK) {
1264 set_error(wf.error);
1265 return NULL;
1266 }
1267 Py_INCREF(Py_None);
1268 return Py_None;
1269 }
1270
1271 PyDoc_STRVAR(dump_doc,
1272 "dump(value, file[, version])\n\
1273 \n\
1274 Write the value on the open file. The value must be a supported type.\n\
1275 The file must be an open file object such as sys.stdout or returned by\n\
1276 open() or os.popen(). It must be opened in binary mode ('wb' or 'w+b').\n\
1277 \n\
1278 If the value has (or contains an object that has) an unsupported type, a\n\
1279 ValueError exception is raised — but garbage data will also be written\n\
1280 to the file. The object will not be properly read back by load()\n\
1281 \n\
1282 New in version 2.4: The version argument indicates the data format that\n\
1283 dump should use.");
1284
1285 static PyObject *
1286 marshal_load(PyObject *self, PyObject *f)
1287 {
1288 RFILE rf;
1289 PyObject *result;
1290 if (!PyFile_Check(f)) {
1291 PyErr_SetString(PyExc_TypeError,
1292 "marshal.load() arg must be file");
1293 return NULL;
1294 }
1295 rf.fp = PyFile_AsFile(f);
1296 rf.strings = PyList_New(0);
1297 rf.depth = 0;
1298 result = read_object(&rf);
1299 Py_DECREF(rf.strings);
1300 return result;
1301 }
1302
1303 PyDoc_STRVAR(load_doc,
1304 "load(file)\n\
1305 \n\
1306 Read one value from the open file and return it. If no valid value is\n\
1307 read (e.g. because the data has a different Python version’s\n\
1308 incompatible marshal format), raise EOFError, ValueError or TypeError.\n\
1309 The file must be an open file object opened in binary mode ('rb' or\n\
1310 'r+b').\n\
1311 \n\
1312 Note: If an object containing an unsupported type was marshalled with\n\
1313 dump(), load() will substitute None for the unmarshallable type.");
1314
1315
1316 static PyObject *
1317 marshal_dumps(PyObject *self, PyObject *args)
1318 {
1319 PyObject *x;
1320 int version = Py_MARSHAL_VERSION;
1321 if (!PyArg_ParseTuple(args, "O|i:dumps", &x, &version))
1322 return NULL;
1323 return PyMarshal_WriteObjectToString(x, version);
1324 }
1325
1326 PyDoc_STRVAR(dumps_doc,
1327 "dumps(value[, version])\n\
1328 \n\
1329 Return the string that would be written to a file by dump(value, file).\n\
1330 The value must be a supported type. Raise a ValueError exception if\n\
1331 value has (or contains an object that has) an unsupported type.\n\
1332 \n\
1333 New in version 2.4: The version argument indicates the data format that\n\
1334 dumps should use.");
1335
1336
1337 static PyObject *
1338 marshal_loads(PyObject *self, PyObject *args)
1339 {
1340 RFILE rf;
1341 char *s;
1342 Py_ssize_t n;
1343 PyObject* result;
1344 if (!PyArg_ParseTuple(args, "s#:loads", &s, &n))
1345 return NULL;
1346 rf.fp = NULL;
1347 rf.ptr = s;
1348 rf.end = s + n;
1349 rf.strings = PyList_New(0);
1350 rf.depth = 0;
1351 result = read_object(&rf);
1352 Py_DECREF(rf.strings);
1353 return result;
1354 }
1355
1356 PyDoc_STRVAR(loads_doc,
1357 "loads(string)\n\
1358 \n\
1359 Convert the string to a value. If no valid value is found, raise\n\
1360 EOFError, ValueError or TypeError. Extra characters in the string are\n\
1361 ignored.");
1362
1363 static PyMethodDef marshal_methods[] = {
1364 {"dump", marshal_dump, METH_VARARGS, dump_doc},
1365 {"load", marshal_load, METH_O, load_doc},
1366 {"dumps", marshal_dumps, METH_VARARGS, dumps_doc},
1367 {"loads", marshal_loads, METH_VARARGS, loads_doc},
1368 {NULL, NULL} /* sentinel */
1369 };
1370
1371 PyDoc_STRVAR(marshal_doc,
1372 "This module contains functions that can read and write Python values in\n\
1373 a binary format. The format is specific to Python, but independent of\n\
1374 machine architecture issues.\n\
1375 \n\
1376 Not all Python object types are supported; in general, only objects\n\
1377 whose value is independent from a particular invocation of Python can be\n\
1378 written and read by this module. The following types are supported:\n\
1379 None, integers, long integers, floating point numbers, strings, Unicode\n\
1380 objects, tuples, lists, sets, dictionaries, and code objects, where it\n\
1381 should be understood that tuples, lists and dictionaries are only\n\
1382 supported as long as the values contained therein are themselves\n\
1383 supported; and recursive lists and dictionaries should not be written\n\
1384 (they will cause infinite loops).\n\
1385 \n\
1386 Variables:\n\
1387 \n\
1388 version -- indicates the format that the module uses. Version 0 is the\n\
1389 historical format, version 1 (added in Python 2.4) shares interned\n\
1390 strings and version 2 (added in Python 2.5) uses a binary format for\n\
1391 floating point numbers. (New in version 2.4)\n\
1392 \n\
1393 Functions:\n\
1394 \n\
1395 dump() -- write value to a file\n\
1396 load() -- read value from a file\n\
1397 dumps() -- write value to a string\n\
1398 loads() -- read value from a string");
1399
1400
1401 PyMODINIT_FUNC
1402 PyMarshal_Init(void)
1403 {
1404 PyObject *mod = Py_InitModule3("marshal", marshal_methods,
1405 marshal_doc);
1406 if (mod == NULL)
1407 return;
1408 PyModule_AddIntConstant(mod, "version", Py_MARSHAL_VERSION);
1409 }