From 3ad52e366fea37b02a3f619e6b7cffa7dfbdfa2e Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Sat, 6 Jun 2020 00:01:02 +0900 Subject: [PATCH 01/19] bpo-1635741: Port mmap module to multiphase initialization (GH-19459) --- ...2020-04-10-23-54-57.bpo-1635741.ZURqoN.rst | 1 + Modules/mmapmodule.c | 154 +++++++++--------- 2 files changed, 81 insertions(+), 74 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-04-10-23-54-57.bpo-1635741.ZURqoN.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-04-10-23-54-57.bpo-1635741.ZURqoN.rst b/Misc/NEWS.d/next/Core and Builtins/2020-04-10-23-54-57.bpo-1635741.ZURqoN.rst new file mode 100644 index 000000000000000..cb849fb9b44308c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-04-10-23-54-57.bpo-1635741.ZURqoN.rst @@ -0,0 +1 @@ +Port :mod:`mmap` to multiphase initialization. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 8a60db1e1c46926..463bd40e78f4faa 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -1509,157 +1509,163 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) } #endif /* MS_WINDOWS */ -static void -setint(PyObject *d, const char *name, long value) +static int +mmap_exec(PyObject *module) { - PyObject *o = PyLong_FromLong(value); - if (o) { - PyDict_SetItemString(d, name, o); - Py_DECREF(o); + if (PyType_Ready(&mmap_object_type) < 0) { + return -1; } -} - -static struct PyModuleDef mmapmodule = { - PyModuleDef_HEAD_INIT, - "mmap", - NULL, - -1, - NULL, - NULL, - NULL, - NULL, - NULL -}; - -PyMODINIT_FUNC -PyInit_mmap(void) -{ - PyObject *dict, *module; + Py_INCREF(PyExc_OSError); + if (PyModule_AddObject(module, "error", PyExc_OSError) < 0) { + Py_DECREF(PyExc_OSError); + return -1; + } + if (PyModule_AddType(module, &mmap_object_type) < 0) { + return -1; + } - if (PyType_Ready(&mmap_object_type) < 0) - return NULL; +#define ADD_INT_MACRO(module, constant) \ + do { \ + if (PyModule_AddIntConstant(module, #constant, constant) < 0) { \ + return -1; \ + } \ + } while (0) - module = PyModule_Create(&mmapmodule); - if (module == NULL) - return NULL; - dict = PyModule_GetDict(module); - if (!dict) - return NULL; - PyDict_SetItemString(dict, "error", PyExc_OSError); - PyDict_SetItemString(dict, "mmap", (PyObject*) &mmap_object_type); #ifdef PROT_EXEC - setint(dict, "PROT_EXEC", PROT_EXEC); + ADD_INT_MACRO(module, PROT_EXEC); #endif #ifdef PROT_READ - setint(dict, "PROT_READ", PROT_READ); + ADD_INT_MACRO(module, PROT_READ); #endif #ifdef PROT_WRITE - setint(dict, "PROT_WRITE", PROT_WRITE); + ADD_INT_MACRO(module, PROT_WRITE); #endif #ifdef MAP_SHARED - setint(dict, "MAP_SHARED", MAP_SHARED); + ADD_INT_MACRO(module, MAP_SHARED); #endif #ifdef MAP_PRIVATE - setint(dict, "MAP_PRIVATE", MAP_PRIVATE); + ADD_INT_MACRO(module, MAP_PRIVATE); #endif #ifdef MAP_DENYWRITE - setint(dict, "MAP_DENYWRITE", MAP_DENYWRITE); + ADD_INT_MACRO(module, MAP_DENYWRITE); #endif #ifdef MAP_EXECUTABLE - setint(dict, "MAP_EXECUTABLE", MAP_EXECUTABLE); + ADD_INT_MACRO(module, MAP_EXECUTABLE); #endif #ifdef MAP_ANONYMOUS - setint(dict, "MAP_ANON", MAP_ANONYMOUS); - setint(dict, "MAP_ANONYMOUS", MAP_ANONYMOUS); + if (PyModule_AddIntConstant(module, "MAP_ANON", MAP_ANONYMOUS) < 0 ) { + return -1; + } + ADD_INT_MACRO(module, MAP_ANONYMOUS); #endif #ifdef MAP_POPULATE - setint(dict, "MAP_POPULATE", MAP_POPULATE); + ADD_INT_MACRO(module, MAP_POPULATE); #endif + if (PyModule_AddIntConstant(module, "PAGESIZE", (long)my_getpagesize()) < 0 ) { + return -1; + } - setint(dict, "PAGESIZE", (long)my_getpagesize()); - - setint(dict, "ALLOCATIONGRANULARITY", (long)my_getallocationgranularity()); + if (PyModule_AddIntConstant(module, "ALLOCATIONGRANULARITY", (long)my_getallocationgranularity()) < 0 ) { + return -1; + } - setint(dict, "ACCESS_DEFAULT", ACCESS_DEFAULT); - setint(dict, "ACCESS_READ", ACCESS_READ); - setint(dict, "ACCESS_WRITE", ACCESS_WRITE); - setint(dict, "ACCESS_COPY", ACCESS_COPY); + ADD_INT_MACRO(module, ACCESS_DEFAULT); + ADD_INT_MACRO(module, ACCESS_READ); + ADD_INT_MACRO(module, ACCESS_WRITE); + ADD_INT_MACRO(module, ACCESS_COPY); #ifdef HAVE_MADVISE // Conventional advice values #ifdef MADV_NORMAL - setint(dict, "MADV_NORMAL", MADV_NORMAL); + ADD_INT_MACRO(module, MADV_NORMAL); #endif #ifdef MADV_RANDOM - setint(dict, "MADV_RANDOM", MADV_RANDOM); + ADD_INT_MACRO(module, MADV_RANDOM); #endif #ifdef MADV_SEQUENTIAL - setint(dict, "MADV_SEQUENTIAL", MADV_SEQUENTIAL); + ADD_INT_MACRO(module, MADV_SEQUENTIAL); #endif #ifdef MADV_WILLNEED - setint(dict, "MADV_WILLNEED", MADV_WILLNEED); + ADD_INT_MACRO(module, MADV_WILLNEED); #endif #ifdef MADV_DONTNEED - setint(dict, "MADV_DONTNEED", MADV_DONTNEED); + ADD_INT_MACRO(module, MADV_DONTNEED); #endif // Linux-specific advice values #ifdef MADV_REMOVE - setint(dict, "MADV_REMOVE", MADV_REMOVE); + ADD_INT_MACRO(module, MADV_REMOVE); #endif #ifdef MADV_DONTFORK - setint(dict, "MADV_DONTFORK", MADV_DONTFORK); + ADD_INT_MACRO(module, MADV_DONTFORK); #endif #ifdef MADV_DOFORK - setint(dict, "MADV_DOFORK", MADV_DOFORK); + ADD_INT_MACRO(module, MADV_DOFORK); #endif #ifdef MADV_HWPOISON - setint(dict, "MADV_HWPOISON", MADV_HWPOISON); + ADD_INT_MACRO(module, MADV_HWPOISON); #endif #ifdef MADV_MERGEABLE - setint(dict, "MADV_MERGEABLE", MADV_MERGEABLE); + ADD_INT_MACRO(module, MADV_MERGEABLE); #endif #ifdef MADV_UNMERGEABLE - setint(dict, "MADV_UNMERGEABLE", MADV_UNMERGEABLE); + ADD_INT_MACRO(module, MADV_UNMERGEABLE); #endif #ifdef MADV_SOFT_OFFLINE - setint(dict, "MADV_SOFT_OFFLINE", MADV_SOFT_OFFLINE); + ADD_INT_MACRO(module, MADV_SOFT_OFFLINE); #endif #ifdef MADV_HUGEPAGE - setint(dict, "MADV_HUGEPAGE", MADV_HUGEPAGE); + ADD_INT_MACRO(module, MADV_HUGEPAGE); #endif #ifdef MADV_NOHUGEPAGE - setint(dict, "MADV_NOHUGEPAGE", MADV_NOHUGEPAGE); + ADD_INT_MACRO(module, MADV_NOHUGEPAGE); #endif #ifdef MADV_DONTDUMP - setint(dict, "MADV_DONTDUMP", MADV_DONTDUMP); + ADD_INT_MACRO(module, MADV_DONTDUMP); #endif #ifdef MADV_DODUMP - setint(dict, "MADV_DODUMP", MADV_DODUMP); + ADD_INT_MACRO(module, MADV_DODUMP); #endif #ifdef MADV_FREE // (Also present on FreeBSD and macOS.) - setint(dict, "MADV_FREE", MADV_FREE); + ADD_INT_MACRO(module, MADV_FREE); #endif // FreeBSD-specific #ifdef MADV_NOSYNC - setint(dict, "MADV_NOSYNC", MADV_NOSYNC); + ADD_INT_MACRO(module, MADV_NOSYNC); #endif #ifdef MADV_AUTOSYNC - setint(dict, "MADV_AUTOSYNC", MADV_AUTOSYNC); + ADD_INT_MACRO(module, MADV_AUTOSYNC); #endif #ifdef MADV_NOCORE - setint(dict, "MADV_NOCORE", MADV_NOCORE); + ADD_INT_MACRO(module, MADV_NOCORE); #endif #ifdef MADV_CORE - setint(dict, "MADV_CORE", MADV_CORE); + ADD_INT_MACRO(module, MADV_CORE); #endif #ifdef MADV_PROTECT - setint(dict, "MADV_PROTECT", MADV_PROTECT); + ADD_INT_MACRO(module, MADV_PROTECT); #endif #endif // HAVE_MADVISE + return 0; +} - return module; +static PyModuleDef_Slot mmap_slots[] = { + {Py_mod_exec, mmap_exec}, + {0, NULL} +}; + +static struct PyModuleDef mmapmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "mmap", + .m_size = 0, + .m_slots = mmap_slots, +}; + +PyMODINIT_FUNC +PyInit_mmap(void) +{ + return PyModuleDef_Init(&mmapmodule); } From 087d612efebe7c64e5f079b07e0454111859830e Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Fri, 5 Jun 2020 19:43:01 +0200 Subject: [PATCH 02/19] bpo-40874: Update to libmpdec-2.5.0 (GH-20652) --- Modules/_decimal/libmpdec/basearith.c | 14 +- Modules/_decimal/libmpdec/basearith.h | 12 +- Modules/_decimal/libmpdec/bits.h | 12 +- Modules/_decimal/libmpdec/constants.c | 5 +- Modules/_decimal/libmpdec/constants.h | 13 +- Modules/_decimal/libmpdec/context.c | 10 +- Modules/_decimal/libmpdec/convolute.c | 7 +- Modules/_decimal/libmpdec/convolute.h | 9 +- Modules/_decimal/libmpdec/crt.c | 11 +- Modules/_decimal/libmpdec/crt.h | 9 +- Modules/_decimal/libmpdec/difradix2.c | 10 +- Modules/_decimal/libmpdec/difradix2.h | 9 +- Modules/_decimal/libmpdec/fnt.c | 12 +- Modules/_decimal/libmpdec/fnt.h | 10 +- Modules/_decimal/libmpdec/fourstep.c | 12 +- Modules/_decimal/libmpdec/fourstep.h | 9 +- Modules/_decimal/libmpdec/io.c | 43 ++- Modules/_decimal/libmpdec/io.h | 13 +- Modules/_decimal/libmpdec/literature/fnt.py | 2 +- .../libmpdec/literature/matrix-transform.txt | 2 +- .../libmpdec/literature/mulmod-64.txt | 2 +- .../libmpdec/literature/mulmod-ppro.txt | 2 +- .../_decimal/libmpdec/literature/six-step.txt | 2 +- .../libmpdec/literature/umodarith.lisp | 16 +- Modules/_decimal/libmpdec/mpalloc.c | 63 ++++- Modules/_decimal/libmpdec/mpalloc.h | 16 +- Modules/_decimal/libmpdec/mpdecimal.c | 264 ++++++++++++++---- Modules/_decimal/libmpdec/mpdecimal.h | 97 +++---- Modules/_decimal/libmpdec/numbertheory.c | 10 +- Modules/_decimal/libmpdec/numbertheory.h | 12 +- Modules/_decimal/libmpdec/sixstep.c | 12 +- Modules/_decimal/libmpdec/sixstep.h | 9 +- Modules/_decimal/libmpdec/transpose.c | 12 +- Modules/_decimal/libmpdec/transpose.h | 9 +- Modules/_decimal/libmpdec/typearith.h | 13 +- Modules/_decimal/libmpdec/umodarith.h | 14 +- Modules/_decimal/libmpdec/vccompat.h | 17 +- Modules/_decimal/libmpdec/vcdiv64.asm | 4 +- 38 files changed, 502 insertions(+), 296 deletions(-) diff --git a/Modules/_decimal/libmpdec/basearith.c b/Modules/_decimal/libmpdec/basearith.c index dfe1523927a4078..85c608fadf51567 100644 --- a/Modules/_decimal/libmpdec/basearith.c +++ b/Modules/_decimal/libmpdec/basearith.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,13 +27,13 @@ #include "mpdecimal.h" -#include -#include -#include + #include +#include + +#include "basearith.h" #include "constants.h" #include "typearith.h" -#include "basearith.h" /*********************************************************************/ @@ -337,6 +337,7 @@ _mpd_basedivmod(mpd_uint_t *q, mpd_uint_t *r, /* D2: loop */ for (j=m; j != MPD_SIZE_MAX; j--) { + assert(2 <= j+n && j+n <= nplusm); /* annotation for scan-build */ /* D3: calculate qhat and rhat */ rhat = _mpd_shortdiv(w2, u+j+n-1, 2, v[n-1]); @@ -652,6 +653,3 @@ _mpd_shortdiv_b(mpd_uint_t *w, const mpd_uint_t *u, mpd_size_t n, return rem; } - - - diff --git a/Modules/_decimal/libmpdec/basearith.h b/Modules/_decimal/libmpdec/basearith.h index 976358a110ecf3e..d35925aaddb48e0 100644 --- a/Modules/_decimal/libmpdec/basearith.h +++ b/Modules/_decimal/libmpdec/basearith.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,11 @@ */ -#ifndef BASEARITH_H -#define BASEARITH_H +#ifndef LIBMPDEC_BASEARITH_H_ +#define LIBMPDEC_BASEARITH_H_ #include "mpdecimal.h" -#include #include "typearith.h" @@ -216,7 +215,4 @@ _mpd_isallnine(const mpd_uint_t *data, mpd_ssize_t len) MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ -#endif /* BASEARITH_H */ - - - +#endif /* LIBMPDEC_BASEARITH_H_ */ diff --git a/Modules/_decimal/libmpdec/bits.h b/Modules/_decimal/libmpdec/bits.h index b5eaa24976ae51b..aa9c3e77980c031 100644 --- a/Modules/_decimal/libmpdec/bits.h +++ b/Modules/_decimal/libmpdec/bits.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,11 @@ */ -#ifndef BITS_H -#define BITS_H +#ifndef LIBMPDEC_BITS_H_ +#define LIBMPDEC_BITS_H_ #include "mpdecimal.h" -#include /* Check if n is a power of 2. */ @@ -186,7 +185,4 @@ mpd_bsf(mpd_size_t a) #endif /* BSR/BSF */ -#endif /* BITS_H */ - - - +#endif /* LIBMPDEC_BITS_H_ */ diff --git a/Modules/_decimal/libmpdec/constants.c b/Modules/_decimal/libmpdec/constants.c index 2c2d5ea48103539..4c4de622bc6018d 100644 --- a/Modules/_decimal/libmpdec/constants.c +++ b/Modules/_decimal/libmpdec/constants.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,7 +27,6 @@ #include "mpdecimal.h" -#include #include "constants.h" @@ -128,5 +127,3 @@ const char *mpd_clamp_string[MPD_CLAMP_GUARD] = { "CLAMP_DEFAULT", "CLAMP_IEEE_754" }; - - diff --git a/Modules/_decimal/libmpdec/constants.h b/Modules/_decimal/libmpdec/constants.h index c0febfc8772d7e2..7c1db839c20ba28 100644 --- a/Modules/_decimal/libmpdec/constants.h +++ b/Modules/_decimal/libmpdec/constants.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,14 @@ */ -#ifndef CONSTANTS_H -#define CONSTANTS_H +#ifndef LIBMPDEC_CONSTANTS_H_ +#define LIBMPDEC_CONSTANTS_H_ #include "mpdecimal.h" +#include + /* Internal header file: all symbols have local scope in the DSO */ MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) @@ -84,7 +86,4 @@ extern const mpd_uint_t UH_P1P2; MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ -#endif /* CONSTANTS_H */ - - - +#endif /* LIBMPDEC_CONSTANTS_H_ */ diff --git a/Modules/_decimal/libmpdec/context.c b/Modules/_decimal/libmpdec/context.c index 24c7b890c1d9895..9cbc20509595d4d 100644 --- a/Modules/_decimal/libmpdec/context.c +++ b/Modules/_decimal/libmpdec/context.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,14 +27,16 @@ #include "mpdecimal.h" + +#include #include #include -#include void -mpd_dflt_traphandler(mpd_context_t *ctx UNUSED) +mpd_dflt_traphandler(mpd_context_t *ctx) { + (void)ctx; raise(SIGFPE); } @@ -282,5 +284,3 @@ mpd_addstatus_raise(mpd_context_t *ctx, uint32_t flags) mpd_traphandler(ctx); } } - - diff --git a/Modules/_decimal/libmpdec/convolute.c b/Modules/_decimal/libmpdec/convolute.c index 4c62e8bd3abd808..4bc8e8b5fd32f48 100644 --- a/Modules/_decimal/libmpdec/convolute.c +++ b/Modules/_decimal/libmpdec/convolute.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,15 +27,14 @@ #include "mpdecimal.h" -#include #include "bits.h" #include "constants.h" +#include "convolute.h" #include "fnt.h" #include "fourstep.h" #include "numbertheory.h" #include "sixstep.h" #include "umodarith.h" -#include "convolute.h" /* Bignum: Fast convolution using the Number Theoretic Transform. Used for @@ -170,5 +169,3 @@ fnt_autoconvolute(mpd_uint_t *c1, mpd_size_t n, int modnum) return 1; } - - diff --git a/Modules/_decimal/libmpdec/convolute.h b/Modules/_decimal/libmpdec/convolute.h index f30a177a6840677..62edb3e45739cb0 100644 --- a/Modules/_decimal/libmpdec/convolute.h +++ b/Modules/_decimal/libmpdec/convolute.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,11 @@ */ -#ifndef CONVOLUTE_H -#define CONVOLUTE_H +#ifndef LIBMPDEC_CONVOLUTE_H_ +#define LIBMPDEC_CONVOLUTE_H_ #include "mpdecimal.h" -#include /* Internal header file: all symbols have local scope in the DSO */ @@ -47,4 +46,4 @@ int fnt_autoconvolute(mpd_uint_t *c1, mpd_size_t n, int modnum); MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ -#endif +#endif /* LIBMPDEC_CONVOLUTE_H_ */ diff --git a/Modules/_decimal/libmpdec/crt.c b/Modules/_decimal/libmpdec/crt.c index 4a1e80a232284fb..613274ee0c5b507 100644 --- a/Modules/_decimal/libmpdec/crt.c +++ b/Modules/_decimal/libmpdec/crt.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,11 +27,14 @@ #include "mpdecimal.h" -#include + #include + +#include "constants.h" +#include "crt.h" #include "numbertheory.h" #include "umodarith.h" -#include "crt.h" +#include "typearith.h" /* Bignum: Chinese Remainder Theorem, extends the maximum transform length. */ @@ -175,5 +178,3 @@ crt3(mpd_uint_t *x1, mpd_uint_t *x2, mpd_uint_t *x3, mpd_size_t rsize) assert(carry[0] == 0 && carry[1] == 0 && carry[2] == 0); } - - diff --git a/Modules/_decimal/libmpdec/crt.h b/Modules/_decimal/libmpdec/crt.h index f61e77293632e16..15a347d4cb31e58 100644 --- a/Modules/_decimal/libmpdec/crt.h +++ b/Modules/_decimal/libmpdec/crt.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,11 @@ */ -#ifndef CRT_H -#define CRT_H +#ifndef LIBMPDEC_CRT_H_ +#define LIBMPDEC_CRT_H_ #include "mpdecimal.h" -#include /* Internal header file: all symbols have local scope in the DSO */ @@ -44,4 +43,4 @@ void crt3(mpd_uint_t *x1, mpd_uint_t *x2, mpd_uint_t *x3, mpd_size_t nmemb); MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ -#endif +#endif /* LIBMPDEC_CRT_H_ */ diff --git a/Modules/_decimal/libmpdec/difradix2.c b/Modules/_decimal/libmpdec/difradix2.c index 06e5ab5e222ee09..049ecff65b6eefc 100644 --- a/Modules/_decimal/libmpdec/difradix2.c +++ b/Modules/_decimal/libmpdec/difradix2.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,12 +27,14 @@ #include "mpdecimal.h" -#include + #include + #include "bits.h" +#include "constants.h" +#include "difradix2.h" #include "numbertheory.h" #include "umodarith.h" -#include "difradix2.h" /* Bignum: The actual transform routine (decimation in frequency). */ @@ -169,5 +171,3 @@ fnt_dif2(mpd_uint_t a[], mpd_size_t n, struct fnt_params *tparams) bitreverse_permute(a, n); } - - diff --git a/Modules/_decimal/libmpdec/difradix2.h b/Modules/_decimal/libmpdec/difradix2.h index 5e22bcf324fac78..cdcbcf9a71043ce 100644 --- a/Modules/_decimal/libmpdec/difradix2.h +++ b/Modules/_decimal/libmpdec/difradix2.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,11 @@ */ -#ifndef DIF_RADIX2_H -#define DIF_RADIX2_H +#ifndef LIBMPDEC_DIFRADIX2_H_ +#define LIBMPDEC_DIFRADIX2_H_ #include "mpdecimal.h" -#include #include "numbertheory.h" @@ -45,4 +44,4 @@ void fnt_dif2(mpd_uint_t a[], mpd_size_t n, struct fnt_params *tparams); MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ -#endif +#endif /* LIBMPDEC_DIFRADIX2_H_ */ diff --git a/Modules/_decimal/libmpdec/fnt.c b/Modules/_decimal/libmpdec/fnt.c index 7e924c85242b08d..0dbe98fc71c9ea6 100644 --- a/Modules/_decimal/libmpdec/fnt.c +++ b/Modules/_decimal/libmpdec/fnt.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,13 +27,14 @@ #include "mpdecimal.h" -#include -#include + #include +#include + #include "bits.h" #include "difradix2.h" -#include "numbertheory.h" #include "fnt.h" +#include "numbertheory.h" /* Bignum: Fast transform for medium-sized coefficients. */ @@ -76,6 +77,3 @@ std_inv_fnt(mpd_uint_t *a, mpd_size_t n, int modnum) mpd_free(tparams); return 1; } - - - diff --git a/Modules/_decimal/libmpdec/fnt.h b/Modules/_decimal/libmpdec/fnt.h index fa2154a798d453b..5222c476a3a4f54 100644 --- a/Modules/_decimal/libmpdec/fnt.h +++ b/Modules/_decimal/libmpdec/fnt.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,11 @@ */ -#ifndef FNT_H -#define FNT_H +#ifndef LIBMPDEC_FNT_H_ +#define LIBMPDEC_FNT_H_ #include "mpdecimal.h" -#include /* Internal header file: all symbols have local scope in the DSO */ @@ -45,5 +44,4 @@ int std_inv_fnt(mpd_uint_t a[], mpd_size_t n, int modnum); MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ -#endif - +#endif /* LIBMPDEC_FNT_H_ */ diff --git a/Modules/_decimal/libmpdec/fourstep.c b/Modules/_decimal/libmpdec/fourstep.c index 21d3e7485df4dab..fb173ed5a52e463 100644 --- a/Modules/_decimal/libmpdec/fourstep.c +++ b/Modules/_decimal/libmpdec/fourstep.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,12 +27,14 @@ #include "mpdecimal.h" + #include + +#include "constants.h" +#include "fourstep.h" #include "numbertheory.h" #include "sixstep.h" -#include "transpose.h" #include "umodarith.h" -#include "fourstep.h" /* Bignum: Cache efficient Matrix Fourier Transform for arrays of the @@ -187,6 +189,7 @@ four_step_fnt(mpd_uint_t *a, mpd_size_t n, int modnum) #if 0 /* An unordered transform is sufficient for convolution. */ /* Transpose the matrix. */ + #include "transpose.h" transpose_3xpow2(a, R, C); #endif @@ -217,6 +220,7 @@ inv_four_step_fnt(mpd_uint_t *a, mpd_size_t n, int modnum) #if 0 /* An unordered transform is sufficient for convolution. */ /* Transpose the matrix, producing an R*C matrix. */ + #include "transpose.h" transpose_3xpow2(a, C, R); #endif @@ -253,5 +257,3 @@ inv_four_step_fnt(mpd_uint_t *a, mpd_size_t n, int modnum) return 1; } - - diff --git a/Modules/_decimal/libmpdec/fourstep.h b/Modules/_decimal/libmpdec/fourstep.h index 80dcd4be3d59b97..5ffb6fcc8ecd0d5 100644 --- a/Modules/_decimal/libmpdec/fourstep.h +++ b/Modules/_decimal/libmpdec/fourstep.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,11 @@ */ -#ifndef FOUR_STEP_H -#define FOUR_STEP_H +#ifndef LIBMPDEC_FOURSTEP_H_ +#define LIBMPDEC_FOURSTEP_H_ #include "mpdecimal.h" -#include /* Internal header file: all symbols have local scope in the DSO */ @@ -45,4 +44,4 @@ int inv_four_step_fnt(mpd_uint_t *a, mpd_size_t n, int modnum); MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ -#endif +#endif /* LIBMPDEC_FOURSTEP_H_ */ diff --git a/Modules/_decimal/libmpdec/io.c b/Modules/_decimal/libmpdec/io.c index f45e558f1a95734..9513a68e3782d12 100644 --- a/Modules/_decimal/libmpdec/io.c +++ b/Modules/_decimal/libmpdec/io.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,16 +27,16 @@ #include "mpdecimal.h" -#include -#include -#include -#include -#include + #include +#include #include +#include #include -#include "bits.h" -#include "constants.h" +#include +#include +#include + #include "typearith.h" #include "io.h" @@ -277,7 +277,7 @@ mpd_qset_string(mpd_t *dec, const char *s, const mpd_context_t *ctx, } } - digits = end - coeff; + digits = end - coeff; if (dpoint) { size_t fracdigits = end-dpoint-1; if (dpoint > coeff) digits--; @@ -326,6 +326,22 @@ mpd_qset_string(mpd_t *dec, const char *s, const mpd_context_t *ctx, mpd_seterror(dec, MPD_Conversion_syntax, status); } +/* convert a character string to a decimal, use a maxcontext for conversion */ +void +mpd_qset_string_exact(mpd_t *dec, const char *s, uint32_t *status) +{ + mpd_context_t maxcontext; + + mpd_maxcontext(&maxcontext); + mpd_qset_string(dec, s, &maxcontext, status); + + if (*status & (MPD_Inexact|MPD_Rounded|MPD_Clamped)) { + /* we want exact results */ + mpd_seterror(dec, MPD_Invalid_operation, status); + } + *status &= MPD_Errors; +} + /* Print word x with n decimal digits to string s. dot is either NULL or the location of a decimal point. */ #define EXTRACT_DIGIT(s, x, d, dot) \ @@ -539,8 +555,8 @@ _mpd_to_string(char **result, const mpd_t *dec, int flags, mpd_ssize_t dplace) dplace = -1 + mod_mpd_ssize_t(dec->exp+2, 3); } else { /* ldigits-1 is the adjusted exponent, which - * should be divisible by three. If not, move - * dplace one or two places to the right. */ + * should be divisible by three. If not, move + * dplace one or two places to the right. */ dplace += mod_mpd_ssize_t(ldigits-1, 3); } } @@ -1247,7 +1263,7 @@ mpd_qformat_spec(const mpd_t *dec, const mpd_spec_t *spec, } if (isupper((uchar)type)) { - type = tolower((uchar)type); + type = (char)tolower((uchar)type); flags |= MPD_FMT_UPPER; } if (spec->sign == ' ') { @@ -1265,6 +1281,7 @@ mpd_qformat_spec(const mpd_t *dec, const mpd_spec_t *spec, stackspec.align = '>'; spec = &stackspec; } + assert(strlen(spec->fill) == 1); /* annotation for scan-build */ if (type == '%') { flags |= MPD_FMT_PERCENT; } @@ -1579,5 +1596,3 @@ mpd_print(const mpd_t *dec) fputs("mpd_fprint: output error\n", stderr); /* GCOV_NOT_REACHED */ } } - - diff --git a/Modules/_decimal/libmpdec/io.h b/Modules/_decimal/libmpdec/io.h index de5486a00ca56f1..79d7c05ce369c66 100644 --- a/Modules/_decimal/libmpdec/io.h +++ b/Modules/_decimal/libmpdec/io.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,17 +26,20 @@ */ -#ifndef IO_H -#define IO_H +#ifndef LIBMPDEC_IO_H_ +#define LIBMPDEC_IO_H_ -#include #include "mpdecimal.h" +#include + #if SIZE_MAX == MPD_SIZE_MAX #define mpd_strtossize _mpd_strtossize #else +#include + static inline mpd_ssize_t mpd_strtossize(const char *s, char **end, int base) { @@ -56,4 +59,4 @@ mpd_strtossize(const char *s, char **end, int base) #endif -#endif +#endif /* LIBMPDEC_IO_H_ */ diff --git a/Modules/_decimal/libmpdec/literature/fnt.py b/Modules/_decimal/libmpdec/literature/fnt.py index 6363536da648737..c1285a565db96bd 100644 --- a/Modules/_decimal/libmpdec/literature/fnt.py +++ b/Modules/_decimal/libmpdec/literature/fnt.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008-2016 Stefan Krah. All rights reserved. +# Copyright (c) 2008-2020 Stefan Krah. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions diff --git a/Modules/_decimal/libmpdec/literature/matrix-transform.txt b/Modules/_decimal/libmpdec/literature/matrix-transform.txt index 701d85d6b43c814..6e7ad7420909fd2 100644 --- a/Modules/_decimal/libmpdec/literature/matrix-transform.txt +++ b/Modules/_decimal/libmpdec/literature/matrix-transform.txt @@ -1,6 +1,6 @@ -(* Copyright (c) 2011 Stefan Krah. All rights reserved. *) +(* Copyright (c) 2011-2020 Stefan Krah. All rights reserved. *) The Matrix Fourier Transform: diff --git a/Modules/_decimal/libmpdec/literature/mulmod-64.txt b/Modules/_decimal/libmpdec/literature/mulmod-64.txt index 029b8de3d7c925b..fa967bf95e3030b 100644 --- a/Modules/_decimal/libmpdec/literature/mulmod-64.txt +++ b/Modules/_decimal/libmpdec/literature/mulmod-64.txt @@ -1,6 +1,6 @@ -(* Copyright (c) 2011 Stefan Krah. All rights reserved. *) +(* Copyright (c) 2011-2020 Stefan Krah. All rights reserved. *) ========================================================================== diff --git a/Modules/_decimal/libmpdec/literature/mulmod-ppro.txt b/Modules/_decimal/libmpdec/literature/mulmod-ppro.txt index 4d17a928e6eae54..ba804e4b4e78641 100644 --- a/Modules/_decimal/libmpdec/literature/mulmod-ppro.txt +++ b/Modules/_decimal/libmpdec/literature/mulmod-ppro.txt @@ -1,6 +1,6 @@ -(* Copyright (c) 2011 Stefan Krah. All rights reserved. *) +(* Copyright (c) 2011-2020 Stefan Krah. All rights reserved. *) ======================================================================== diff --git a/Modules/_decimal/libmpdec/literature/six-step.txt b/Modules/_decimal/libmpdec/literature/six-step.txt index 8e45f48758478ae..852d5b0df8bf37b 100644 --- a/Modules/_decimal/libmpdec/literature/six-step.txt +++ b/Modules/_decimal/libmpdec/literature/six-step.txt @@ -1,6 +1,6 @@ -(* Copyright (c) 2011 Stefan Krah. All rights reserved. *) +(* Copyright (c) 2011-2020 Stefan Krah. All rights reserved. *) The Six Step Transform: diff --git a/Modules/_decimal/libmpdec/literature/umodarith.lisp b/Modules/_decimal/libmpdec/literature/umodarith.lisp index 99d71c373d1abd6..d71f074a26dccaa 100644 --- a/Modules/_decimal/libmpdec/literature/umodarith.lisp +++ b/Modules/_decimal/libmpdec/literature/umodarith.lisp @@ -1,5 +1,5 @@ ; -; Copyright (c) 2008-2016 Stefan Krah. All rights reserved. +; Copyright (c) 2008-2020 Stefan Krah. All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -149,7 +149,7 @@ (defthmd addmod-correct (implies (and (< 0 m) (< m base) - (< a m) (<= b m) + (< a m) (<= b m) (natp m) (natp base) (natp a) (natp b)) (equal (addmod a b m base) @@ -179,7 +179,7 @@ (defthmd submod-correct (implies (and (< 0 m) (< m base) - (< a m) (<= b m) + (< a m) (<= b m) (natp m) (natp base) (natp a) (natp b)) (equal (submod a b m base) @@ -200,7 +200,7 @@ (defthm submod-2-correct (implies (and (< 0 m) (< m base) - (< a m) (<= b m) + (< a m) (<= b m) (natp m) (natp base) (natp a) (natp b)) (equal (submod-2 a b m base) @@ -231,7 +231,7 @@ (defthmd ext-submod-ext-submod-2-equal (implies (and (< 0 m) (< m base) - (< a (* 2 m)) (< b (* 2 m)) + (< a (* 2 m)) (< b (* 2 m)) (natp m) (natp base) (natp a) (natp b)) (equal (ext-submod a b m base) @@ -239,7 +239,7 @@ (defthmd ext-submod-2-correct (implies (and (< 0 m) (< m base) - (< a (* 2 m)) (< b (* 2 m)) + (< a (* 2 m)) (< b (* 2 m)) (natp m) (natp base) (natp a) (natp b)) (equal (ext-submod-2 a b m base) @@ -257,7 +257,7 @@ (defthmd dw-reduce-correct (implies (and (< 0 m) (< m base) - (< hi base) (< lo base) + (< hi base) (< lo base) (natp m) (natp base) (natp hi) (natp lo)) (equal (dw-reduce hi lo m base) @@ -322,7 +322,7 @@ (defthmd dw-submod-correct (implies (and (< 0 m) (< m base) (natp a) (< a m) - (< hi base) (< lo base) + (< hi base) (< lo base) (natp m) (natp base) (natp hi) (natp lo)) (equal (dw-submod a hi lo m base) diff --git a/Modules/_decimal/libmpdec/mpalloc.c b/Modules/_decimal/libmpdec/mpalloc.c index a854e09911bd332..eb5ee7a807b3338 100644 --- a/Modules/_decimal/libmpdec/mpalloc.c +++ b/Modules/_decimal/libmpdec/mpalloc.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,10 +27,14 @@ #include "mpdecimal.h" + +#include #include #include -#include "typearith.h" +#include + #include "mpalloc.h" +#include "typearith.h" #if defined(_MSC_VER) @@ -294,4 +298,59 @@ mpd_realloc_dyn(mpd_t *result, mpd_ssize_t nwords, uint32_t *status) return 1; } +/* + * Input: 'result' is a static mpd_t with a static coefficient. + * Assumption: 'nwords' >= result->alloc. + * + * Resize the static coefficient to a larger dynamic one and copy the + * existing data. + * + * On failure the value of 'result' is unchanged. + */ +int +mpd_switch_to_dyn_cxx(mpd_t *result, mpd_ssize_t nwords) +{ + assert(nwords >= result->alloc); + + mpd_uint_t *data = mpd_alloc(nwords, sizeof *result->data); + if (data == NULL) { + return 0; + } + + memcpy(data, result->data, result->alloc * (sizeof *result->data)); + result->data = data; + result->alloc = nwords; + mpd_set_dynamic_data(result); + return 1; +} +/* + * Input: 'result' is a static or a dynamic mpd_t with a dynamic coefficient. + * Resize the coefficient to length 'nwords': + * Case nwords > result->alloc: + * If realloc is successful: + * 'result' has a larger coefficient but the same value. Return 1. + * Otherwise: + * 'result' has a the same coefficient. Return 0. + * Case nwords < result->alloc: + * If realloc is successful: + * 'result' has a smaller coefficient. result->len is undefined. Return 1. + * Otherwise (unlikely): + * 'result' is unchanged. Reuse the now oversized coefficient. Return 1. + */ +int +mpd_realloc_dyn_cxx(mpd_t *result, mpd_ssize_t nwords) +{ + uint8_t err = 0; + + mpd_uint_t *p = mpd_realloc(result->data, nwords, sizeof *result->data, &err); + if (!err) { + result->data = p; + result->alloc = nwords; + } + else if (nwords > result->alloc) { + return 0; + } + + return 1; +} diff --git a/Modules/_decimal/libmpdec/mpalloc.h b/Modules/_decimal/libmpdec/mpalloc.h index efd711953a39827..186808457b25c01 100644 --- a/Modules/_decimal/libmpdec/mpalloc.h +++ b/Modules/_decimal/libmpdec/mpalloc.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,14 @@ */ -#ifndef MPALLOC_H -#define MPALLOC_H +#ifndef LIBMPDEC_MPALLOC_H_ +#define LIBMPDEC_MPALLOC_H_ #include "mpdecimal.h" +#include + /* Internal header file: all symbols have local scope in the DSO */ MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) @@ -41,11 +43,11 @@ int mpd_switch_to_dyn(mpd_t *result, mpd_ssize_t size, uint32_t *status); int mpd_switch_to_dyn_zero(mpd_t *result, mpd_ssize_t size, uint32_t *status); int mpd_realloc_dyn(mpd_t *result, mpd_ssize_t size, uint32_t *status); - -MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ +int mpd_switch_to_dyn_cxx(mpd_t *result, mpd_ssize_t size); +int mpd_realloc_dyn_cxx(mpd_t *result, mpd_ssize_t size); -#endif - +MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ +#endif /* LIBMPDEC_MPALLOC_H_ */ diff --git a/Modules/_decimal/libmpdec/mpdecimal.c b/Modules/_decimal/libmpdec/mpdecimal.c index 0986edb576a10ff..ad8db508b36f0a5 100644 --- a/Modules/_decimal/libmpdec/mpdecimal.c +++ b/Modules/_decimal/libmpdec/mpdecimal.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,18 +27,21 @@ #include "mpdecimal.h" + +#include +#include +#include #include #include #include -#include -#include + #include "basearith.h" #include "bits.h" +#include "constants.h" #include "convolute.h" #include "crt.h" #include "mpalloc.h" #include "typearith.h" -#include "umodarith.h" #ifdef PPRO #if defined(_MSC_VER) @@ -241,7 +244,7 @@ mpd_lsd(mpd_uint_t word) } /* Coefficient size needed to store 'digits' */ -ALWAYS_INLINE mpd_ssize_t +mpd_ssize_t mpd_digits_to_size(mpd_ssize_t digits) { mpd_ssize_t q, r; @@ -260,8 +263,9 @@ mpd_exp_digits(mpd_ssize_t exp) /* Canonical */ ALWAYS_INLINE int -mpd_iscanonical(const mpd_t *dec UNUSED) +mpd_iscanonical(const mpd_t *dec) { + (void)dec; return 1; } @@ -512,6 +516,28 @@ mpd_qresize(mpd_t *result, mpd_ssize_t nwords, uint32_t *status) return mpd_realloc_dyn(result, nwords, status); } +/* Same as mpd_qresize, but do not set the result no NaN on failure. */ +static ALWAYS_INLINE int +mpd_qresize_cxx(mpd_t *result, mpd_ssize_t nwords) +{ + assert(!mpd_isconst_data(result)); /* illegal operation for a const */ + assert(!mpd_isshared_data(result)); /* illegal operation for a shared */ + assert(MPD_MINALLOC <= result->alloc); + + nwords = (nwords <= MPD_MINALLOC) ? MPD_MINALLOC : nwords; + if (nwords == result->alloc) { + return 1; + } + if (mpd_isstatic_data(result)) { + if (nwords > result->alloc) { + return mpd_switch_to_dyn_cxx(result, nwords); + } + return 1; + } + + return mpd_realloc_dyn_cxx(result, nwords); +} + /* Same as mpd_qresize, but the complete coefficient (including the old * memory area!) is initialized to zero. */ ALWAYS_INLINE int @@ -1192,7 +1218,7 @@ _c32setu64(mpd_t *result, uint64_t u, uint8_t sign, uint32_t *status) result->data[i] = w[i]; } - mpd_set_sign(result, sign); + mpd_set_flags(result, sign); result->exp = 0; result->len = len; mpd_setdigits(result); @@ -1244,6 +1270,26 @@ mpd_qset_i64(mpd_t *result, int64_t a, const mpd_context_t *ctx, #endif } +/* quietly set a decimal from an int64_t, use a maxcontext for conversion */ +void +mpd_qset_i64_exact(mpd_t *result, int64_t a, uint32_t *status) +{ + mpd_context_t maxcontext; + + mpd_maxcontext(&maxcontext); +#ifdef CONFIG_64 + mpd_qset_ssize(result, a, &maxcontext, status); +#else + _c32_qset_i64(result, a, &maxcontext, status); +#endif + + if (*status & (MPD_Inexact|MPD_Rounded|MPD_Clamped)) { + /* we want exact results */ + mpd_seterror(result, MPD_Invalid_operation, status); + } + *status &= MPD_Errors; +} + /* quietly set a decimal from a uint64_t */ void mpd_qset_u64(mpd_t *result, uint64_t a, const mpd_context_t *ctx, @@ -1255,8 +1301,27 @@ mpd_qset_u64(mpd_t *result, uint64_t a, const mpd_context_t *ctx, _c32_qset_u64(result, a, ctx, status); #endif } -#endif /* !LEGACY_COMPILER */ +/* quietly set a decimal from a uint64_t, use a maxcontext for conversion */ +void +mpd_qset_u64_exact(mpd_t *result, uint64_t a, uint32_t *status) +{ + mpd_context_t maxcontext; + + mpd_maxcontext(&maxcontext); +#ifdef CONFIG_64 + mpd_qset_uint(result, a, &maxcontext, status); +#else + _c32_qset_u64(result, a, &maxcontext, status); +#endif + + if (*status & (MPD_Inexact|MPD_Rounded|MPD_Clamped)) { + /* we want exact results */ + mpd_seterror(result, MPD_Invalid_operation, status); + } + *status &= MPD_Errors; +} +#endif /* !LEGACY_COMPILER */ /* * Quietly get an mpd_uint_t from a decimal. Assumes @@ -1345,11 +1410,13 @@ mpd_qabs_uint(const mpd_t *a, uint32_t *status) mpd_ssize_t mpd_qget_ssize(const mpd_t *a, uint32_t *status) { + uint32_t workstatus = 0; mpd_uint_t u; int isneg; - u = mpd_qabs_uint(a, status); - if (*status&MPD_Invalid_operation) { + u = mpd_qabs_uint(a, &workstatus); + if (workstatus&MPD_Invalid_operation) { + *status |= workstatus; return MPD_SSIZE_MAX; } @@ -1469,9 +1536,11 @@ mpd_qget_i64(const mpd_t *a, uint32_t *status) uint32_t mpd_qget_u32(const mpd_t *a, uint32_t *status) { - uint64_t x = mpd_qget_uint(a, status); + uint32_t workstatus = 0; + uint64_t x = mpd_qget_uint(a, &workstatus); - if (*status&MPD_Invalid_operation) { + if (workstatus&MPD_Invalid_operation) { + *status |= workstatus; return UINT32_MAX; } if (x > UINT32_MAX) { @@ -1486,9 +1555,11 @@ mpd_qget_u32(const mpd_t *a, uint32_t *status) int32_t mpd_qget_i32(const mpd_t *a, uint32_t *status) { - int64_t x = mpd_qget_ssize(a, status); + uint32_t workstatus = 0; + int64_t x = mpd_qget_ssize(a, &workstatus); - if (*status&MPD_Invalid_operation) { + if (workstatus&MPD_Invalid_operation) { + *status |= workstatus; return INT32_MAX; } if (x < INT32_MIN || x > INT32_MAX) { @@ -1504,14 +1575,20 @@ mpd_qget_i32(const mpd_t *a, uint32_t *status) uint64_t mpd_qget_u64(const mpd_t *a, uint32_t *status) { - return _c32_qget_u64(1, a, status); + uint32_t workstatus = 0; + uint64_t x = _c32_qget_u64(1, a, &workstatus); + *status |= workstatus; + return x; } /* quietly get an int64_t from a decimal */ int64_t mpd_qget_i64(const mpd_t *a, uint32_t *status) { - return _c32_qget_i64(a, status); + uint32_t workstatus = 0; + int64_t x = _c32_qget_i64(a, &workstatus); + *status |= workstatus; + return x; } #endif @@ -1937,6 +2014,25 @@ mpd_qcopy(mpd_t *result, const mpd_t *a, uint32_t *status) return 1; } +/* Same as mpd_qcopy, but do not set the result to NaN on failure. */ +int +mpd_qcopy_cxx(mpd_t *result, const mpd_t *a) +{ + if (result == a) return 1; + + if (!mpd_qresize_cxx(result, a->len)) { + return 0; + } + + mpd_copy_flags(result, a); + result->exp = a->exp; + result->digits = a->digits; + result->len = a->len; + memcpy(result->data, a->data, a->len * (sizeof *result->data)); + + return 1; +} + /* * Copy to a decimal with a static buffer. The caller has to make sure that * the buffer is big enough. Cannot fail. @@ -3780,11 +3876,31 @@ void mpd_qdiv(mpd_t *q, const mpd_t *a, const mpd_t *b, const mpd_context_t *ctx, uint32_t *status) { - _mpd_qdiv(SET_IDEAL_EXP, q, a, b, ctx, status); + MPD_NEW_STATIC(aa,0,0,0,0); + MPD_NEW_STATIC(bb,0,0,0,0); + uint32_t xstatus = 0; - if (*status & MPD_Malloc_error) { + if (q == a) { + if (!mpd_qcopy(&aa, a, status)) { + mpd_seterror(q, MPD_Malloc_error, status); + goto out; + } + a = &aa; + } + + if (q == b) { + if (!mpd_qcopy(&bb, b, status)) { + mpd_seterror(q, MPD_Malloc_error, status); + goto out; + } + b = &bb; + } + + _mpd_qdiv(SET_IDEAL_EXP, q, a, b, ctx, &xstatus); + + if (xstatus & (MPD_Malloc_error|MPD_Division_impossible)) { /* Inexact quotients (the usual case) fill the entire context precision, - * which can lead to malloc() failures for very high precisions. Retry + * which can lead to the above errors for very high precisions. Retry * the operation with a lower precision in case the result is exact. * * We need an upper bound for the number of digits of a_coeff / b_coeff @@ -3799,25 +3915,33 @@ mpd_qdiv(mpd_t *q, const mpd_t *a, const mpd_t *b, * We arrive at a total upper bound: * * maxdigits(a_coeff') + maxdigits(1 / b_coeff') <= - * a->digits + log2(b_coeff) = - * a->digits + log10(b_coeff) / log10(2) <= + * log10(a_coeff) + log2(b_coeff) = + * log10(a_coeff) + log10(b_coeff) / log10(2) <= * a->digits + b->digits * 4; */ - uint32_t workstatus = 0; mpd_context_t workctx = *ctx; + uint32_t ystatus = 0; + workctx.prec = a->digits + b->digits * 4; if (workctx.prec >= ctx->prec) { - return; /* No point in retrying, keep the original error. */ + *status |= (xstatus&MPD_Errors); + goto out; /* No point in retrying, keep the original error. */ } - _mpd_qdiv(SET_IDEAL_EXP, q, a, b, &workctx, &workstatus); - if (workstatus == 0) { /* The result is exact, unrounded, normal etc. */ - *status = 0; - return; + _mpd_qdiv(SET_IDEAL_EXP, q, a, b, &workctx, &ystatus); + if (ystatus != 0) { + ystatus = *status | ((ystatus|xstatus)&MPD_Errors); + mpd_seterror(q, ystatus, status); } - - mpd_seterror(q, *status, status); } + else { + *status |= xstatus; + } + + +out: + mpd_del(&aa); + mpd_del(&bb); } /* Internal function. */ @@ -3907,6 +4031,7 @@ _mpd_qdivmod(mpd_t *q, mpd_t *r, const mpd_t *a, const mpd_t *b, } if (b->len == 1) { + assert(b->data[0] != 0); /* annotation for scan-build */ if (a->len == 1) { _mpd_div_word(&q->data[0], &r->data[0], a->data[0], b->data[0]); } @@ -6251,9 +6376,11 @@ _mpd_qpow_int(mpd_t *result, const mpd_t *base, const mpd_t *exp, workctx.round = MPD_ROUND_HALF_EVEN; workctx.clamp = 0; if (mpd_isnegative(exp)) { + uint32_t workstatus = 0; workctx.prec += 1; - mpd_qdiv(&tbase, &one, base, &workctx, status); - if (*status&MPD_Errors) { + mpd_qdiv(&tbase, &one, base, &workctx, &workstatus); + *status |= workstatus; + if (workstatus&MPD_Errors) { mpd_setspecial(result, MPD_POS, MPD_NAN); goto finish; } @@ -6988,6 +7115,8 @@ mpd_qrem_near(mpd_t *r, const mpd_t *a, const mpd_t *b, mpd_ssize_t expdiff, qdigits; int cmp, isodd, allnine; + assert(r != NULL); /* annotation for scan-build */ + if (mpd_isspecial(a) || mpd_isspecial(b)) { if (mpd_qcheck_nans(r, a, b, ctx, status)) { return; @@ -7218,6 +7347,11 @@ void mpd_qtrunc(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status) { + if (mpd_isspecial(a)) { + mpd_seterror(result, MPD_Invalid_operation, status); + return; + } + (void)_mpd_qround_to_integral(TO_INT_TRUNC, result, a, ctx, status); } @@ -7226,6 +7360,12 @@ mpd_qfloor(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status) { mpd_context_t workctx = *ctx; + + if (mpd_isspecial(a)) { + mpd_seterror(result, MPD_Invalid_operation, status); + return; + } + workctx.round = MPD_ROUND_FLOOR; (void)_mpd_qround_to_integral(TO_INT_SILENT, result, a, &workctx, status); @@ -7236,6 +7376,12 @@ mpd_qceil(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status) { mpd_context_t workctx = *ctx; + + if (mpd_isspecial(a)) { + mpd_seterror(result, MPD_Invalid_operation, status); + return; + } + workctx.round = MPD_ROUND_CEILING; (void)_mpd_qround_to_integral(TO_INT_SILENT, result, a, &workctx, status); @@ -7877,9 +8023,20 @@ void mpd_qsqrt(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, uint32_t *status) { - _mpd_qsqrt(result, a, ctx, status); + MPD_NEW_STATIC(aa,0,0,0,0); + uint32_t xstatus = 0; + + if (result == a) { + if (!mpd_qcopy(&aa, a, status)) { + mpd_seterror(result, MPD_Malloc_error, status); + goto out; + } + a = &aa; + } + + _mpd_qsqrt(result, a, ctx, &xstatus); - if (*status & (MPD_Malloc_error|MPD_Division_impossible)) { + if (xstatus & (MPD_Malloc_error|MPD_Division_impossible)) { /* The above conditions can occur at very high context precisions * if intermediate values get too large. Retry the operation with * a lower context precision in case the result is exact. @@ -7889,22 +8046,27 @@ mpd_qsqrt(mpd_t *result, const mpd_t *a, const mpd_context_t *ctx, * * NOTE: sqrt(40e9) = 2.0e+5 /\ digits(40e9) = digits(2.0e+5) = 2 */ - uint32_t workstatus = 0; + uint32_t ystatus = 0; mpd_context_t workctx = *ctx; - workctx.prec = a->digits; + workctx.prec = a->digits; if (workctx.prec >= ctx->prec) { - return; /* No point in repeating this, keep the original error. */ + *status |= (xstatus|MPD_Errors); + goto out; /* No point in repeating this, keep the original error. */ } - _mpd_qsqrt(result, a, &workctx, &workstatus); - if (workstatus == 0) { - *status = 0; - return; + _mpd_qsqrt(result, a, &workctx, &ystatus); + if (ystatus != 0) { + ystatus = *status | ((xstatus|ystatus)&MPD_Errors); + mpd_seterror(result, ystatus, status); } - - mpd_seterror(result, *status, status); } + else { + *status |= xstatus; + } + +out: + mpd_del(&aa); } @@ -7918,6 +8080,7 @@ mpd_sizeinbase(const mpd_t *a, uint32_t base) { double x; size_t digits; + double upper_bound; assert(mpd_isinteger(a)); assert(base >= 2); @@ -7934,10 +8097,14 @@ mpd_sizeinbase(const mpd_t *a, uint32_t base) if (digits > 2711437152599294ULL) { return SIZE_MAX; } + + upper_bound = (double)((1ULL<<53)-1); +#else + upper_bound = (double)(SIZE_MAX-1); #endif x = (double)digits / log10(base); - return (x > SIZE_MAX-1) ? SIZE_MAX : (size_t)x + 1; + return (x > upper_bound) ? SIZE_MAX : (size_t)x + 1; } /* Space needed to import a base 'base' integer of length 'srclen'. */ @@ -7945,6 +8112,7 @@ static mpd_ssize_t _mpd_importsize(size_t srclen, uint32_t base) { double x; + double upper_bound; assert(srclen > 0); assert(base >= 2); @@ -7953,10 +8121,15 @@ _mpd_importsize(size_t srclen, uint32_t base) if (srclen > (1ULL<<53)) { return MPD_SSIZE_MAX; } + + assert((1ULL<<53) <= MPD_MAXIMPORT); + upper_bound = (double)((1ULL<<53)-1); +#else + upper_bound = MPD_MAXIMPORT-1; #endif x = (double)srclen * (log10(base)/MPD_RDIGITS); - return (x >= MPD_MAXIMPORT) ? MPD_SSIZE_MAX : (mpd_ssize_t)x + 1; + return (x > upper_bound) ? MPD_SSIZE_MAX : (mpd_ssize_t)x + 1; } static uint8_t @@ -8483,6 +8656,3 @@ mpd_qimport_u32(mpd_t *result, mpd_qresize(result, result->len, status); mpd_qfinalize(result, ctx, status); } - - - diff --git a/Modules/_decimal/libmpdec/mpdecimal.h b/Modules/_decimal/libmpdec/mpdecimal.h index a67dd9bc126c27c..108b76efa859417 100644 --- a/Modules/_decimal/libmpdec/mpdecimal.h +++ b/Modules/_decimal/libmpdec/mpdecimal.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,59 +26,51 @@ */ -#ifndef MPDECIMAL_H -#define MPDECIMAL_H +#ifndef LIBMPDEC_MPDECIMAL_H_ +#define LIBMPDEC_MPDECIMAL_H_ +#ifndef _MSC_VER + #include "pyconfig.h" +#endif + #ifdef __cplusplus + #include + #include + #include + #include + #include extern "C" { - #ifndef __STDC_LIMIT_MACROS - #define __STDC_LIMIT_MACROS - #define MPD_CLEAR_STDC_LIMIT_MACROS - #endif +#else + #include + #include + #include + #include + #include #endif -#ifndef _MSC_VER - #include "pyconfig.h" +#if (defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__)) && \ + defined(__GNUC__) && __GNUC__ >= 4 && !defined(__INTEL_COMPILER) + #define MPD_PRAGMA(x) _Pragma(x) + #define MPD_HIDE_SYMBOLS_START "GCC visibility push(hidden)" + #define MPD_HIDE_SYMBOLS_END "GCC visibility pop" +#else + #define MPD_PRAGMA(x) + #define MPD_HIDE_SYMBOLS_START + #define MPD_HIDE_SYMBOLS_END #endif -#include -#include -#include -#include -#include -#include -#include +#if defined(__GNUC__) && !defined(__INTEL_COMPILER) + #define UNUSED __attribute__((unused)) +#else + #define UNUSED +#endif -#ifdef _MSC_VER +#if defined(_MSC_VER) #include "vccompat.h" - #ifndef UNUSED - #define UNUSED - #endif - #define MPD_PRAGMA(x) - #define MPD_HIDE_SYMBOLS_START - #define MPD_HIDE_SYMBOLS_END #define EXTINLINE extern inline #else - #ifndef __GNUC_STDC_INLINE__ - #define __GNUC_STDC_INLINE__ 1 - #endif - #if defined(__GNUC__) && !defined(__INTEL_COMPILER) - #define UNUSED __attribute__((unused)) - #else - #define UNUSED - #endif - #if (defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__)) && \ - defined(__GNUC__) && __GNUC__ >= 4 && !defined(__INTEL_COMPILER) - #define MPD_PRAGMA(x) _Pragma(x) - #define MPD_HIDE_SYMBOLS_START "GCC visibility push(hidden)" - #define MPD_HIDE_SYMBOLS_END "GCC visibility pop" - #else - #define MPD_PRAGMA(x) - #define MPD_HIDE_SYMBOLS_START - #define MPD_HIDE_SYMBOLS_END - #endif #define EXTINLINE #endif @@ -103,10 +95,10 @@ MPD_PRAGMA(MPD_HIDE_SYMBOLS_START) /******************************************************************************/ #define MPD_MAJOR_VERSION 2 -#define MPD_MINOR_VERSION 4 -#define MPD_MICRO_VERSION 2 +#define MPD_MINOR_VERSION 5 +#define MPD_MICRO_VERSION 0 -#define MPD_VERSION "2.4.2" +#define MPD_VERSION "2.5.0" #define MPD_VERSION_HEX ((MPD_MAJOR_VERSION << 24) | \ (MPD_MINOR_VERSION << 16) | \ @@ -423,6 +415,7 @@ void mpd_print(const mpd_t *dec); /* assignment from a string */ void mpd_qset_string(mpd_t *dec, const char *s, const mpd_context_t *ctx, uint32_t *status); +void mpd_qset_string_exact(mpd_t *dec, const char *s, uint32_t *status); /* set to NaN with error flags */ void mpd_seterror(mpd_t *result, uint32_t flags, uint32_t *status); @@ -440,6 +433,8 @@ void mpd_qset_u32(mpd_t *result, uint32_t a, const mpd_context_t *ctx, uint32_t #ifndef LEGACY_COMPILER void mpd_qset_i64(mpd_t *result, int64_t a, const mpd_context_t *ctx, uint32_t *status); void mpd_qset_u64(mpd_t *result, uint64_t a, const mpd_context_t *ctx, uint32_t *status); +void mpd_qset_i64_exact(mpd_t *result, int64_t a, uint32_t *status); +void mpd_qset_u64_exact(mpd_t *result, uint64_t a, uint32_t *status); #endif /* quietly assign a C integer type to an mpd_t with a static coefficient */ @@ -467,7 +462,8 @@ void mpd_qfinalize(mpd_t *result, const mpd_context_t *ctx, uint32_t *status); const char *mpd_class(const mpd_t *a, const mpd_context_t *ctx); -int mpd_qcopy(mpd_t *result, const mpd_t *a, uint32_t *status); +int mpd_qcopy(mpd_t *result, const mpd_t *a, uint32_t *status); +int mpd_qcopy_cxx(mpd_t *result, const mpd_t *a); mpd_t *mpd_qncopy(const mpd_t *a); int mpd_qcopy_abs(mpd_t *result, const mpd_t *a, uint32_t *status); int mpd_qcopy_negate(mpd_t *result, const mpd_t *a, uint32_t *status); @@ -721,7 +717,7 @@ EXTINLINE mpd_uint_t mpd_lsd(mpd_uint_t word); EXTINLINE mpd_ssize_t mpd_digits_to_size(mpd_ssize_t digits); /* number of digits in the exponent, undefined for MPD_SSIZE_MIN */ EXTINLINE int mpd_exp_digits(mpd_ssize_t exp); -EXTINLINE int mpd_iscanonical(const mpd_t *dec UNUSED); +EXTINLINE int mpd_iscanonical(const mpd_t *dec); EXTINLINE int mpd_isfinite(const mpd_t *dec); EXTINLINE int mpd_isinfinite(const mpd_t *dec); EXTINLINE int mpd_isinteger(const mpd_t *dec); @@ -833,15 +829,8 @@ MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ #ifdef __cplusplus - #ifdef MPD_CLEAR_STDC_LIMIT_MACROS - #undef MPD_CLEAR_STDC_LIMIT_MACROS - #undef __STDC_LIMIT_MACROS - #endif } /* END extern "C" */ #endif -#endif /* MPDECIMAL_H */ - - - +#endif /* LIBMPDEC_MPDECIMAL_H_ */ diff --git a/Modules/_decimal/libmpdec/numbertheory.c b/Modules/_decimal/libmpdec/numbertheory.c index 4e035477e280001..210e0deb3712034 100644 --- a/Modules/_decimal/libmpdec/numbertheory.c +++ b/Modules/_decimal/libmpdec/numbertheory.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,11 +27,13 @@ #include "mpdecimal.h" -#include + #include +#include + #include "bits.h" -#include "umodarith.h" #include "numbertheory.h" +#include "umodarith.h" /* Bignum: Initialize the Number Theoretic Transform. */ @@ -128,5 +130,3 @@ _mpd_init_w3table(mpd_uint_t w3table[3], int sign, int modnum) w3table[1] = kernel; w3table[2] = POWMOD(kernel, 2); } - - diff --git a/Modules/_decimal/libmpdec/numbertheory.h b/Modules/_decimal/libmpdec/numbertheory.h index e94c157910c83e9..47b7753b831b89b 100644 --- a/Modules/_decimal/libmpdec/numbertheory.h +++ b/Modules/_decimal/libmpdec/numbertheory.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,12 @@ */ -#ifndef NUMBER_THEORY_H -#define NUMBER_THEORY_H +#ifndef LIBMPDEC_NUMBERTHEORY_H_ +#define LIBMPDEC_NUMBERTHEORY_H_ -#include "constants.h" #include "mpdecimal.h" +#include "constants.h" /* Internal header file: all symbols have local scope in the DSO */ @@ -73,6 +73,4 @@ std_setmodulus(int modnum, mpd_uint_t *umod) MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ -#endif - - +#endif /* LIBMPDEC_NUMBERTHEORY_H_ */ diff --git a/Modules/_decimal/libmpdec/sixstep.c b/Modules/_decimal/libmpdec/sixstep.c index 92d513ebe18286a..a4d1dbed7813c6c 100644 --- a/Modules/_decimal/libmpdec/sixstep.c +++ b/Modules/_decimal/libmpdec/sixstep.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,15 +27,17 @@ #include "mpdecimal.h" -#include -#include + #include +#include + #include "bits.h" +#include "constants.h" #include "difradix2.h" #include "numbertheory.h" +#include "sixstep.h" #include "transpose.h" #include "umodarith.h" -#include "sixstep.h" /* Bignum: Cache efficient Matrix Fourier Transform for arrays of the @@ -210,5 +212,3 @@ inv_six_step_fnt(mpd_uint_t *a, mpd_size_t n, int modnum) return 1; } - - diff --git a/Modules/_decimal/libmpdec/sixstep.h b/Modules/_decimal/libmpdec/sixstep.h index 4a8b015e3a9b90e..89b4a33afc79207 100644 --- a/Modules/_decimal/libmpdec/sixstep.h +++ b/Modules/_decimal/libmpdec/sixstep.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,11 @@ */ -#ifndef SIX_STEP_H -#define SIX_STEP_H +#ifndef LIBMPDEC_SIXSTEP_H_ +#define LIBMPDEC_SIXSTEP_H_ #include "mpdecimal.h" -#include /* Internal header file: all symbols have local scope in the DSO */ @@ -45,4 +44,4 @@ int inv_six_step_fnt(mpd_uint_t *a, mpd_size_t n, int modnum); MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ -#endif +#endif /* LIBMPDEC_SIXSTEP_H_ */ diff --git a/Modules/_decimal/libmpdec/transpose.c b/Modules/_decimal/libmpdec/transpose.c index 55d6d8992279009..56321b5f39a733e 100644 --- a/Modules/_decimal/libmpdec/transpose.c +++ b/Modules/_decimal/libmpdec/transpose.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,15 +27,17 @@ #include "mpdecimal.h" + +#include +#include #include #include #include -#include -#include + #include "bits.h" #include "constants.h" -#include "typearith.h" #include "transpose.h" +#include "typearith.h" #define BUFSIZE 4096 @@ -272,5 +274,3 @@ transpose_pow2(mpd_uint_t *matrix, mpd_size_t rows, mpd_size_t cols) return 1; } - - diff --git a/Modules/_decimal/libmpdec/transpose.h b/Modules/_decimal/libmpdec/transpose.h index e1cd1fa17dd7730..e91c18d74356bcf 100644 --- a/Modules/_decimal/libmpdec/transpose.h +++ b/Modules/_decimal/libmpdec/transpose.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,11 @@ */ -#ifndef TRANSPOSE_H -#define TRANSPOSE_H +#ifndef LIBMPDEC_TRANSPOSE_H_ +#define LIBMPDEC_TRANSPOSE_H_ #include "mpdecimal.h" -#include /* Internal header file: all symbols have local scope in the DSO */ @@ -59,4 +58,4 @@ static inline void pointerswap(mpd_uint_t **a, mpd_uint_t **b) MPD_PRAGMA(MPD_HIDE_SYMBOLS_END) /* restore previous scope rules */ -#endif +#endif /* LIBMPDEC_TRANSPOSE_H_ */ diff --git a/Modules/_decimal/libmpdec/typearith.h b/Modules/_decimal/libmpdec/typearith.h index 405237dac516ad0..47961788d764198 100644 --- a/Modules/_decimal/libmpdec/typearith.h +++ b/Modules/_decimal/libmpdec/typearith.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,14 @@ */ -#ifndef TYPEARITH_H -#define TYPEARITH_H +#ifndef LIBMPDEC_TYPEARITH_H_ +#define LIBMPDEC_TYPEARITH_H_ #include "mpdecimal.h" +#include + /*****************************************************************************/ /* Low level native arithmetic on basic types */ @@ -663,7 +665,4 @@ mulmod_size_t(mpd_size_t a, mpd_size_t b, mpd_size_t m) } -#endif /* TYPEARITH_H */ - - - +#endif /* LIBMPDEC_TYPEARITH_H_ */ diff --git a/Modules/_decimal/libmpdec/umodarith.h b/Modules/_decimal/libmpdec/umodarith.h index 68d15188cb39e5a..d7dbbbe6a7331a8 100644 --- a/Modules/_decimal/libmpdec/umodarith.h +++ b/Modules/_decimal/libmpdec/umodarith.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,12 +26,13 @@ */ -#ifndef UMODARITH_H -#define UMODARITH_H +#ifndef LIBMPDEC_UMODARITH_H_ +#define LIBMPDEC_UMODARITH_H_ -#include "constants.h" #include "mpdecimal.h" + +#include "constants.h" #include "typearith.h" @@ -644,7 +645,4 @@ ppro_powmod(mpd_uint_t base, mpd_uint_t exp, double *dmod, uint32_t *dinvmod) #endif /* CONFIG_32 */ -#endif /* UMODARITH_H */ - - - +#endif /* LIBMPDEC_UMODARITH_H_ */ diff --git a/Modules/_decimal/libmpdec/vccompat.h b/Modules/_decimal/libmpdec/vccompat.h index 2ba805dcc564633..e2e1c42cc0250e1 100644 --- a/Modules/_decimal/libmpdec/vccompat.h +++ b/Modules/_decimal/libmpdec/vccompat.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + * Copyright (c) 2008-2020 Stefan Krah. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,14 +26,16 @@ */ -#ifndef VCCOMPAT_H -#define VCCOMPAT_H +#ifndef LIBMPDEC_VCCOMPAT_H_ +#define LIBMPDEC_VCCOMPAT_H_ /* Visual C fixes: no snprintf ... */ #ifdef _MSC_VER - #undef inline - #define inline __inline + #ifndef __cplusplus + #undef inline + #define inline __inline + #endif #undef random #define random rand #undef srandom @@ -51,7 +53,4 @@ #endif -#endif /* VCCOMPAT_H */ - - - +#endif /* LIBMPDEC_VCCOMPAT_H_ */ diff --git a/Modules/_decimal/libmpdec/vcdiv64.asm b/Modules/_decimal/libmpdec/vcdiv64.asm index 6b6645673ab5ae6..597e9ba9352c8fc 100644 --- a/Modules/_decimal/libmpdec/vcdiv64.asm +++ b/Modules/_decimal/libmpdec/vcdiv64.asm @@ -1,5 +1,5 @@ ; -; Copyright (c) 2008-2016 Stefan Krah. All rights reserved. +; Copyright (c) 2008-2020 Stefan Krah. All rights reserved. ; ; Redistribution and use in source and binary forms, with or without ; modification, are permitted provided that the following conditions @@ -44,5 +44,3 @@ _mpd_div_words PROC _mpd_div_words ENDP _TEXT ENDS END - - From fef1fae9df3b03510f9defb25bd0388135b4c591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Furkan=20=C3=96nder?= Date: Fri, 5 Jun 2020 22:56:32 +0300 Subject: [PATCH 03/19] bpo-19468: delete unnecessary instance check in importlib.reload() (GH-19424) Automerge-Triggered-By: @brettcannon --- Lib/importlib/__init__.py | 8 ++++---- .../2020-05-30-23-18-35.bpo-19468.S-TA7p.rst | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-05-30-23-18-35.bpo-19468.S-TA7p.rst diff --git a/Lib/importlib/__init__.py b/Lib/importlib/__init__.py index 0c73c505f98dba0..bea37d766262f88 100644 --- a/Lib/importlib/__init__.py +++ b/Lib/importlib/__init__.py @@ -54,7 +54,6 @@ # Fully bootstrapped at this point, import whatever you like, circular # dependencies and startup overhead minimisation permitting :) -import types import warnings @@ -136,12 +135,13 @@ def reload(module): The module must have been successfully imported before. """ - if not module or not isinstance(module, types.ModuleType): - raise TypeError("reload() argument must be a module") try: name = module.__spec__.name except AttributeError: - name = module.__name__ + try: + name = module.__name__ + except AttributeError: + raise TypeError("reload() argument must be a module") if sys.modules.get(name) is not module: msg = "module {} not in sys.modules" diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-05-30-23-18-35.bpo-19468.S-TA7p.rst b/Misc/NEWS.d/next/Core and Builtins/2020-05-30-23-18-35.bpo-19468.S-TA7p.rst new file mode 100644 index 000000000000000..e35750e37f4da39 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-05-30-23-18-35.bpo-19468.S-TA7p.rst @@ -0,0 +1,2 @@ +Delete unnecessary instance check in importlib.reload(). +Patch by Furkan Γ–nder. From 5fe1df1886e2e53b04bf76ef916857271d3c8f20 Mon Sep 17 00:00:00 2001 From: Stefan Krah Date: Fri, 5 Jun 2020 22:01:18 +0200 Subject: [PATCH 04/19] Fix missing FloatOperation in EXTRA_FUNCTIONALITY path. (#20655) --- Lib/test/test_decimal.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index f1abd2aecb12268..ed483a470952751 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -5201,6 +5201,7 @@ def test_c_signal_dict(self): DefaultContext = C.DefaultContext InvalidOperation = C.InvalidOperation + FloatOperation = C.FloatOperation DivisionByZero = C.DivisionByZero Overflow = C.Overflow Subnormal = C.Subnormal @@ -5274,6 +5275,7 @@ def assertIsExclusivelySet(signal, signal_dict): Underflow: C.DecUnderflow, Overflow: C.DecOverflow, DivisionByZero: C.DecDivisionByZero, + FloatOperation: C.DecFloatOperation, InvalidOperation: C.DecIEEEInvalidOperation } IntCond = [ From 161541ab45278df6603dd870113b10f13e4d9e16 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 5 Jun 2020 16:34:16 -0400 Subject: [PATCH 05/19] bpo-39791: Refresh importlib.metadata from importlib_metadata 1.6.1. (GH-20659) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Refresh importlib.metadata from importlib_metadata 1.6.1. * πŸ“œπŸ€– Added by blurb_it. Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Doc/library/importlib.metadata.rst | 14 +++++++-- Lib/importlib/metadata.py | 30 +++++++++++++++---- Lib/test/test_importlib/fixtures.py | 15 ++++++++++ Lib/test/test_importlib/test_main.py | 16 ++++++++++ Lib/test/test_importlib/test_zip.py | 26 +++++++++------- .../2020-06-05-19-29-10.bpo-39791._CcO3d.rst | 1 + 6 files changed, 84 insertions(+), 18 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-06-05-19-29-10.bpo-39791._CcO3d.rst diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 15e58b860d97d07..21da143f3bebf9a 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -77,7 +77,9 @@ Entry points The ``entry_points()`` function returns a dictionary of all entry points, keyed by group. Entry points are represented by ``EntryPoint`` instances; each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and -a ``.load()`` method to resolve the value. +a ``.load()`` method to resolve the value. There are also ``.module``, +``.attr``, and ``.extras`` attributes for getting the components of the +``.value`` attribute:: >>> eps = entry_points() # doctest: +SKIP >>> list(eps) # doctest: +SKIP @@ -86,6 +88,12 @@ a ``.load()`` method to resolve the value. >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] # doctest: +SKIP >>> wheel # doctest: +SKIP EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') + >>> wheel.module # doctest: +SKIP + 'wheel.cli' + >>> wheel.attr # doctest: +SKIP + 'main' + >>> wheel.extras # doctest: +SKIP + [] >>> main = wheel.load() # doctest: +SKIP >>> main # doctest: +SKIP @@ -94,7 +102,7 @@ The ``group`` and ``name`` are arbitrary values defined by the package author and usually a client will wish to resolve all entry points for a particular group. Read `the setuptools docs `_ -for more information on entrypoints, their definition, and usage. +for more information on entry points, their definition, and usage. .. _metadata: @@ -235,7 +243,7 @@ method:: """ The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` -properties indicating the path to search and names to match and may +properties indicating the path to search and name to match and may supply other relevant context. What this means in practice is that to support finding distribution package diff --git a/Lib/importlib/metadata.py b/Lib/importlib/metadata.py index 831f593277ccd47..ffa0cba45706d6e 100644 --- a/Lib/importlib/metadata.py +++ b/Lib/importlib/metadata.py @@ -78,6 +78,16 @@ def load(self): attrs = filter(None, (match.group('attr') or '').split('.')) return functools.reduce(getattr, attrs, module) + @property + def module(self): + match = self.pattern.match(self.value) + return match.group('module') + + @property + def attr(self): + match = self.pattern.match(self.value) + return match.group('attr') + @property def extras(self): match = self.pattern.match(self.value) @@ -170,7 +180,7 @@ def from_name(cls, name): """ for resolver in cls._discover_resolvers(): dists = resolver(DistributionFinder.Context(name=name)) - dist = next(dists, None) + dist = next(iter(dists), None) if dist is not None: return dist else: @@ -213,6 +223,17 @@ def _discover_resolvers(): ) return filter(None, declared) + @classmethod + def _local(cls, root='.'): + from pep517 import build, meta + system = build.compat_system(root) + builder = functools.partial( + meta.build, + source_dir=root, + system=system, + ) + return PathDistribution(zipfile.Path(meta.build_as_zip(builder))) + @property def metadata(self): """Return the parsed metadata for this Distribution. @@ -391,7 +412,7 @@ class FastPath: def __init__(self, root): self.root = root - self.base = os.path.basename(root).lower() + self.base = os.path.basename(self.root).lower() def joinpath(self, child): return pathlib.Path(self.root, child) @@ -408,8 +429,8 @@ def zip_children(self): names = zip_path.root.namelist() self.joinpath = zip_path.joinpath - return ( - posixpath.split(child)[0] + return dict.fromkeys( + child.split(posixpath.sep, 1)[0] for child in names ) @@ -475,7 +496,6 @@ def _search_paths(cls, name, paths): ) - class PathDistribution(Distribution): def __init__(self, path): """Construct a distribution from a path to the metadata directory. diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py index d923cec26ea8f5f..b25febb7fe75641 100644 --- a/Lib/test/test_importlib/fixtures.py +++ b/Lib/test/test_importlib/fixtures.py @@ -161,6 +161,21 @@ def setUp(self): build_files(EggInfoFile.files, prefix=self.site_dir) +class LocalPackage: + files = { + "setup.py": """ + import setuptools + setuptools.setup(name="local-pkg", version="2.0.1") + """, + } + + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + self.fixtures.enter_context(tempdir_as_cwd()) + build_files(self.files) + + def build_files(file_defs, prefix=pathlib.Path()): """Build a set of files/directories, as described by the diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py index 42a79992ecc8c08..7b18c3de16eea08 100644 --- a/Lib/test/test_importlib/test_main.py +++ b/Lib/test/test_importlib/test_main.py @@ -246,3 +246,19 @@ def test_json_dump(self): """ with self.assertRaises(Exception): json.dumps(self.ep) + + def test_module(self): + assert self.ep.module == 'value' + + def test_attr(self): + assert self.ep.attr is None + + +class FileSystem(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + def test_unicode_dir_on_sys_path(self): + """ + Ensure a Unicode subdirectory of a directory on sys.path + does not crash. + """ + fixtures.build_files({'β˜ƒ': {}}, prefix=self.site_dir) + list(distributions()) diff --git a/Lib/test/test_importlib/test_zip.py b/Lib/test/test_importlib/test_zip.py index fa87cd7cb109622..a5399c16682fbef 100644 --- a/Lib/test/test_importlib/test_zip.py +++ b/Lib/test/test_importlib/test_zip.py @@ -3,9 +3,10 @@ from contextlib import ExitStack from importlib.metadata import ( - distribution, entry_points, files, PackageNotFoundError, version, + distribution, entry_points, files, PackageNotFoundError, + version, distributions, ) -from importlib.resources import path +from importlib import resources from test.support import requires_zlib @@ -14,15 +15,19 @@ class TestZip(unittest.TestCase): root = 'test.test_importlib.data' + def _fixture_on_path(self, filename): + pkg_file = resources.files(self.root).joinpath(filename) + file = self.resources.enter_context(resources.as_file(pkg_file)) + assert file.name.startswith('example-'), file.name + sys.path.insert(0, str(file)) + self.resources.callback(sys.path.pop, 0) + def setUp(self): # Find the path to the example-*.whl so we can add it to the front of # sys.path, where we'll then try to find the metadata thereof. self.resources = ExitStack() self.addCleanup(self.resources.close) - wheel = self.resources.enter_context( - path(self.root, 'example-21.12-py3-none-any.whl')) - sys.path.insert(0, str(wheel)) - self.resources.callback(sys.path.pop, 0) + self._fixture_on_path('example-21.12-py3-none-any.whl') def test_zip_version(self): self.assertEqual(version('example'), '21.12') @@ -49,6 +54,10 @@ def test_files(self): path = str(file.dist.locate_file(file)) assert '.whl/' in path, path + def test_one_distribution(self): + dists = list(distributions(path=sys.path[:1])) + assert len(dists) == 1 + @requires_zlib() class TestEgg(TestZip): @@ -57,10 +66,7 @@ def setUp(self): # sys.path, where we'll then try to find the metadata thereof. self.resources = ExitStack() self.addCleanup(self.resources.close) - egg = self.resources.enter_context( - path(self.root, 'example-21.12-py3.6.egg')) - sys.path.insert(0, str(egg)) - self.resources.callback(sys.path.pop, 0) + self._fixture_on_path('example-21.12-py3.6.egg') def test_files(self): for file in files('example'): diff --git a/Misc/NEWS.d/next/Library/2020-06-05-19-29-10.bpo-39791._CcO3d.rst b/Misc/NEWS.d/next/Library/2020-06-05-19-29-10.bpo-39791._CcO3d.rst new file mode 100644 index 000000000000000..73e0cbb013f8407 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-06-05-19-29-10.bpo-39791._CcO3d.rst @@ -0,0 +1 @@ +Refresh importlib.metadata from importlib_metadata 1.6.1. \ No newline at end of file From 235f918f44bb89e27190db2f1823d191dbd4ad28 Mon Sep 17 00:00:00 2001 From: Ram Rachum Date: Fri, 5 Jun 2020 23:56:06 +0300 Subject: [PATCH 06/19] bpo-40876: Clarify error message in the csv module (GH-20653) --- .../next/Library/2020-06-05-20-00-18.bpo-40876.zDhiZj.rst | 1 + Modules/_csv.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2020-06-05-20-00-18.bpo-40876.zDhiZj.rst diff --git a/Misc/NEWS.d/next/Library/2020-06-05-20-00-18.bpo-40876.zDhiZj.rst b/Misc/NEWS.d/next/Library/2020-06-05-20-00-18.bpo-40876.zDhiZj.rst new file mode 100644 index 000000000000000..75f62addbabbc52 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-06-05-20-00-18.bpo-40876.zDhiZj.rst @@ -0,0 +1 @@ +Clarify error message in the :mod:`csv` module. diff --git a/Modules/_csv.c b/Modules/_csv.c index f33733aaf850d16..7e44419c0876ba7 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -810,7 +810,7 @@ Reader_iternext(ReaderObj *self) PyErr_Format(_csvstate_global->error_obj, "iterator should return strings, " "not %.200s " - "(did you open the file in text mode?)", + "(the file should be opened in text mode)", Py_TYPE(lineobj)->tp_name ); Py_DECREF(lineobj); From 45af786e111aed5f687e1f0d8b45b6a5e678a6bc Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Fri, 5 Jun 2020 23:32:09 +0200 Subject: [PATCH 07/19] bpo-40867: Remove unused include from Module/_randommodule.c (GH-20635) --- Modules/_randommodule.c | 1 - 1 file changed, 1 deletion(-) diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index 3589173edcb6201..3e3139e4990cc35 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -67,7 +67,6 @@ /* ---------------------------------------------------------------*/ #include "Python.h" -#include "pycore_byteswap.h" // _Py_bswap32() #ifdef HAVE_PROCESS_H # include // getpid() #endif From b084d1b97e369293d2d2bc0791e2135822c923a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Lapeyre?= Date: Sat, 6 Jun 2020 00:00:42 +0200 Subject: [PATCH 08/19] bpo-40862: Raise TypeError when const is given to argparse.BooleanOptionalAction (GH-20623) --- Lib/argparse.py | 1 - Lib/test/test_argparse.py | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/argparse.py b/Lib/argparse.py index 2677ef63e9e541f..2fb1da59f942cff 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -857,7 +857,6 @@ class BooleanOptionalAction(Action): def __init__(self, option_strings, dest, - const=None, default=None, type=None, choices=None, diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index e82a0c39c21a8b0..22cae626ccc2973 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -700,6 +700,14 @@ class TestBooleanOptionalAction(ParserTestCase): ('--no-foo --foo', NS(foo=True)), ] + def test_const(self): + # See bpo-40862 + parser = argparse.ArgumentParser() + with self.assertRaises(TypeError) as cm: + parser.add_argument('--foo', const=True, action=argparse.BooleanOptionalAction) + + self.assertIn("got an unexpected keyword argument 'const'", str(cm.exception)) + class TestBooleanOptionalActionRequired(ParserTestCase): """Tests BooleanOptionalAction required""" From a54096e30523534e8eebb8dc1011b4536ed237a8 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 6 Jun 2020 00:52:15 +0100 Subject: [PATCH 09/19] bpo-40883: Fix memory leak in fstring_compile_expr in parse_string.c (GH-20667) --- .../Core and Builtins/2020-06-05-23-25-00.bpo-40883.M6sQ-Q.rst | 1 + Parser/pegen/parse_string.c | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-06-05-23-25-00.bpo-40883.M6sQ-Q.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-06-05-23-25-00.bpo-40883.M6sQ-Q.rst b/Misc/NEWS.d/next/Core and Builtins/2020-06-05-23-25-00.bpo-40883.M6sQ-Q.rst new file mode 100644 index 000000000000000..ebeb0cc60d16bb4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-06-05-23-25-00.bpo-40883.M6sQ-Q.rst @@ -0,0 +1 @@ +Fix memory leak in when parsing f-strings in the new parser. Patch by Pablo Galindo \ No newline at end of file diff --git a/Parser/pegen/parse_string.c b/Parser/pegen/parse_string.c index e24ecc58d3aa1dc..efe82df47658bdb 100644 --- a/Parser/pegen/parse_string.c +++ b/Parser/pegen/parse_string.c @@ -604,6 +604,7 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end, struct tok_state* tok = PyTokenizer_FromString(str, 1); if (tok == NULL) { + PyMem_RawFree(str); return NULL; } Py_INCREF(p->tok->filename); @@ -629,6 +630,7 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end, result = expr; exit: + PyMem_RawFree(str); _PyPegen_Parser_Free(p2); PyTokenizer_Free(tok); return result; From 2e6593db0086004a1ca7f7049218ff9573d473c2 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Sat, 6 Jun 2020 00:52:27 +0100 Subject: [PATCH 10/19] bpo-40880: Fix invalid read in newline_in_string in pegen.c (#20666) * bpo-40880: Fix invalid read in newline_in_string in pegen.c * Update Parser/pegen/pegen.c Co-authored-by: Lysandros Nikolaou * Add NEWS entry Co-authored-by: Lysandros Nikolaou --- .../2020-06-06-00-23-19.bpo-40880.fjdzSh.rst | 2 ++ Parser/pegen/pegen.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-06-06-00-23-19.bpo-40880.fjdzSh.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-06-06-00-23-19.bpo-40880.fjdzSh.rst b/Misc/NEWS.d/next/Core and Builtins/2020-06-06-00-23-19.bpo-40880.fjdzSh.rst new file mode 100644 index 000000000000000..ab42f5c205f81b4 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-06-06-00-23-19.bpo-40880.fjdzSh.rst @@ -0,0 +1,2 @@ +Fix invalid memory read in the new parser when checking newlines in string +literals. Patch by Pablo Galindo. diff --git a/Parser/pegen/pegen.c b/Parser/pegen/pegen.c index c55ff7e45c0da09..afe75d7f862eed0 100644 --- a/Parser/pegen/pegen.c +++ b/Parser/pegen/pegen.c @@ -937,8 +937,8 @@ _PyPegen_number_token(Parser *p) static int // bool newline_in_string(Parser *p, const char *cur) { - for (char c = *cur; cur >= p->tok->buf; c = *--cur) { - if (c == '\'' || c == '"') { + for (const char *c = cur; c >= p->tok->buf; c--) { + if (*c == '\'' || *c == '"') { return 1; } } From ba6fd87e41dceb01dcdacc57c722aca12cde42a9 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Sat, 6 Jun 2020 07:21:40 +0300 Subject: [PATCH 11/19] Refactor scripts in Tools/peg_generator/scripts (GH-20401) --- Modules/_peg_parser.c | 16 +- Tools/peg_generator/Makefile | 14 +- Tools/peg_generator/scripts/benchmark.py | 31 +++- .../peg_generator/scripts/grammar_grapher.py | 19 +- Tools/peg_generator/scripts/show_parse.py | 21 ++- .../scripts/test_parse_directory.py | 171 ++++++++---------- .../scripts/test_pypi_packages.py | 15 +- 7 files changed, 143 insertions(+), 144 deletions(-) diff --git a/Modules/_peg_parser.c b/Modules/_peg_parser.c index b66d5a83a84f641..ca2a3cf7b5fd807 100644 --- a/Modules/_peg_parser.c +++ b/Modules/_peg_parser.c @@ -80,14 +80,15 @@ _Py_compile_string(PyObject *self, PyObject *args, PyObject *kwds) PyObject * _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds) { - static char *keywords[] = {"string", "filename", "mode", "oldparser", NULL}; + static char *keywords[] = {"string", "filename", "mode", "oldparser", "ast", NULL}; char *the_string; char *filename = ""; char *mode_str = "exec"; int oldparser = 0; + int ast = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|ssp", keywords, - &the_string, &filename, &mode_str, &oldparser)) { + if (!PyArg_ParseTupleAndKeywords(args, kwds, "s|sspp", keywords, + &the_string, &filename, &mode_str, &oldparser, &ast)) { return NULL; } @@ -110,7 +111,14 @@ _Py_parse_string(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyObject *result = PyAST_mod2obj(mod); + PyObject *result; + if (ast) { + result = PyAST_mod2obj(mod); + } + else { + Py_INCREF(Py_None); + result = Py_None; + } PyArena_Free(arena); return result; } diff --git a/Tools/peg_generator/Makefile b/Tools/peg_generator/Makefile index e7a190c1bcd13d8..fb727c048b3118c 100644 --- a/Tools/peg_generator/Makefile +++ b/Tools/peg_generator/Makefile @@ -70,23 +70,21 @@ stats: peg_extension/parse.c data/xxl.py time: time_compile time_compile: venv data/xxl.py - $(VENVPYTHON) scripts/benchmark.py --parser=pegen --target=xxl compile + $(VENVPYTHON) scripts/benchmark.py --parser=new --target=xxl compile time_parse: venv data/xxl.py - $(VENVPYTHON) scripts/benchmark.py --parser=pegen --target=xxl parse + $(VENVPYTHON) scripts/benchmark.py --parser=new --target=xxl parse time_old: time_old_compile time_old_compile: venv data/xxl.py - $(VENVPYTHON) scripts/benchmark.py --parser=cpython --target=xxl compile + $(VENVPYTHON) scripts/benchmark.py --parser=old --target=xxl compile time_old_parse: venv data/xxl.py - $(VENVPYTHON) scripts/benchmark.py --parser=cpython --target=xxl parse + $(VENVPYTHON) scripts/benchmark.py --parser=old --target=xxl parse time_peg_dir: venv $(VENVPYTHON) scripts/test_parse_directory.py \ - --grammar-file $(GRAMMAR) \ - --tokens-file $(TOKENS) \ -d $(TESTDIR) \ $(TESTFLAGS) \ --exclude "*/failset/*" \ @@ -95,12 +93,8 @@ time_peg_dir: venv time_stdlib: $(CPYTHON) venv $(VENVPYTHON) scripts/test_parse_directory.py \ - --grammar-file $(GRAMMAR) \ - --tokens-file $(TOKENS) \ -d $(CPYTHON) \ $(TESTFLAGS) \ - --exclude "*/test2to3/*" \ - --exclude "*/test2to3/**/*" \ --exclude "*/bad*" \ --exclude "*/lib2to3/tests/data/*" diff --git a/Tools/peg_generator/scripts/benchmark.py b/Tools/peg_generator/scripts/benchmark.py index 71512c22a355b32..af356bed78391d5 100644 --- a/Tools/peg_generator/scripts/benchmark.py +++ b/Tools/peg_generator/scripts/benchmark.py @@ -24,7 +24,7 @@ argparser.add_argument( "--parser", action="store", - choices=["pegen", "cpython"], + choices=["new", "old"], default="pegen", help="Which parser to benchmark (default is pegen)", ) @@ -40,7 +40,12 @@ command_compile = subcommands.add_parser( "compile", help="Benchmark parsing and compiling to bytecode" ) -command_parse = subcommands.add_parser("parse", help="Benchmark parsing and generating an ast.AST") +command_parse = subcommands.add_parser( + "parse", help="Benchmark parsing and generating an ast.AST" +) +command_notree = subcommands.add_parser( + "notree", help="Benchmark parsing and dumping the tree" +) def benchmark(func): @@ -62,7 +67,7 @@ def wrapper(*args): @benchmark def time_compile(source, parser): - if parser == "cpython": + if parser == "old": return _peg_parser.compile_string( source, oldparser=True, @@ -73,32 +78,40 @@ def time_compile(source, parser): @benchmark def time_parse(source, parser): - if parser == "cpython": + if parser == "old": return _peg_parser.parse_string(source, oldparser=True) else: return _peg_parser.parse_string(source) +@benchmark +def time_notree(source, parser): + if parser == "old": + return _peg_parser.parse_string(source, oldparser=True, ast=False) + else: + return _peg_parser.parse_string(source, ast=False) + + def run_benchmark_xxl(subcommand, parser, source): if subcommand == "compile": time_compile(source, parser) elif subcommand == "parse": time_parse(source, parser) + elif subcommand == "notree": + time_notree(source, parser) def run_benchmark_stdlib(subcommand, parser): + modes = {"compile": 2, "parse": 1, "notree": 0} for _ in range(3): parse_directory( "../../Lib", - "../../Grammar/python.gram", - "../../Grammar/Tokens", verbose=False, excluded_files=["*/bad*", "*/lib2to3/tests/data/*",], - skip_actions=False, tree_arg=0, short=True, - mode=2 if subcommand == "compile" else 1, - parser=parser, + mode=modes[subcommand], + oldparser=(parser == "old"), ) diff --git a/Tools/peg_generator/scripts/grammar_grapher.py b/Tools/peg_generator/scripts/grammar_grapher.py index 3aa25466c70d476..4afdbce8f966f2c 100755 --- a/Tools/peg_generator/scripts/grammar_grapher.py +++ b/Tools/peg_generator/scripts/grammar_grapher.py @@ -42,6 +42,13 @@ ) argparser = argparse.ArgumentParser(prog="graph_grammar", description="Graph a grammar tree",) +argparser.add_argument( + "-s", + "--start", + choices=["exec", "eval", "single"], + default="exec", + help="Choose the grammar's start rule (exec, eval or single)", +) argparser.add_argument("grammar_file", help="The grammar file to graph") @@ -91,19 +98,15 @@ def main() -> None: references[name] = set(references_for_item(rule)) # Flatten the start node if has only a single reference - root_node = "start" - if start := references["start"]: - if len(start) == 1: - root_node = list(start)[0] - del references["start"] + root_node = {"exec": "file", "eval": "eval", "single": "interactive"}[args.start] print("digraph g1 {") print('\toverlap="scale";') # Force twopi to scale the graph to avoid overlaps print(f'\troot="{root_node}";') - print(f"\t{root_node} [color=green, shape=circle]") + print(f"\t{root_node} [color=green, shape=circle];") for name, refs in references.items(): - if refs: # Ignore empty sets - print(f"\t{name} -> {','.join(refs)};") + for ref in refs: + print(f"\t{name} -> {ref};") print("}") diff --git a/Tools/peg_generator/scripts/show_parse.py b/Tools/peg_generator/scripts/show_parse.py index 1c1996f40f74e01..b4ee5a1b357f74e 100755 --- a/Tools/peg_generator/scripts/show_parse.py +++ b/Tools/peg_generator/scripts/show_parse.py @@ -41,7 +41,13 @@ parser.add_argument( "-d", "--diff", action="store_true", help="show diff between grammar and ast (requires -g)" ) -parser.add_argument("-g", "--grammar-file", help="grammar to use (default: use the ast module)") +parser.add_argument( + "-p", + "--parser", + choices=["new", "old"], + default="new", + help="choose the parser to use" +) parser.add_argument( "-m", "--multiline", @@ -84,19 +90,18 @@ def print_parse(source: str, verbose: bool = False) -> None: def main() -> None: args = parser.parse_args() - if args.diff and not args.grammar_file: - parser.error("-d/--diff requires -g/--grammar-file") + new_parser = args.parser == "new" if args.multiline: sep = "\n" else: sep = " " program = sep.join(args.program) - if args.grammar_file: + if new_parser: tree = _peg_parser.parse_string(program) if args.diff: - a = tree - b = _peg_parser.parse_string(program, oldparser=True) + a = _peg_parser.parse_string(program, oldparser=True) + b = tree diff = diff_trees(a, b, args.verbose) if diff: for line in diff: @@ -104,11 +109,11 @@ def main() -> None: else: print("# Trees are the same") else: - print(f"# Parsed using {args.grammar_file}") + print("# Parsed using the new parser") print(format_tree(tree, args.verbose)) else: tree = _peg_parser.parse_string(program, oldparser=True) - print("# Parse using the old parser") + print("# Parsed using the old parser") print(format_tree(tree, args.verbose)) diff --git a/Tools/peg_generator/scripts/test_parse_directory.py b/Tools/peg_generator/scripts/test_parse_directory.py index e88afe1539ce101..63204ce9dc1934d 100755 --- a/Tools/peg_generator/scripts/test_parse_directory.py +++ b/Tools/peg_generator/scripts/test_parse_directory.py @@ -11,7 +11,7 @@ from glob import glob from pathlib import PurePath -from typing import List, Optional, Any +from typing import List, Optional, Any, Tuple sys.path.insert(0, os.getcwd()) from pegen.ast_dump import ast_dump @@ -22,13 +22,15 @@ FAIL = "\033[91m" ENDC = "\033[0m" +COMPILE = 2 +PARSE = 1 +NOTREE = 0 + argparser = argparse.ArgumentParser( prog="test_parse_directory", description="Helper program to test directories or files for pegen", ) argparser.add_argument("-d", "--directory", help="Directory path containing files to test") -argparser.add_argument("--grammar-file", help="Grammar file path") -argparser.add_argument("--tokens-file", help="Tokens file path") argparser.add_argument( "-e", "--exclude", action="append", default=[], help="Glob(s) for matching files to exclude" ) @@ -38,9 +40,6 @@ argparser.add_argument( "-v", "--verbose", action="store_true", help="Display detailed errors for failures" ) -argparser.add_argument( - "--skip-actions", action="store_true", help="Suppress code emission for rule actions", -) argparser.add_argument( "-t", "--tree", action="count", help="Compare parse tree to official AST", default=0 ) @@ -113,92 +112,35 @@ def compare_trees( return 1 -def parse_directory( - directory: str, - grammar_file: str, - tokens_file: str, - verbose: bool, - excluded_files: List[str], - skip_actions: bool, - tree_arg: int, - short: bool, - mode: int, - parser: str, -) -> int: - if parser == "cpython" and (tree_arg or mode == 0): - print("Cannot specify tree argument or mode=0 with the cpython parser.", file=sys.stderr) - return 1 - - if not directory: - print("You must specify a directory of files to test.", file=sys.stderr) - return 1 - - if grammar_file and tokens_file: - if not os.path.exists(grammar_file): - print(f"The specified grammar file, {grammar_file}, does not exist.", file=sys.stderr) - return 1 +def parse_file(source: str, file: str, mode: int, oldparser: bool) -> Tuple[Any, float]: + t0 = time.time() + if mode == COMPILE: + result = _peg_parser.compile_string( + source, + filename=file, + oldparser=oldparser, + ) else: - print( - "A grammar file or a tokens file was not provided - attempting to use existing parser from stdlib...\n" + result = _peg_parser.parse_string( + source, + filename=file, + oldparser=oldparser, + ast=(mode == PARSE), ) + t1 = time.time() + return result, t1 - t0 - if tree_arg: - assert mode == 1, "Mode should be 1 (parse), when comparing the generated trees" - # For a given directory, traverse files and attempt to parse each one - # - Output success/failure for each file - errors = 0 - files = [] - trees = {} # Trees to compare (after everything else is done) - total_seconds = 0 +def is_parsing_failure(source: str) -> bool: + try: + _peg_parser.parse_string(source, mode="exec", oldparser=True) + except SyntaxError: + return False + return True - for file in sorted(glob(f"{directory}/**/*.py", recursive=True)): - # Only attempt to parse Python files and files that are not excluded - should_exclude_file = False - for pattern in excluded_files: - if PurePath(file).match(pattern): - should_exclude_file = True - break - - if not should_exclude_file: - with tokenize.open(file) as f: - source = f.read() - try: - t0 = time.time() - if mode == 2: - result = _peg_parser.compile_string( - source, - filename=file, - oldparser=parser == "cpython", - ) - else: - result = _peg_parser.parse_string( - source, - filename=file, - oldparser=parser == "cpython" - ) - t1 = time.time() - total_seconds += (t1 - t0) - if tree_arg: - trees[file] = result - if not short: - report_status(succeeded=True, file=file, verbose=verbose) - except Exception as error: - try: - _peg_parser.parse_string(source, mode="exec", oldparser=True) - except Exception: - if not short: - print(f"File {file} cannot be parsed by either pegen or the ast module.") - else: - report_status( - succeeded=False, file=file, verbose=verbose, error=error, short=short - ) - errors += 1 - files.append(file) - t1 = time.time() +def generate_time_stats(files, total_seconds) -> None: total_files = len(files) - total_bytes = 0 total_lines = 0 for file in files: @@ -217,6 +159,57 @@ def parse_directory( f"or {total_bytes / total_seconds :,.0f} bytes/sec.", ) + +def parse_directory( + directory: str, + verbose: bool, + excluded_files: List[str], + tree_arg: int, + short: bool, + mode: int, + oldparser: bool, +) -> int: + if tree_arg: + assert mode == PARSE, "Mode should be 1 (parse), when comparing the generated trees" + + if oldparser and tree_arg: + print("Cannot specify tree argument with the cpython parser.", file=sys.stderr) + return 1 + + # For a given directory, traverse files and attempt to parse each one + # - Output success/failure for each file + errors = 0 + files = [] + trees = {} # Trees to compare (after everything else is done) + total_seconds = 0 + + for file in sorted(glob(f"{directory}/**/*.py", recursive=True)): + # Only attempt to parse Python files and files that are not excluded + if any(PurePath(file).match(pattern) for pattern in excluded_files): + continue + + with tokenize.open(file) as f: + source = f.read() + + try: + result, dt = parse_file(source, file, mode, oldparser) + total_seconds += dt + if tree_arg: + trees[file] = result + report_status(succeeded=True, file=file, verbose=verbose, short=short) + except SyntaxError as error: + if is_parsing_failure(source): + print(f"File {file} cannot be parsed by either parser.") + else: + report_status( + succeeded=False, file=file, verbose=verbose, error=error, short=short + ) + errors += 1 + files.append(file) + + t1 = time.time() + + generate_time_stats(files, total_seconds) if short: print_memstats() @@ -240,26 +233,20 @@ def parse_directory( def main() -> None: args = argparser.parse_args() directory = args.directory - grammar_file = args.grammar_file - tokens_file = args.tokens_file verbose = args.verbose excluded_files = args.exclude - skip_actions = args.skip_actions tree = args.tree short = args.short mode = 1 if args.tree else 2 sys.exit( parse_directory( directory, - grammar_file, - tokens_file, verbose, excluded_files, - skip_actions, tree, short, mode, - "pegen", + oldparser=False, ) ) diff --git a/Tools/peg_generator/scripts/test_pypi_packages.py b/Tools/peg_generator/scripts/test_pypi_packages.py index 98f77785cdd1c6a..f014753b3cd23e7 100755 --- a/Tools/peg_generator/scripts/test_pypi_packages.py +++ b/Tools/peg_generator/scripts/test_pypi_packages.py @@ -57,22 +57,11 @@ def find_dirname(package_name: str) -> str: def run_tests(dirname: str, tree: int) -> int: return test_parse_directory.parse_directory( dirname, - HERE / ".." / ".." / ".." / "Grammar" / "python.gram", - HERE / ".." / ".." / ".." / "Grammar" / "Tokens", verbose=False, - excluded_files=[ - "*/failset/*", - "*/failset/**", - "*/failset/**/*", - "*/test2to3/*", - "*/test2to3/**/*", - "*/bad*", - "*/lib2to3/tests/data/*", - ], - skip_actions=False, + excluded_files=[], tree_arg=tree, short=True, - mode=1, + mode=1 if tree else 0, parser="pegen", ) From 5552850f8e6ad6bf610c2633c74ed42dacc81b46 Mon Sep 17 00:00:00 2001 From: Ammar Askar Date: Sat, 6 Jun 2020 11:21:46 +0000 Subject: [PATCH 12/19] [workflow] Use gcc problem matcher for Ubuntu action build (GH-18567) --- .github/problem-matchers/gcc.json | 18 ++++++++++++++++++ .github/workflows/build.yml | 2 ++ 2 files changed, 20 insertions(+) create mode 100644 .github/problem-matchers/gcc.json diff --git a/.github/problem-matchers/gcc.json b/.github/problem-matchers/gcc.json new file mode 100644 index 000000000000000..bd5ab6c00a76085 --- /dev/null +++ b/.github/problem-matchers/gcc.json @@ -0,0 +1,18 @@ +{ + "__comment": "Taken from vscode-cpptools's Extension/package.json gcc rule", + "problemMatcher": [ + { + "owner": "gcc-problem-matcher", + "pattern": [ + { + "regexp": "^(.*):(\\d+):(\\d+):\\s+(?:fatal\\s+)?(warning|error):\\s+(.*)$", + "file": 1, + "line": 2, + "column": 3, + "severity": 4, + "message": 5 + } + ] + } + ] +} \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6bb52cb6a5daa5c..5649a6670e75f5c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -87,6 +87,8 @@ jobs: OPENSSL_VER: 1.1.1f steps: - uses: actions/checkout@v2 + - name: Register gcc problem matcher + run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: 'Restore OpenSSL build' From 68874a8502da440a1dc4746cf73262648b870aee Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Sat, 6 Jun 2020 15:44:16 +0300 Subject: [PATCH 13/19] bpo-40870: Invalidate usage of some constants with ast.Name (GH-20649) --- Lib/test/test_ast.py | 7 ++++++ .../2020-06-05-12-48-28.bpo-40870.9cd2sk.rst | 2 ++ Python/ast.c | 22 +++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-06-05-12-48-28.bpo-40870.9cd2sk.rst diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index a3b366ec35da125..78e4a5653d4efdf 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -668,6 +668,13 @@ def test_issue40614_feature_version(self): with self.assertRaises(SyntaxError): ast.parse('f"{x=}"', feature_version=(3, 7)) + def test_constant_as_name(self): + for constant in "True", "False", "None": + expr = ast.Expression(ast.Name(constant, ast.Load())) + ast.fix_missing_locations(expr) + with self.assertRaisesRegex(ValueError, f"Name node can't be used with '{constant}' constant"): + compile(expr, "", "eval") + class ASTHelpers_Test(unittest.TestCase): maxDiff = None diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-06-05-12-48-28.bpo-40870.9cd2sk.rst b/Misc/NEWS.d/next/Core and Builtins/2020-06-05-12-48-28.bpo-40870.9cd2sk.rst new file mode 100644 index 000000000000000..8e943a29f337fe0 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-06-05-12-48-28.bpo-40870.9cd2sk.rst @@ -0,0 +1,2 @@ +Raise :exc:`ValueError` when validating custom AST's where the constants +``True``, ``False`` and ``None`` are used within a :class:`ast.Name` node. diff --git a/Python/ast.c b/Python/ast.c index c524b8e34e8731a..408591f32536f3b 100644 --- a/Python/ast.c +++ b/Python/ast.c @@ -21,6 +21,25 @@ static int validate_nonempty_seq(asdl_seq *, const char *, const char *); static int validate_stmt(stmt_ty); static int validate_expr(expr_ty, expr_context_ty); +static int +validate_name(PyObject *name) +{ + assert(PyUnicode_Check(name)); + static const char * const forbidden[] = { + "None", + "True", + "False", + NULL + }; + for (int i = 0; forbidden[i] != NULL; i++) { + if (_PyUnicode_EqualToASCIIString(name, forbidden[i])) { + PyErr_Format(PyExc_ValueError, "Name node can't be used with '%s' constant", forbidden[i]); + return 0; + } + } + return 1; +} + static int validate_comprehension(asdl_seq *gens) { @@ -173,6 +192,9 @@ validate_expr(expr_ty exp, expr_context_ty ctx) actual_ctx = exp->v.Starred.ctx; break; case Name_kind: + if (!validate_name(exp->v.Name.id)) { + return 0; + } actual_ctx = exp->v.Name.ctx; break; case List_kind: From f7c4e236429606e1c982cacf24e10fc86ef4462f Mon Sep 17 00:00:00 2001 From: scoder Date: Sat, 6 Jun 2020 21:35:10 +0200 Subject: [PATCH 14/19] bpo-40724: Support setting buffer slots from type specs (GH-20648) This is not part of the limited API but makes the buffer slots available for type specs. --- Include/typeslots.h | 5 ++ Lib/test/test_capi.py | 5 ++ .../2020-06-04-08-01-23.bpo-40724.qIIdSi.rst | 1 + Modules/_testcapimodule.c | 47 +++++++++++++++++++ Objects/typeslots.inc | 4 +- 5 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2020-06-04-08-01-23.bpo-40724.qIIdSi.rst diff --git a/Include/typeslots.h b/Include/typeslots.h index 0ce6a377dcfbd69..64f6fff51444937 100644 --- a/Include/typeslots.h +++ b/Include/typeslots.h @@ -1,7 +1,12 @@ /* Do not renumber the file; these numbers are part of the stable ABI. */ +#if defined(Py_LIMITED_API) /* Disabled, see #10181 */ #undef Py_bf_getbuffer #undef Py_bf_releasebuffer +#else +#define Py_bf_getbuffer 1 +#define Py_bf_releasebuffer 2 +#endif #define Py_mp_ass_subscript 3 #define Py_mp_length 4 #define Py_mp_subscript 5 diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 5b8b9f6a86f4b27..73e167a0b05a533 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -477,6 +477,11 @@ def test_heaptype_with_weakref(self): self.assertEqual(ref(), inst) self.assertEqual(inst.weakreflist, ref) + def test_heaptype_with_buffer(self): + inst = _testcapi.HeapCTypeWithBuffer() + b = bytes(inst) + self.assertEqual(b, b"1234") + def test_c_subclass_of_heap_ctype_with_tpdealloc_decrefs_once(self): subclass_instance = _testcapi.HeapCTypeSubclass() type_refcnt = sys.getrefcount(_testcapi.HeapCTypeSubclass) diff --git a/Misc/NEWS.d/next/C API/2020-06-04-08-01-23.bpo-40724.qIIdSi.rst b/Misc/NEWS.d/next/C API/2020-06-04-08-01-23.bpo-40724.qIIdSi.rst new file mode 100644 index 000000000000000..82793dbf7ad5f12 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2020-06-04-08-01-23.bpo-40724.qIIdSi.rst @@ -0,0 +1 @@ +Allow defining buffer slots in type specs. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 101d54932d913df..d6a90b807d0267e 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -6298,6 +6298,47 @@ static PyType_Spec HeapCTypeSubclass_spec = { HeapCTypeSubclass_slots }; +PyDoc_STRVAR(heapctypewithbuffer__doc__, +"Heap type with buffer support.\n\n" +"The buffer is set to [b'1', b'2', b'3', b'4']"); + +typedef struct { + HeapCTypeObject base; + char buffer[4]; +} HeapCTypeWithBufferObject; + +static int +heapctypewithbuffer_getbuffer(HeapCTypeWithBufferObject *self, Py_buffer *view, int flags) +{ + self->buffer[0] = '1'; + self->buffer[1] = '2'; + self->buffer[2] = '3'; + self->buffer[3] = '4'; + return PyBuffer_FillInfo( + view, (PyObject*)self, (void *)self->buffer, 4, 1, flags); +} + +static int +heapctypewithbuffer_releasebuffer(HeapCTypeWithBufferObject *self, Py_buffer *view) +{ + assert(view->obj == (void*) self); +} + +static PyType_Slot HeapCTypeWithBuffer_slots[] = { + {Py_bf_getbuffer, heapctypewithbuffer_getbuffer}, + {Py_bf_releasebuffer, heapctypewithbuffer_releasebuffer}, + {Py_tp_doc, (char*)heapctypewithbuffer__doc__}, + {0, 0}, +}; + +static PyType_Spec HeapCTypeWithBuffer_spec = { + "_testcapi.HeapCTypeWithBuffer", + sizeof(HeapCTypeWithBufferObject), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + HeapCTypeWithBuffer_slots +}; + PyDoc_STRVAR(heapctypesubclasswithfinalizer__doc__, "Subclass of HeapCType with a finalizer that reassigns __class__.\n\n" "__class__ is set to plain HeapCTypeSubclass during finalization.\n" @@ -6775,6 +6816,12 @@ PyInit__testcapi(void) } PyModule_AddObject(m, "HeapCTypeWithWeakref", HeapCTypeWithWeakref); + PyObject *HeapCTypeWithBuffer = PyType_FromSpec(&HeapCTypeWithBuffer_spec); + if (HeapCTypeWithBuffer == NULL) { + return NULL; + } + PyModule_AddObject(m, "HeapCTypeWithBuffer", HeapCTypeWithBuffer); + PyObject *subclass_with_finalizer_bases = PyTuple_Pack(1, HeapCTypeSubclass); if (subclass_with_finalizer_bases == NULL) { return NULL; diff --git a/Objects/typeslots.inc b/Objects/typeslots.inc index dc750cc0c419759..ffc9bb2e1c7710b 100644 --- a/Objects/typeslots.inc +++ b/Objects/typeslots.inc @@ -1,6 +1,6 @@ /* Generated by typeslots.py */ -0, -0, +offsetof(PyHeapTypeObject, as_buffer.bf_getbuffer), +offsetof(PyHeapTypeObject, as_buffer.bf_releasebuffer), offsetof(PyHeapTypeObject, as_mapping.mp_ass_subscript), offsetof(PyHeapTypeObject, as_mapping.mp_length), offsetof(PyHeapTypeObject, as_mapping.mp_subscript), From 0e96c419d7287c3c7f155c9f2de3c61020386256 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sat, 6 Jun 2020 12:42:54 -0700 Subject: [PATCH 15/19] Update comments to reflect the current API (GH-20682) --- Lib/collections/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index 2acf67289f2250f..03393f35b11c507 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -760,10 +760,12 @@ def __repr__(self): # set(cp - cq) == sp - sq # set(cp | cq) == sp | sq # set(cp & cq) == sp & sq - # cp.isequal(cq) == (sp == sq) - # cp.issubset(cq) == sp.issubset(sq) - # cp.issuperset(cq) == sp.issuperset(sq) - # cp.isdisjoint(cq) == sp.isdisjoint(sq) + # (cp == cq) == (sp == sq) + # (cp != cq) == (sp != sq) + # (cp <= cq) == (sp <= sq) + # (cp < cq) == (sp < sq) + # (cp >= cq) == (sp >= sq) + # (cp > cq) == (sp > sq) def __add__(self, other): '''Add counts from two counters. From b8867e5d5aca33511942632b5f4e359b9245b2fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Lapeyre?= Date: Sun, 7 Jun 2020 09:05:33 +0200 Subject: [PATCH 16/19] Fix return type of test helper function heapctypewithbuffer_releasebuffer() (GH-20685) --- Modules/_testcapimodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index d6a90b807d0267e..e0457ae5dfa55d4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -6318,7 +6318,7 @@ heapctypewithbuffer_getbuffer(HeapCTypeWithBufferObject *self, Py_buffer *view, view, (PyObject*)self, (void *)self->buffer, 4, 1, flags); } -static int +static void heapctypewithbuffer_releasebuffer(HeapCTypeWithBufferObject *self, Py_buffer *view) { assert(view->obj == (void*) self); From 47a23fc63fa5df2da8dbc542e78e521d4a7f10c9 Mon Sep 17 00:00:00 2001 From: Hai Shi Date: Sun, 7 Jun 2020 20:05:36 +0800 Subject: [PATCH 17/19] bpo-40898: Remove redundant if statements in tp_traverse (GH-20692) --- Modules/_functoolsmodule.c | 3 +-- Modules/_io/_iomodule.c | 4 +--- Modules/itertoolsmodule.c | 3 +-- Modules/overlapped.c | 8 ++------ 4 files changed, 5 insertions(+), 13 deletions(-) diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index d158d3bae157b24..f1ee23f294fa351 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -484,8 +484,7 @@ static int keyobject_traverse(keyobject *ko, visitproc visit, void *arg) { Py_VISIT(ko->cmp); - if (ko->object) - Py_VISIT(ko->object); + Py_VISIT(ko->object); return 0; } diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index a55e5cad6a392b5..e430352a48e2117 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -623,9 +623,7 @@ iomodule_traverse(PyObject *mod, visitproc visit, void *arg) { _PyIO_State *state = get_io_state(mod); if (!state->initialized) return 0; - if (state->locale_module != NULL) { - Py_VISIT(state->locale_module); - } + Py_VISIT(state->locale_module); Py_VISIT(state->unsupported_operation); return 0; } diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 18fcebdf25b465f..3f2f7165b171b57 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -999,8 +999,7 @@ cycle_dealloc(cycleobject *lz) static int cycle_traverse(cycleobject *lz, visitproc visit, void *arg) { - if (lz->it) - Py_VISIT(lz->it); + Py_VISIT(lz->it); Py_VISIT(lz->saved); return 0; } diff --git a/Modules/overlapped.c b/Modules/overlapped.c index df6282cba819ba4..eed8fbf0393002a 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -1504,12 +1504,8 @@ Overlapped_traverse(OverlappedObject *self, visitproc visit, void *arg) } break; case TYPE_READ_FROM: - if(self->read_from.result) { - Py_VISIT(self->read_from.result); - } - if(self->read_from.allocated_buffer) { - Py_VISIT(self->read_from.allocated_buffer); - } + Py_VISIT(self->read_from.result); + Py_VISIT(self->read_from.allocated_buffer); } return 0; } From 2efe18bf277dd0f38a1d248ae6bdd30947c26880 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 7 Jun 2020 10:57:45 -0400 Subject: [PATCH 18/19] bpo-39791: Support file systems that cannot support non-ascii filenames (skipping tests in that case). (#20681) --- Lib/test/test_importlib/fixtures.py | 11 +++++++++++ Lib/test/test_importlib/test_main.py | 9 +++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py index b25febb7fe75641..2e55d14b9aab971 100644 --- a/Lib/test/test_importlib/fixtures.py +++ b/Lib/test/test_importlib/fixtures.py @@ -210,6 +210,17 @@ def build_files(file_defs, prefix=pathlib.Path()): f.write(DALS(contents)) +class FileBuilder: + def unicode_filename(self): + try: + import test.support + except ImportError: + # outside CPython, hard-code a unicode snowman + return 'β˜ƒ' + return test.support.FS_NONASCII or \ + self.skip("File system does not support non-ascii.") + + def DALS(str): "Dedent and left-strip" return textwrap.dedent(str).lstrip() diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py index 7b18c3de16eea08..91e501a2eb7cdc4 100644 --- a/Lib/test/test_importlib/test_main.py +++ b/Lib/test/test_importlib/test_main.py @@ -254,11 +254,16 @@ def test_attr(self): assert self.ep.attr is None -class FileSystem(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): +class FileSystem( + fixtures.OnSysPath, fixtures.SiteDir, fixtures.FileBuilder, + unittest.TestCase): def test_unicode_dir_on_sys_path(self): """ Ensure a Unicode subdirectory of a directory on sys.path does not crash. """ - fixtures.build_files({'β˜ƒ': {}}, prefix=self.site_dir) + fixtures.build_files( + {self.unicode_filename(): {}}, + prefix=self.site_dir, + ) list(distributions()) From 76333081aeccfa866d909b49793eeb70a3d6737e Mon Sep 17 00:00:00 2001 From: Francis Laniel Date: Mon, 5 Oct 2020 13:53:09 +0200 Subject: [PATCH 19/19] Update documentations of strftime function for date, time and datetime. The documentation is now clearer than before. This commit should help pandas-dev/pandas#36745. --- Modules/_datetimemodule.c | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index acdde83dc845a5d..0dcb63cef02c068 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -3510,8 +3510,13 @@ static PyMethodDef date_methods[] = { {"ctime", (PyCFunction)date_ctime, METH_NOARGS, PyDoc_STR("Return ctime() style string.")}, - {"strftime", (PyCFunction)(void(*)(void))date_strftime, METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("format -> strftime() style string.")}, + {"strftime", (PyCFunction)(void(*)(void))date_strftime, METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("strftime(format) -> style\n" + "\n" + "Return a string representing the date, controlled by an " + "explicit format string.\n" + "Format codes referring to hours, minutes or seconds will see " + "0 values.")}, {"__format__", (PyCFunction)date_format, METH_VARARGS, PyDoc_STR("Formats self with strftime.")}, @@ -4676,7 +4681,10 @@ static PyMethodDef time_methods[] = { "timespec specifies what components of the time to include.\n")}, {"strftime", (PyCFunction)(void(*)(void))time_strftime, METH_VARARGS | METH_KEYWORDS, - PyDoc_STR("format -> strftime() style string.")}, + PyDoc_STR("strftime(format) -> style\n" + "\n" + "Return a string representing the date, controlled by an " + "explicit format string.")}, {"__format__", (PyCFunction)date_format, METH_VARARGS, PyDoc_STR("Formats self with strftime.")}, @@ -6366,6 +6374,12 @@ static PyMethodDef datetime_methods[] = { {"ctime", (PyCFunction)datetime_ctime, METH_NOARGS, PyDoc_STR("Return ctime() style string.")}, + {"strftime", (PyCFunction)(void(*)(void))date_strftime, METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("strftime(format) -> style\n" + "\n" + "Return a string representing the date, controlled by an " + "explicit format string.")}, + {"timetuple", (PyCFunction)datetime_timetuple, METH_NOARGS, PyDoc_STR("Return time tuple, compatible with time.localtime().")},