commit
697ea796dc
GPG Key ID:
1EC08A25C9DB2503
@ -0,0 +1 @@
|
||||
SOURCES/numexpr-2.8.5.tar.gz
|
||||
@ -0,0 +1 @@
|
||||
fbd681371c05653c3595149bf25102d5b806b2b6 SOURCES/numexpr-2.8.5.tar.gz
|
||||
@ -0,0 +1,254 @@
|
||||
From be21651a1fada038b8ca00938d063fbb3336b989 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= <zbyszek@in.waw.pl>
|
||||
Date: Wed, 16 Aug 2023 17:23:26 +0200
|
||||
Subject: [PATCH 2/7] Revert "Make more difficult sanitize of the expression
|
||||
string before eval"
|
||||
|
||||
This reverts commit 00b035c78ca5ac209b58b56b5dcc99596cac423c.
|
||||
---
|
||||
ANNOUNCE.rst | 23 ++---------------------
|
||||
RELEASE_NOTES.rst | 19 +------------------
|
||||
doc/user_guide.rst | 27 ++++++++++-----------------
|
||||
numexpr/necompiler.py | 27 ++++++++-------------------
|
||||
numexpr/tests/test_numexpr.py | 18 ++----------------
|
||||
5 files changed, 23 insertions(+), 91 deletions(-)
|
||||
|
||||
diff --git a/ANNOUNCE.rst b/ANNOUNCE.rst
|
||||
index 4e9070f2cc..f038df4f44 100644
|
||||
--- a/ANNOUNCE.rst
|
||||
+++ b/ANNOUNCE.rst
|
||||
@@ -4,10 +4,7 @@ Announcing NumExpr 2.8.5
|
||||
|
||||
Hi everyone,
|
||||
|
||||
-In 2.8.5 we have added a new function, `validate` which checks an expression `ex`
|
||||
-for validity, for usage where the program is parsing a user input. There are also
|
||||
-consequences for this sort of usage, since `eval(ex)` is called, and as such we
|
||||
-do some string sanitization as described below.
|
||||
+**Under development.**
|
||||
|
||||
Project documentation is available at:
|
||||
|
||||
@@ -16,23 +13,7 @@ http://numexpr.readthedocs.io/
|
||||
Changes from 2.8.4 to 2.8.5
|
||||
---------------------------
|
||||
|
||||
-* A `validate` function has been added. This function checks the inputs, returning
|
||||
- `None` on success or raising an exception on invalid inputs. This function was
|
||||
- added as numerous projects seem to be using NumExpr for parsing user inputs.
|
||||
- `re_evaluate` may be called directly following `validate`.
|
||||
-* As an addendum to the use of NumExpr for parsing user inputs, is that NumExpr
|
||||
- calls `eval` on the inputs. A regular expression is now applied to help sanitize
|
||||
- the input expression string, forbidding '__', ':', and ';'. Attribute access
|
||||
- is also banned except for '.r' for real and '.i' for imag.
|
||||
-* Thanks to timbrist for a fix to behavior of NumExpr with integers to negative
|
||||
- powers. NumExpr was pre-checking integer powers for negative values, which
|
||||
- was both inefficient and causing parsing errors in some situations. Now NumExpr
|
||||
- will simply return 0 as a result for such cases. While NumExpr generally tries
|
||||
- to follow NumPy behavior, performance is also critical.
|
||||
-* Thanks to peadar for some fixes to how NumExpr launches threads for embedded
|
||||
- applications.
|
||||
-* Thanks to de11n for making parsing of the `site.cfg` for MKL consistent among
|
||||
- all shared platforms.
|
||||
+**Under development.**
|
||||
|
||||
|
||||
What's Numexpr?
|
||||
diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst
|
||||
index 9cf5d3977b..4929a42e12 100644
|
||||
--- a/RELEASE_NOTES.rst
|
||||
+++ b/RELEASE_NOTES.rst
|
||||
@@ -5,24 +5,7 @@ Release notes for NumExpr 2.8 series
|
||||
Changes from 2.8.4 to 2.8.5
|
||||
---------------------------
|
||||
|
||||
-* A `validate` function has been added. This function checks the inputs, returning
|
||||
- `None` on success or raising an exception on invalid inputs. This function was
|
||||
- added as numerous projects seem to be using NumExpr for parsing user inputs.
|
||||
- `re_evaluate` may be called directly following `validate`.
|
||||
-* As an addendum to the use of NumExpr for parsing user inputs, is that NumExpr
|
||||
- calls `eval` on the inputs. A regular expression is now applied to help sanitize
|
||||
- the input expression string, forbidding '__', ':', and ';'. Attribute access
|
||||
- is also banned except for '.r' for real and '.i' for imag.
|
||||
-* Thanks to timbrist for a fix to behavior of NumExpr with integers to negative
|
||||
- powers. NumExpr was pre-checking integer powers for negative values, which
|
||||
- was both inefficient and causing parsing errors in some situations. Now NumExpr
|
||||
- will simply return 0 as a result for such cases. While NumExpr generally tries
|
||||
- to follow NumPy behavior, performance is also critical.
|
||||
-* Thanks to peadar for some fixes to how NumExpr launches threads for embedded
|
||||
- applications.
|
||||
-* Thanks to de11n for making parsing of the `site.cfg` for MKL consistent among
|
||||
- all shared platforms.
|
||||
-
|
||||
+**Under development.**
|
||||
|
||||
Changes from 2.8.3 to 2.8.4
|
||||
---------------------------
|
||||
diff --git a/doc/user_guide.rst b/doc/user_guide.rst
|
||||
index 3a3cf63d9c..74306eb658 100644
|
||||
--- a/doc/user_guide.rst
|
||||
+++ b/doc/user_guide.rst
|
||||
@@ -1,7 +1,7 @@
|
||||
-NumExpr 2.8 User Guide
|
||||
+NumExpr 2.0 User Guide
|
||||
======================
|
||||
|
||||
-The NumExpr package supplies routines for the fast evaluation of
|
||||
+The :code:`numexpr` package supplies routines for the fast evaluation of
|
||||
array expressions elementwise by using a vector-based virtual
|
||||
machine.
|
||||
|
||||
@@ -11,33 +11,23 @@ Using it is simple::
|
||||
>>> import numexpr as ne
|
||||
>>> a = np.arange(10)
|
||||
>>> b = np.arange(0, 20, 2)
|
||||
- >>> c = ne.evaluate('2*a + 3*b')
|
||||
+ >>> c = ne.evaluate("2*a+3*b")
|
||||
>>> c
|
||||
array([ 0, 8, 16, 24, 32, 40, 48, 56, 64, 72])
|
||||
|
||||
|
||||
-It is also possible to use NumExpr to validate an expression::
|
||||
-
|
||||
- >>> ne.validate('2*a + 3*b')
|
||||
-
|
||||
-which returns `None` on success or raises an exception on invalid inputs.
|
||||
-
|
||||
-and it can also re_evaluate an expression::
|
||||
-
|
||||
- >>> b = np.arange(0, 40, 4)
|
||||
- >>> ne.re_evaluate()
|
||||
-
|
||||
Building
|
||||
--------
|
||||
|
||||
-*NumExpr* requires Python_ 3.7 or greater, and NumPy_ 1.13 or greater. It is
|
||||
+*NumExpr* requires Python_ 2.6 or greater, and NumPy_ 1.7 or greater. It is
|
||||
built in the standard Python way:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
- $ pip install .
|
||||
+ $ python setup.py build
|
||||
+ $ python setup.py install
|
||||
|
||||
-You must have a C-compiler (i.e. MSVC Build tools on Windows and GCC on Linux) installed.
|
||||
+You must have a C-compiler (i.e. MSVC on Windows and GCC on Linux) installed.
|
||||
|
||||
Then change to a directory that is not the repository directory (e.g. `/tmp`) and
|
||||
test :code:`numexpr` with:
|
||||
@@ -278,6 +268,9 @@ General routines
|
||||
* :code:`detect_number_of_cores()`: Detects the number of cores on a system.
|
||||
|
||||
|
||||
+
|
||||
+
|
||||
+
|
||||
Intel's VML specific support routines
|
||||
-------------------------------------
|
||||
|
||||
diff --git a/numexpr/necompiler.py b/numexpr/necompiler.py
|
||||
index cbf290932b..fef886baf5 100644
|
||||
--- a/numexpr/necompiler.py
|
||||
+++ b/numexpr/necompiler.py
|
||||
@@ -260,17 +260,15 @@ class Immediate(Register):
|
||||
def __str__(self):
|
||||
return 'Immediate(%d)' % (self.node.value,)
|
||||
|
||||
-
|
||||
-_forbidden_re = re.compile('[\;[\:]|__|\.[abcdefghjklmnopqstuvwxyzA-Z_]')
|
||||
+_forbidden_re = re.compile('[\;[\:]|__')
|
||||
def stringToExpression(s, types, context):
|
||||
"""Given a string, convert it to a tree of ExpressionNode's.
|
||||
"""
|
||||
# sanitize the string for obvious attack vectors that NumExpr cannot
|
||||
# parse into its homebrew AST. This is to protect the call to `eval` below.
|
||||
- # We forbid `;`, `:`. `[` and `__`, and attribute access via '.'.
|
||||
- # We cannot ban `.real` or `.imag` however...
|
||||
- no_whitespace = re.sub(r'\s+', '', s)
|
||||
- if _forbidden_re.search(no_whitespace) is not None:
|
||||
+ # We forbid `;`, `:`. `[` and `__`
|
||||
+ # We would like to forbid `.` but it is both a reference and decimal point.
|
||||
+ if _forbidden_re.search(s) is not None:
|
||||
raise ValueError(f'Expression {s} has forbidden control characters.')
|
||||
|
||||
old_ctx = expressions._context.get_current_context()
|
||||
@@ -768,6 +766,7 @@ def getArguments(names, local_dict=None, global_dict=None, _frame_depth: int=2):
|
||||
_names_cache = CacheDict(256)
|
||||
_numexpr_cache = CacheDict(256)
|
||||
_numexpr_last = {}
|
||||
+_numexpr_sanity = set()
|
||||
evaluate_lock = threading.Lock()
|
||||
|
||||
# MAYBE: decorate this function to add attributes instead of having the
|
||||
@@ -829,13 +828,6 @@ def validate(ex: str,
|
||||
_frame_depth: int
|
||||
The calling frame depth. Unless you are a NumExpr developer you should
|
||||
not set this value.
|
||||
-
|
||||
- Note
|
||||
- ----
|
||||
- Both `validate` and by extension `evaluate` call `eval(ex)`, which is
|
||||
- potentially dangerous on unsanitized inputs. As such, NumExpr does some
|
||||
- sanitization, banning the character ':;[', the dunder '__', and attribute
|
||||
- access to all but '.r' for real and '.i' for imag access to complex numbers.
|
||||
"""
|
||||
global _numexpr_last
|
||||
|
||||
@@ -865,6 +857,8 @@ def validate(ex: str,
|
||||
kwargs = {'out': out, 'order': order, 'casting': casting,
|
||||
'ex_uses_vml': ex_uses_vml}
|
||||
_numexpr_last = dict(ex=compiled_ex, argnames=names, kwargs=kwargs)
|
||||
+ # with evaluate_lock:
|
||||
+ # return compiled_ex(*arguments, **kwargs)
|
||||
except Exception as e:
|
||||
return e
|
||||
return None
|
||||
@@ -924,12 +918,7 @@ def evaluate(ex: str,
|
||||
The calling frame depth. Unless you are a NumExpr developer you should
|
||||
not set this value.
|
||||
|
||||
- Note
|
||||
- ----
|
||||
- Both `validate` and by extension `evaluate` call `eval(ex)`, which is
|
||||
- potentially dangerous on unsanitized inputs. As such, NumExpr does some
|
||||
- sanitization, banning the character ':;[', the dunder '__', and attribute
|
||||
- access to all but '.r' for real and '.i' for imag access to complex numbers.
|
||||
+
|
||||
"""
|
||||
# We could avoid code duplication if we called validate and then re_evaluate
|
||||
# here, but they we have difficulties with the `sys.getframe(2)` call in
|
||||
diff --git a/numexpr/tests/test_numexpr.py b/numexpr/tests/test_numexpr.py
|
||||
index a9f917fccd..ebc41c8d54 100644
|
||||
--- a/numexpr/tests/test_numexpr.py
|
||||
+++ b/numexpr/tests/test_numexpr.py
|
||||
@@ -536,27 +536,13 @@ class test_evaluate(TestCase):
|
||||
|
||||
# Forbid semicolon
|
||||
try:
|
||||
- evaluate('import os;')
|
||||
+ evaluate('import os; os.cpu_count()')
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
self.fail()
|
||||
|
||||
- # Attribute access
|
||||
- try:
|
||||
- evaluate('os.cpucount()')
|
||||
- except ValueError:
|
||||
- pass
|
||||
- else:
|
||||
- self.fail()
|
||||
-
|
||||
- # But decimal point must pass
|
||||
- a = 3.0
|
||||
- evaluate('a*2.')
|
||||
- evaluate('2.+a')
|
||||
-
|
||||
-
|
||||
-
|
||||
+ # I struggle to come up with cases for our ban on `'` and `"`
|
||||
|
||||
|
||||
|
||||
@ -0,0 +1,183 @@
|
||||
From aa18c52226ddf3d5a3448702fc2b1a14b0ee3c2f Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= <zbyszek@in.waw.pl>
|
||||
Date: Wed, 16 Aug 2023 17:23:36 +0200
|
||||
Subject: [PATCH 3/7] Revert "Add in protections against call to
|
||||
`eval(expression)`"
|
||||
|
||||
This reverts commit 4b2d89cf14e75030d27629925b9998e1e91d23c7.
|
||||
---
|
||||
numexpr/necompiler.py | 26 +++++++-----------
|
||||
numexpr/tests/test_numexpr.py | 50 ++++-------------------------------
|
||||
2 files changed, 15 insertions(+), 61 deletions(-)
|
||||
|
||||
diff --git a/numexpr/necompiler.py b/numexpr/necompiler.py
|
||||
index fef886baf5..37052acadb 100644
|
||||
--- a/numexpr/necompiler.py
|
||||
+++ b/numexpr/necompiler.py
|
||||
@@ -13,7 +13,6 @@ import __future__
|
||||
import sys
|
||||
import numpy
|
||||
import threading
|
||||
-import re
|
||||
|
||||
is_cpu_amd_intel = False # DEPRECATION WARNING: WILL BE REMOVED IN FUTURE RELEASE
|
||||
from numexpr import interpreter, expressions, use_vml
|
||||
@@ -260,17 +259,10 @@ class Immediate(Register):
|
||||
def __str__(self):
|
||||
return 'Immediate(%d)' % (self.node.value,)
|
||||
|
||||
-_forbidden_re = re.compile('[\;[\:]|__')
|
||||
+
|
||||
def stringToExpression(s, types, context):
|
||||
"""Given a string, convert it to a tree of ExpressionNode's.
|
||||
"""
|
||||
- # sanitize the string for obvious attack vectors that NumExpr cannot
|
||||
- # parse into its homebrew AST. This is to protect the call to `eval` below.
|
||||
- # We forbid `;`, `:`. `[` and `__`
|
||||
- # We would like to forbid `.` but it is both a reference and decimal point.
|
||||
- if _forbidden_re.search(s) is not None:
|
||||
- raise ValueError(f'Expression {s} has forbidden control characters.')
|
||||
-
|
||||
old_ctx = expressions._context.get_current_context()
|
||||
try:
|
||||
expressions._context.set_new_context(context)
|
||||
@@ -293,10 +285,8 @@ def stringToExpression(s, types, context):
|
||||
t = types.get(name, default_type)
|
||||
names[name] = expressions.VariableNode(name, type_to_kind[t])
|
||||
names.update(expressions.functions)
|
||||
-
|
||||
# now build the expression
|
||||
ex = eval(c, names)
|
||||
-
|
||||
if expressions.isConstant(ex):
|
||||
ex = expressions.ConstantNode(ex, expressions.getKind(ex))
|
||||
elif not isinstance(ex, expressions.ExpressionNode):
|
||||
@@ -621,7 +611,9 @@ def NumExpr(ex, signature=(), **kwargs):
|
||||
|
||||
Returns a `NumExpr` object containing the compiled function.
|
||||
"""
|
||||
-
|
||||
+ # NumExpr can be called either directly by the end-user, in which case
|
||||
+ # kwargs need to be sanitized by getContext, or by evaluate,
|
||||
+ # in which case kwargs are in already sanitized.
|
||||
# In that case _frame_depth is wrong (it should be 2) but it doesn't matter
|
||||
# since it will not be used (because truediv='auto' has already been
|
||||
# translated to either True or False).
|
||||
@@ -766,7 +758,7 @@ def getArguments(names, local_dict=None, global_dict=None, _frame_depth: int=2):
|
||||
_names_cache = CacheDict(256)
|
||||
_numexpr_cache = CacheDict(256)
|
||||
_numexpr_last = {}
|
||||
-_numexpr_sanity = set()
|
||||
+
|
||||
evaluate_lock = threading.Lock()
|
||||
|
||||
# MAYBE: decorate this function to add attributes instead of having the
|
||||
@@ -869,7 +861,7 @@ def evaluate(ex: str,
|
||||
out: numpy.ndarray = None,
|
||||
order: str = 'K',
|
||||
casting: str = 'safe',
|
||||
- _frame_depth: int = 3,
|
||||
+ _frame_depth: int=3,
|
||||
**kwargs) -> numpy.ndarray:
|
||||
"""
|
||||
Evaluate a simple array expression element-wise using the virtual machine.
|
||||
@@ -917,8 +909,6 @@ def evaluate(ex: str,
|
||||
_frame_depth: int
|
||||
The calling frame depth. Unless you are a NumExpr developer you should
|
||||
not set this value.
|
||||
-
|
||||
-
|
||||
"""
|
||||
# We could avoid code duplication if we called validate and then re_evaluate
|
||||
# here, but they we have difficulties with the `sys.getframe(2)` call in
|
||||
@@ -931,6 +921,10 @@ def evaluate(ex: str,
|
||||
else:
|
||||
raise e
|
||||
|
||||
+
|
||||
+
|
||||
+
|
||||
+
|
||||
def re_evaluate(local_dict: Optional[Dict] = None,
|
||||
_frame_depth: int=2) -> numpy.ndarray:
|
||||
"""
|
||||
diff --git a/numexpr/tests/test_numexpr.py b/numexpr/tests/test_numexpr.py
|
||||
index ebc41c8d54..ccb0b6cb07 100644
|
||||
--- a/numexpr/tests/test_numexpr.py
|
||||
+++ b/numexpr/tests/test_numexpr.py
|
||||
@@ -373,9 +373,8 @@ class test_evaluate(TestCase):
|
||||
a1 = array([1., 2., 3.])
|
||||
b1 = array([4., 5., 6.])
|
||||
c1 = array([7., 8., 9.])
|
||||
- local_dict={'a': a1, 'b': b1, 'c': c1}
|
||||
- x = evaluate("2*a + 3*b*c", local_dict=local_dict)
|
||||
- x = re_evaluate(local_dict=local_dict)
|
||||
+ x = evaluate("2*a + 3*b*c", local_dict={'a': a1, 'b': b1, 'c': c1})
|
||||
+ x = re_evaluate()
|
||||
assert_array_equal(x, array([86., 124., 168.]))
|
||||
|
||||
def test_validate(self):
|
||||
@@ -401,10 +400,9 @@ class test_evaluate(TestCase):
|
||||
a1 = array([1., 2., 3.])
|
||||
b1 = array([4., 5., 6.])
|
||||
c1 = array([7., 8., 9.])
|
||||
- local_dict={'a': a1, 'b': b1, 'c': c1}
|
||||
- retval = validate("2*a + 3*b*c", local_dict=local_dict)
|
||||
+ retval = validate("2*a + 3*b*c", local_dict={'a': a1, 'b': b1, 'c': c1})
|
||||
assert(retval is None)
|
||||
- x = re_evaluate(local_dict=local_dict)
|
||||
+ x = re_evaluate()
|
||||
assert_array_equal(x, array([86., 124., 168.]))
|
||||
|
||||
# Test for issue #22
|
||||
@@ -504,49 +502,11 @@ class test_evaluate(TestCase):
|
||||
a = arange(3)
|
||||
try:
|
||||
evaluate("a < [0, 0, 0]")
|
||||
- except (ValueError, TypeError):
|
||||
+ except TypeError:
|
||||
pass
|
||||
else:
|
||||
self.fail()
|
||||
|
||||
- def test_forbidden_tokens(self):
|
||||
- # Forbid dunder
|
||||
- try:
|
||||
- evaluate('__builtins__')
|
||||
- except ValueError:
|
||||
- pass
|
||||
- else:
|
||||
- self.fail()
|
||||
-
|
||||
- # Forbid colon for lambda funcs
|
||||
- try:
|
||||
- evaluate('lambda x: x')
|
||||
- except ValueError:
|
||||
- pass
|
||||
- else:
|
||||
- self.fail()
|
||||
-
|
||||
- # Forbid indexing
|
||||
- try:
|
||||
- evaluate('locals()[]')
|
||||
- except ValueError:
|
||||
- pass
|
||||
- else:
|
||||
- self.fail()
|
||||
-
|
||||
- # Forbid semicolon
|
||||
- try:
|
||||
- evaluate('import os; os.cpu_count()')
|
||||
- except ValueError:
|
||||
- pass
|
||||
- else:
|
||||
- self.fail()
|
||||
-
|
||||
- # I struggle to come up with cases for our ban on `'` and `"`
|
||||
-
|
||||
-
|
||||
-
|
||||
-
|
||||
def test_disassemble(self):
|
||||
assert_equal(disassemble(NumExpr(
|
||||
"where(m, a, -1)", [('m', bool), ('a', float)])),
|
||||
@ -0,0 +1,80 @@
|
||||
From d3f418e5f469be66f8e974e5b53487bd8698dea4 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= <zbyszek@in.waw.pl>
|
||||
Date: Wed, 16 Aug 2023 17:23:41 +0200
|
||||
Subject: [PATCH 4/7] Revert "Adding tests for `validate` and noticed that
|
||||
`re_evaluate` tests using `local_dict` argument are flawed and do not
|
||||
actually work"
|
||||
|
||||
This reverts commit 74d597398ba3c379c196e469d40f516de110eaa5.
|
||||
---
|
||||
numexpr/__init__.py | 3 +--
|
||||
numexpr/tests/test_numexpr.py | 32 ++------------------------------
|
||||
2 files changed, 3 insertions(+), 32 deletions(-)
|
||||
|
||||
diff --git a/numexpr/__init__.py b/numexpr/__init__.py
|
||||
index 7946f8522d..9cabe69589 100644
|
||||
--- a/numexpr/__init__.py
|
||||
+++ b/numexpr/__init__.py
|
||||
@@ -31,8 +31,7 @@ is_cpu_amd_intel = False # DEPRECATION WARNING: WILL BE REMOVED IN FUTURE RELEAS
|
||||
import os, os.path
|
||||
import platform
|
||||
from numexpr.expressions import E
|
||||
-from numexpr.necompiler import (NumExpr, disassemble, evaluate, re_evaluate,
|
||||
- validate)
|
||||
+from numexpr.necompiler import NumExpr, disassemble, evaluate, re_evaluate
|
||||
|
||||
from numexpr.utils import (_init_num_threads,
|
||||
get_vml_version, set_vml_accuracy_mode, set_vml_num_threads,
|
||||
diff --git a/numexpr/tests/test_numexpr.py b/numexpr/tests/test_numexpr.py
|
||||
index ccb0b6cb07..32f5be4b75 100644
|
||||
--- a/numexpr/tests/test_numexpr.py
|
||||
+++ b/numexpr/tests/test_numexpr.py
|
||||
@@ -31,7 +31,7 @@ from numpy.testing import (assert_equal, assert_array_equal,
|
||||
from numpy import shape, allclose, array_equal, ravel, isnan, isinf
|
||||
|
||||
import numexpr
|
||||
-from numexpr import E, NumExpr, evaluate, re_evaluate, validate, disassemble, use_vml
|
||||
+from numexpr import E, NumExpr, evaluate, re_evaluate, disassemble, use_vml
|
||||
from numexpr.expressions import ConstantNode
|
||||
|
||||
import unittest
|
||||
@@ -370,38 +370,10 @@ class test_evaluate(TestCase):
|
||||
assert_array_equal(x, array([86., 124., 168.]))
|
||||
|
||||
def test_re_evaluate_dict(self):
|
||||
- a1 = array([1., 2., 3.])
|
||||
- b1 = array([4., 5., 6.])
|
||||
- c1 = array([7., 8., 9.])
|
||||
- x = evaluate("2*a + 3*b*c", local_dict={'a': a1, 'b': b1, 'c': c1})
|
||||
- x = re_evaluate()
|
||||
- assert_array_equal(x, array([86., 124., 168.]))
|
||||
-
|
||||
- def test_validate(self):
|
||||
a = array([1., 2., 3.])
|
||||
b = array([4., 5., 6.])
|
||||
c = array([7., 8., 9.])
|
||||
- retval = validate("2*a + 3*b*c")
|
||||
- assert(retval is None)
|
||||
- x = re_evaluate()
|
||||
- assert_array_equal(x, array([86., 124., 168.]))
|
||||
-
|
||||
- def test_validate_missing_var(self):
|
||||
- a = array([1., 2., 3.])
|
||||
- b = array([4., 5., 6.])
|
||||
- retval = validate("2*a + 3*b*c")
|
||||
- assert(isinstance(retval, KeyError))
|
||||
-
|
||||
- def test_validate_syntax(self):
|
||||
- retval = validate("2+")
|
||||
- assert(isinstance(retval, SyntaxError))
|
||||
-
|
||||
- def test_validate_dict(self):
|
||||
- a1 = array([1., 2., 3.])
|
||||
- b1 = array([4., 5., 6.])
|
||||
- c1 = array([7., 8., 9.])
|
||||
- retval = validate("2*a + 3*b*c", local_dict={'a': a1, 'b': b1, 'c': c1})
|
||||
- assert(retval is None)
|
||||
+ x = evaluate("2*a + 3*b*c", local_dict={'a': a, 'b': b, 'c': c})
|
||||
x = re_evaluate()
|
||||
assert_array_equal(x, array([86., 124., 168.]))
|
||||
|
||||
@ -0,0 +1,26 @@
|
||||
From 5bdab6e1f4d896c251270fdbe6efe77120670500 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= <zbyszek@in.waw.pl>
|
||||
Date: Wed, 16 Aug 2023 17:24:33 +0200
|
||||
Subject: [PATCH 5/7] Revert "Add in docstring intro for `validate(...)`"
|
||||
|
||||
This reverts commit da39566c2c379888991e4eb1dae95d917e13db8a.
|
||||
---
|
||||
numexpr/necompiler.py | 5 +----
|
||||
1 file changed, 1 insertion(+), 4 deletions(-)
|
||||
|
||||
diff --git a/numexpr/necompiler.py b/numexpr/necompiler.py
|
||||
index 37052acadb..f846c1d23d 100644
|
||||
--- a/numexpr/necompiler.py
|
||||
+++ b/numexpr/necompiler.py
|
||||
@@ -772,10 +772,7 @@ def validate(ex: str,
|
||||
_frame_depth: int = 2,
|
||||
**kwargs) -> Optional[Exception]:
|
||||
"""
|
||||
- Validate a NumExpr expression with the given `local_dict` or `locals()`.
|
||||
- Returns `None` on success and the Exception object if one occurs. Note that
|
||||
- you can proceed directly to call `re_evaluate()` if you use `validate()`
|
||||
- to sanitize your expressions and variables in advance.
|
||||
+ Returns
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@ -0,0 +1,307 @@
|
||||
From b773b8ba12431b7d15f9812e29861de327d0cd8f Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= <zbyszek@in.waw.pl>
|
||||
Date: Wed, 16 Aug 2023 17:24:36 +0200
|
||||
Subject: [PATCH 6/7] Revert "Add a `validate(...)` function that can be used
|
||||
to sansitize the expression and variables before starting the virtual
|
||||
machine."
|
||||
|
||||
This reverts commit 21ff376a0853aff6aea63343c6010d6434917319.
|
||||
---
|
||||
numexpr/necompiler.py | 194 +++++++++++-------------------------------
|
||||
1 file changed, 51 insertions(+), 143 deletions(-)
|
||||
|
||||
diff --git a/numexpr/necompiler.py b/numexpr/necompiler.py
|
||||
index f846c1d23d..9291d44244 100644
|
||||
--- a/numexpr/necompiler.py
|
||||
+++ b/numexpr/necompiler.py
|
||||
@@ -8,7 +8,6 @@
|
||||
# rights to use.
|
||||
####################################################################
|
||||
|
||||
-from typing import Optional, Dict
|
||||
import __future__
|
||||
import sys
|
||||
import numpy
|
||||
@@ -527,7 +526,7 @@ context_info = [
|
||||
]
|
||||
|
||||
|
||||
-def getContext(kwargs, _frame_depth=1):
|
||||
+def getContext(kwargs, frame_depth=1):
|
||||
d = kwargs.copy()
|
||||
context = {}
|
||||
for name, allowed, default in context_info:
|
||||
@@ -536,11 +535,11 @@ def getContext(kwargs, _frame_depth=1):
|
||||
context[name] = value
|
||||
else:
|
||||
raise ValueError("'%s' must be one of %s" % (name, allowed))
|
||||
-
|
||||
+
|
||||
if d:
|
||||
raise ValueError("Unknown keyword argument '%s'" % d.popitem()[0])
|
||||
if context['truediv'] == 'auto':
|
||||
- caller_globals = sys._getframe(_frame_depth + 1).f_globals
|
||||
+ caller_globals = sys._getframe(frame_depth + 1).f_globals
|
||||
context['truediv'] = caller_globals.get('division', None) == __future__.division
|
||||
|
||||
return context
|
||||
@@ -614,11 +613,11 @@ def NumExpr(ex, signature=(), **kwargs):
|
||||
# NumExpr can be called either directly by the end-user, in which case
|
||||
# kwargs need to be sanitized by getContext, or by evaluate,
|
||||
# in which case kwargs are in already sanitized.
|
||||
- # In that case _frame_depth is wrong (it should be 2) but it doesn't matter
|
||||
+ # In that case frame_depth is wrong (it should be 2) but it doesn't matter
|
||||
# since it will not be used (because truediv='auto' has already been
|
||||
# translated to either True or False).
|
||||
- _frame_depth = 1
|
||||
- context = getContext(kwargs, _frame_depth=_frame_depth)
|
||||
+
|
||||
+ context = getContext(kwargs, frame_depth=1)
|
||||
threeAddrProgram, inputsig, tempsig, constants, input_names = precompile(ex, signature, context)
|
||||
program = compileThreeAddrForm(threeAddrProgram)
|
||||
return interpreter.NumExpr(inputsig.encode('ascii'),
|
||||
@@ -718,11 +717,11 @@ def getExprNames(text, context):
|
||||
return [a.value for a in input_order], ex_uses_vml
|
||||
|
||||
|
||||
-def getArguments(names, local_dict=None, global_dict=None, _frame_depth: int=2):
|
||||
+def getArguments(names, local_dict=None, global_dict=None):
|
||||
"""
|
||||
Get the arguments based on the names.
|
||||
"""
|
||||
- call_frame = sys._getframe(_frame_depth)
|
||||
+ call_frame = sys._getframe(2)
|
||||
|
||||
clear_local_dict = False
|
||||
if local_dict is None:
|
||||
@@ -761,40 +760,32 @@ _numexpr_last = {}
|
||||
|
||||
evaluate_lock = threading.Lock()
|
||||
|
||||
-# MAYBE: decorate this function to add attributes instead of having the
|
||||
-# _numexpr_last dictionary?
|
||||
-def validate(ex: str,
|
||||
- local_dict: Optional[Dict] = None,
|
||||
- global_dict: Optional[Dict] = None,
|
||||
- out: numpy.ndarray = None,
|
||||
- order: str = 'K',
|
||||
- casting: str = 'safe',
|
||||
- _frame_depth: int = 2,
|
||||
- **kwargs) -> Optional[Exception]:
|
||||
+def evaluate(ex, local_dict=None, global_dict=None,
|
||||
+ out=None, order='K', casting='safe', **kwargs):
|
||||
"""
|
||||
- Returns
|
||||
+ Evaluate a simple array expression element-wise, using the new iterator.
|
||||
+
|
||||
+ ex is a string forming an expression, like "2*a+3*b". The values for "a"
|
||||
+ and "b" will by default be taken from the calling function's frame
|
||||
+ (through use of sys._getframe()). Alternatively, they can be specifed
|
||||
+ using the 'local_dict' or 'global_dict' arguments.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
- ex: str
|
||||
- a string forming an expression, like "2*a+3*b". The values for "a"
|
||||
- and "b" will by default be taken from the calling function's frame
|
||||
- (through use of sys._getframe()). Alternatively, they can be specified
|
||||
- using the 'local_dict' or 'global_dict' arguments.
|
||||
|
||||
- local_dict: dictionary, optional
|
||||
+ local_dict : dictionary, optional
|
||||
A dictionary that replaces the local operands in current frame.
|
||||
|
||||
- global_dict: dictionary, optional
|
||||
+ global_dict : dictionary, optional
|
||||
A dictionary that replaces the global operands in current frame.
|
||||
|
||||
- out: NumPy array, optional
|
||||
+ out : NumPy array, optional
|
||||
An existing array where the outcome is going to be stored. Care is
|
||||
required so that this array has the same shape and type than the
|
||||
actual outcome of the computation. Useful for avoiding unnecessary
|
||||
new array allocations.
|
||||
|
||||
- order: {'C', 'F', 'A', or 'K'}, optional
|
||||
+ order : {'C', 'F', 'A', or 'K'}, optional
|
||||
Controls the iteration order for operands. 'C' means C order, 'F'
|
||||
means Fortran order, 'A' means 'F' order if all the arrays are
|
||||
Fortran contiguous, 'C' order otherwise, and 'K' means as close to
|
||||
@@ -802,7 +793,7 @@ def validate(ex: str,
|
||||
efficient computations, typically 'K'eep order (the default) is
|
||||
desired.
|
||||
|
||||
- casting: {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional
|
||||
+ casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional
|
||||
Controls what kind of data casting may occur when making a copy or
|
||||
buffering. Setting this to 'unsafe' is not recommended, as it can
|
||||
adversely affect accumulations.
|
||||
@@ -813,117 +804,37 @@ def validate(ex: str,
|
||||
* 'same_kind' means only safe casts or casts within a kind,
|
||||
like float64 to float32, are allowed.
|
||||
* 'unsafe' means any data conversions may be done.
|
||||
-
|
||||
- _frame_depth: int
|
||||
- The calling frame depth. Unless you are a NumExpr developer you should
|
||||
- not set this value.
|
||||
"""
|
||||
global _numexpr_last
|
||||
-
|
||||
- try:
|
||||
-
|
||||
- if not isinstance(ex, str):
|
||||
- raise ValueError("must specify expression as a string")
|
||||
-
|
||||
- # Get the names for this expression
|
||||
- context = getContext(kwargs)
|
||||
- expr_key = (ex, tuple(sorted(context.items())))
|
||||
- if expr_key not in _names_cache:
|
||||
- _names_cache[expr_key] = getExprNames(ex, context)
|
||||
- names, ex_uses_vml = _names_cache[expr_key]
|
||||
- arguments = getArguments(names, local_dict, global_dict, _frame_depth=_frame_depth)
|
||||
-
|
||||
- # Create a signature
|
||||
- signature = [(name, getType(arg)) for (name, arg) in
|
||||
- zip(names, arguments)]
|
||||
-
|
||||
- # Look up numexpr if possible.
|
||||
- numexpr_key = expr_key + (tuple(signature),)
|
||||
- try:
|
||||
- compiled_ex = _numexpr_cache[numexpr_key]
|
||||
- except KeyError:
|
||||
- compiled_ex = _numexpr_cache[numexpr_key] = NumExpr(ex, signature, **context)
|
||||
- kwargs = {'out': out, 'order': order, 'casting': casting,
|
||||
- 'ex_uses_vml': ex_uses_vml}
|
||||
- _numexpr_last = dict(ex=compiled_ex, argnames=names, kwargs=kwargs)
|
||||
- # with evaluate_lock:
|
||||
- # return compiled_ex(*arguments, **kwargs)
|
||||
- except Exception as e:
|
||||
- return e
|
||||
- return None
|
||||
-
|
||||
-def evaluate(ex: str,
|
||||
- local_dict: Optional[Dict] = None,
|
||||
- global_dict: Optional[Dict] = None,
|
||||
- out: numpy.ndarray = None,
|
||||
- order: str = 'K',
|
||||
- casting: str = 'safe',
|
||||
- _frame_depth: int=3,
|
||||
- **kwargs) -> numpy.ndarray:
|
||||
- """
|
||||
- Evaluate a simple array expression element-wise using the virtual machine.
|
||||
-
|
||||
- Parameters
|
||||
- ----------
|
||||
- ex: str
|
||||
- a string forming an expression, like "2*a+3*b". The values for "a"
|
||||
- and "b" will by default be taken from the calling function's frame
|
||||
- (through use of sys._getframe()). Alternatively, they can be specified
|
||||
- using the 'local_dict' or 'global_dict' arguments.
|
||||
-
|
||||
- local_dict: dictionary, optional
|
||||
- A dictionary that replaces the local operands in current frame.
|
||||
-
|
||||
- global_dict: dictionary, optional
|
||||
- A dictionary that replaces the global operands in current frame.
|
||||
-
|
||||
- out: NumPy array, optional
|
||||
- An existing array where the outcome is going to be stored. Care is
|
||||
- required so that this array has the same shape and type than the
|
||||
- actual outcome of the computation. Useful for avoiding unnecessary
|
||||
- new array allocations.
|
||||
-
|
||||
- order: {'C', 'F', 'A', or 'K'}, optional
|
||||
- Controls the iteration order for operands. 'C' means C order, 'F'
|
||||
- means Fortran order, 'A' means 'F' order if all the arrays are
|
||||
- Fortran contiguous, 'C' order otherwise, and 'K' means as close to
|
||||
- the order the array elements appear in memory as possible. For
|
||||
- efficient computations, typically 'K'eep order (the default) is
|
||||
- desired.
|
||||
-
|
||||
- casting: {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional
|
||||
- Controls what kind of data casting may occur when making a copy or
|
||||
- buffering. Setting this to 'unsafe' is not recommended, as it can
|
||||
- adversely affect accumulations.
|
||||
-
|
||||
- * 'no' means the data types should not be cast at all.
|
||||
- * 'equiv' means only byte-order changes are allowed.
|
||||
- * 'safe' means only casts which can preserve values are allowed.
|
||||
- * 'same_kind' means only safe casts or casts within a kind,
|
||||
- like float64 to float32, are allowed.
|
||||
- * 'unsafe' means any data conversions may be done.
|
||||
-
|
||||
- _frame_depth: int
|
||||
- The calling frame depth. Unless you are a NumExpr developer you should
|
||||
- not set this value.
|
||||
- """
|
||||
- # We could avoid code duplication if we called validate and then re_evaluate
|
||||
- # here, but they we have difficulties with the `sys.getframe(2)` call in
|
||||
- # `getArguments`
|
||||
- e = validate(ex, local_dict=local_dict, global_dict=global_dict,
|
||||
- out=out, order=order, casting=casting,
|
||||
- _frame_depth=_frame_depth, **kwargs)
|
||||
- if e is None:
|
||||
- return re_evaluate(local_dict=local_dict, _frame_depth=_frame_depth)
|
||||
- else:
|
||||
- raise e
|
||||
+ if not isinstance(ex, str):
|
||||
+ raise ValueError("must specify expression as a string")
|
||||
|
||||
+ # Get the names for this expression
|
||||
+ context = getContext(kwargs, frame_depth=1)
|
||||
+ expr_key = (ex, tuple(sorted(context.items())))
|
||||
+ if expr_key not in _names_cache:
|
||||
+ _names_cache[expr_key] = getExprNames(ex, context)
|
||||
+ names, ex_uses_vml = _names_cache[expr_key]
|
||||
+ arguments = getArguments(names, local_dict, global_dict)
|
||||
|
||||
-
|
||||
+ # Create a signature
|
||||
+ signature = [(name, getType(arg)) for (name, arg) in
|
||||
+ zip(names, arguments)]
|
||||
|
||||
+ # Look up numexpr if possible.
|
||||
+ numexpr_key = expr_key + (tuple(signature),)
|
||||
+ try:
|
||||
+ compiled_ex = _numexpr_cache[numexpr_key]
|
||||
+ except KeyError:
|
||||
+ compiled_ex = _numexpr_cache[numexpr_key] = NumExpr(ex, signature, **context)
|
||||
+ kwargs = {'out': out, 'order': order, 'casting': casting,
|
||||
+ 'ex_uses_vml': ex_uses_vml}
|
||||
+ _numexpr_last = dict(ex=compiled_ex, argnames=names, kwargs=kwargs)
|
||||
+ with evaluate_lock:
|
||||
+ return compiled_ex(*arguments, **kwargs)
|
||||
|
||||
-def re_evaluate(local_dict: Optional[Dict] = None,
|
||||
- _frame_depth: int=2) -> numpy.ndarray:
|
||||
+
|
||||
+def re_evaluate(local_dict=None):
|
||||
"""
|
||||
Re-evaluate the previous executed array expression without any check.
|
||||
|
||||
@@ -933,20 +844,17 @@ def re_evaluate(local_dict: Optional[Dict] = None,
|
||||
|
||||
Parameters
|
||||
----------
|
||||
- local_dict: dictionary, optional
|
||||
+
|
||||
+ local_dict : dictionary, optional
|
||||
A dictionary that replaces the local operands in current frame.
|
||||
- _frame_depth: int
|
||||
- The calling frame depth. Unless you are a NumExpr developer you should
|
||||
- not set this value.
|
||||
+
|
||||
"""
|
||||
- global _numexpr_last
|
||||
-
|
||||
try:
|
||||
compiled_ex = _numexpr_last['ex']
|
||||
except KeyError:
|
||||
- raise RuntimeError("A previous evaluate() execution was not found, please call `validate` or `evaluate` once before `re_evaluate`")
|
||||
+ raise RuntimeError("not a previous evaluate() execution found")
|
||||
argnames = _numexpr_last['argnames']
|
||||
- args = getArguments(argnames, local_dict, _frame_depth=_frame_depth)
|
||||
+ args = getArguments(argnames, local_dict)
|
||||
kwargs = _numexpr_last['kwargs']
|
||||
with evaluate_lock:
|
||||
return compiled_ex(*args, **kwargs)
|
||||
@ -0,0 +1,31 @@
|
||||
From 89696f397f167958a6dfc8ccb1f0d796fb4307a3 Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Zbigniew=20J=C4=99drzejewski-Szmek?= <zbyszek@in.waw.pl>
|
||||
Date: Wed, 16 Aug 2023 17:11:28 +0200
|
||||
Subject: [PATCH 7/7] Use r"" to avoid warning about unknown escapes
|
||||
|
||||
---
|
||||
numexpr/cpuinfo.py | 4 ++--
|
||||
1 file changed, 2 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/numexpr/cpuinfo.py b/numexpr/cpuinfo.py
|
||||
index 4f934b7ba2..4a57d3cb03 100755
|
||||
--- a/numexpr/cpuinfo.py
|
||||
+++ b/numexpr/cpuinfo.py
|
||||
@@ -105,7 +105,7 @@ class CPUInfoBase(object):
|
||||
|
||||
def __get_nbits(self):
|
||||
abits = platform.architecture()[0]
|
||||
- nbits = re.compile('(\d+)bit').search(abits).group(1)
|
||||
+ nbits = re.compile(r'(\d+)bit').search(abits).group(1)
|
||||
return nbits
|
||||
|
||||
def _is_32bit(self):
|
||||
@@ -658,7 +658,7 @@ class Win32CPUInfo(CPUInfoBase):
|
||||
#XXX: Bad style to use so long `try:...except:...`. Fix it!
|
||||
|
||||
prgx = re.compile(r"family\s+(?P<FML>\d+)\s+model\s+(?P<MDL>\d+)"
|
||||
- "\s+stepping\s+(?P<STP>\d+)", re.IGNORECASE)
|
||||
+ r"\s+stepping\s+(?P<STP>\d+)", re.IGNORECASE)
|
||||
chnd = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, self.pkey)
|
||||
pnum = 0
|
||||
while 1:
|
||||
@ -0,0 +1,28 @@
|
||||
From fcceab4c0c59bdbcf79ebca844ac3865bb6e795d Mon Sep 17 00:00:00 2001
|
||||
From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= <miro@hroncok.cz>
|
||||
Date: Thu, 30 May 2024 17:04:50 +0200
|
||||
Subject: [PATCH] Fix necompiler.getArguments() on Python 3.13.0b1
|
||||
|
||||
The FrameLocalsProxy object (PEP 667) cannot be cleared.
|
||||
|
||||
Fixes https://github.com/pydata/numexpr/issues/488
|
||||
---
|
||||
numexpr/necompiler.py | 2 +-
|
||||
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||
|
||||
diff --git a/numexpr/necompiler.py b/numexpr/necompiler.py
|
||||
index 5126bd7..1e60b5a 100644
|
||||
--- a/numexpr/necompiler.py
|
||||
+++ b/numexpr/necompiler.py
|
||||
@@ -767,7 +767,7 @@ def getArguments(names, local_dict=None, global_dict=None, _frame_depth: int=2):
|
||||
# If we generated local_dict via an explicit reference to f_locals,
|
||||
# clear the dict to prevent creating extra ref counts in the caller's scope
|
||||
# See https://github.com/pydata/numexpr/issues/310
|
||||
- if clear_local_dict:
|
||||
+ if clear_local_dict and hasattr(local_dict, 'clear'):
|
||||
local_dict.clear()
|
||||
|
||||
return arguments
|
||||
--
|
||||
2.45.0
|
||||
|
||||
Loading…
Reference in new issue