Falcon GZIP filter.

Falcon is a minimalist Python web framework for building cloud APIs and app backends. Using this filter I added GZIP support for my JSON REST API's.
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
import io

import gzip


class GzipMiddleware(object):
    """GZIP compression middleware

    """
    __slots__ = ('compression_min_size', 'compress_level')

    def __init__(self, compression_min_size=2048, compress_level=5):
        self.compression_min_size = compression_min_size
        self.compress_level = compress_level

    def process_response(self, req, resp, resource):
        """Post-processing of the response (after routing).

        Args:
            req: Request object.
            resp: Response object.
            resource: Resource object to which the request was
                routed. May be None if no route was found
                for the request.
        """
        headers = req.headers
        accept_hdr = headers.get('ACCEPT-ENCODING')
        if accept_hdr and 'gzip' in accept_hdr.lower():
            data = resp.body
            if data and len(data) >= self.compression_min_size:
                gzip_stream = io.BytesIO()
                with gzip.GzipFile(fileobj=gzip_stream, mode="w", compresslevel=self.compress_level) as f:
                    f.write(bytes(data, "utf-8"))
                resp.data = gzip_stream.getvalue()
                resp.body = None
                resp.append_header('Content-Encoding', 'gzip')

Python AST expression evaluator

Using Python AST (Abstract Syntax Tree) module I designed an interesting expression evaluator.
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
#!/usr/bin/python
# -*- coding: utf-8 -*-
__author__ = 'Marius'

import ast


def eval_node(node, params):
    """
    Evaluate node
    """

    if isinstance(node, ast.Num):
        return ast.literal_eval(node)
    elif isinstance(node, ast.Str):
        return ast.literal_eval(node)
    elif isinstance(node, ast.Name):
        return params[node.__dict__['id']]
    elif isinstance(node, ast.BinOp):
        return eval_bin_node(node, params)
    elif isinstance(node, ast.UnaryOp):
        return eval_unary_node(node, params)
    elif isinstance(node, ast.BoolOp):
        return eval_bool_node(node, params)
    elif isinstance(node, ast.Compare):
        return eval_comp_node(node, params)
    else:
        raise Exception('Unsupported node type.')


def eval_bin_node(node, params):
    """
    Evaluate BinOp node.
    """

    # evaluate left and right nodes

    left_value = eval_node(node.__dict__['left'], params)
    right_value = eval_node(node.__dict__['right'], params)

    # operator

    operator = node.__dict__['op']

    # Arithmetic Operators

    if isinstance(operator, ast.Add):
        return left_value + right_value
    if isinstance(operator, ast.Sub):
        return left_value - right_value
    if isinstance(operator, ast.Mult):
        return left_value * right_value
    if isinstance(operator, ast.Div):
        return left_value / right_value
    if isinstance(operator, ast.Mod):
        return left_value % right_value
    if isinstance(operator, ast.Pow):
        return left_value ** right_value
    if isinstance(operator, ast.FloorDiv):
        return left_value // right_value

    # Bitwise Operators

    if isinstance(operator, ast.BitOr):
        return left_value | right_value
    if isinstance(operator, ast.BitAnd):
        return left_value & right_value
    if isinstance(operator, ast.BitXor):
        return left_value ^ right_value
    if isinstance(operator, ast.LShift):
        return left_value << right_value
    if isinstance(operator, ast.RShift):
        return left_value >> right_value
    else:
        raise Exception('Unsupported operator.')


def eval_comp_node(node, params):

    # left operand

    left_value = eval_node(node.__dict__['left'], params)

    # operator

    operators = node.__dict__['ops']

    # right operators

    comparators = node.__dict__['comparators']
    for i in range(len(operators)):

        # evaluate right node

        right_value = eval_node(comparators[i], params)

        # operator

        operator = operators[i]

        # Comparison (i.e., Relational) Operators

        if isinstance(operator, ast.Eq):
            left_value = left_value == right_value
        elif isinstance(operator, ast.NotEq):
            left_value = left_value != right_value
        elif isinstance(operator, ast.Lt):
            left_value = left_value < right_value
        elif isinstance(operator, ast.LtE):
            left_value = left_value <= right_value
        elif isinstance(operator, ast.Gt):
            left_value = left_value > right_value
        elif isinstance(operator, ast.GtE):
            left_value = left_value >= right_value
        else:
            raise Exception('Unsupported operator')

    return left_value


def eval_bool_node(node, params):
    """
    Evaluate boolean expressions
    """

    left_value = None

    # operator

    operator = node.__dict__['op']

    for right in node.__dict__['values']:
        right_value = eval_node(right, params)
        if left_value is None:
            left_value = right_value
        else:

            # Boolean Operators

            if isinstance(operator, ast.And):
                left_value = left_value and right_value
            elif isinstance(operator, ast.Or):
                left_value = left_value or right_value
            else:
                raise Exception('Unknown operator.')

    return left_value


def eval_unary_node(node, params):
    """
    Evaluate unary expressions
    """

    # Boolean Operators

    operator = node.__dict__['op']
    if isinstance(operator, ast.Not):
        return not eval_node(node.__dict__['operand'], params)
    else:
        raise Exception('Unknown operator.')


def eval_expr(expression, params):
    """
    Evaluate an expression using python ast (abstract syntax tree)
    """

    tree = ast.parse(expression)

    # module

    module = ast.walk(tree).next()
    if not isinstance(module, ast.Module):
        raise Exception('Missing ast.Module node.')

    # print ast.dump(module, True)

    expr_node = module.__dict__['body'][0]
    if not isinstance(expr_node, ast.Expr):
        raise Exception('Missing ast.Expr node.')

    # evaluate expression

    expr_value = expr_node.__dict__['value']
    if isinstance(expr_value, ast.BinOp):
        return eval_bin_node(expr_value, params)
    if isinstance(expr_value, ast.UnaryOp):
        return eval_unary_node(expr_value, params)
    if isinstance(expr_value, ast.BoolOp):
        return eval_bool_node(expr_value, params)
    elif isinstance(expr_value, ast.Compare):
        return eval_comp_node(expr_value, params)
    else:
        print expr_value
        raise Exception('Unsupported node type.')


if __name__ == '__main__':
    formula = '2.67 + (1 + data / 1000.0 + data**2)'
    print eval_expr(formula, {'data': 314})
    print eval_expr('x and y', {'x': True, 'y': True})
    print eval_expr('not x', {'x': True})


			

Python statistics

Python language could be used to implement statistics formula.
  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
__author__ = 'Marius'
				
from array import array
from math import sqrt, pow, fsum, log, pi, e


class Statistics():
    def __init__(self):
        pass

    @staticmethod
    def avg(x=array('d')):
        """
        Calculate the average value of a population.

        :param x:   population
        :return:    average value
        """
        _size = len(x)
        if _size < 1:
            raise Exception('Data set size too small.')
        # return reduce(lambda x, y: x + y, population) / _size
        return fsum(x) / _size

    @classmethod
    def var(cls, x=array('d')):
        """
        Measures how far a set of numbers is spread out.

        :param x:   population
        :return:
        """
        _size = len(x) - 1
        _mean = cls.avg(x)
        _sum = 0.0
        for v in x:
            xv = v - _mean
            _sum += (xv * xv)
        return _sum / _size

    @classmethod
    def stdev(cls, x=array('d')):
        """
        Measure that is used to quantify the amount of variation or dispersion of a set of data values.

        :param x:   population
        :return:    standard deviation
        """
        return sqrt(cls.var(x))

    @classmethod
    def stderrm(cls, x=array('d')):
        _size = len(x)
        return cls.avg(x) / sqrt(_size)

    @classmethod
    def stderrd(cls, x=array('d')):
        _size = len(x)
        return cls.stdev(x) / sqrt(_size)

    @classmethod
    def devsq(cls, x=array('d')):
        """
        Calculate sum of squares of deviations of data points from their sample mean.

        :param x:   population
        :return:    sum of squares of deviations of data points from their sample mean.
        """
        sigma = cls.avg(x)
        _sum = 0.0
        for v in x:
            xv = v - sigma
            _sum += (xv * xv)
        return _sum

    @staticmethod
    def harmean(x=array('d')):
        """
        Returns the harmonic mean of a data set.
        The harmonic mean is the reciprocal of the arithmetic mean of reciprocals.

        :param x:   population
        :return:    harmonic mean
        """
        _size = len(x)
        if _size == 0:
            raise Exception('Data set is empty.')
        _sum = 0.0
        for v in x:
            _sum += (1.0 / v)
        return _size / _sum

    @staticmethod
    def geomean(x=array('d')):
        """
        In mathematics, the geometric mean is a type of mean or average, which indicates the central tendency or
        typical value of a set of numbers by using the product of their values (as opposed to the arithmetic mean
        which uses their sum). The geometric mean is defined as the nth root of the product of n numbers.

        :param x:   population
        :return:    geometric mean
        """
        _size = len(x)
        if _size == 0:
            raise Exception('Data set is empty.')
        _prod = 1.0
        for v in x:
            _prod *= v
        return pow(_prod, 1.0 / _size)

    @classmethod
    def covar(cls, x=array('d'), y=array('d')):
        """
        Measure of linear relation between random variables.
        :param x:   first population
        :param y:   second population
        :return:
        """
        _size = len(x)
        if _size != len(y):
            raise Exception('Data sets have different lengths.')
        xm = cls.avg(x)
        ym = cls.avg(y)
        _sum = 0.0
        for i in range(_size):
            _sum += ((x[i] - xm) * (y[i] - ym))
        return _sum / _size

    @classmethod
    def correl_base(cls, x=array('d'), y=array('d')):
        """
        The correlation coefficient base computation

        :param x:   first population
        :param y:   second population
        :return:    population sums.
        """
        _size = len(x)
        if _size != len(y):
            raise Exception('Data sets have different lengths.')
        xm = cls.avg(x)
        ym = cls.avg(y)
        _sum1 = 0.0
        _sum2 = 0.0
        _sum3 = 0.0
        for i in range(_size):
            xv = x[i] - xm
            yv = y[i] - ym
            _sum1 += (xv * yv)
            _sum2 += (xv * xv)
            _sum3 += (yv * yv)
        return _sum1, _sum2, _sum3

    @classmethod
    def correl(cls, x=array('d'), y=array('d')):
        """
        The correlation coefficient (a value between -1 and +1) tells you how strongly
        two variables are related to each other.

        :param x:   first population
        :param y:   second population
        :return:    correlation coefficient.
        """
        _sum1, _sum2, _sum3 = cls.correl_base(x, y)
        return _sum1 / sqrt(_sum2 * _sum3)

    @classmethod
    def pearson(cls, x=array('d'), y=array('d')):
        """
        The Pearson correlation coefficient (a value between -1 and +1) tells you how strongly
        two variables are related to each other.

        :param x:   first population
        :param y:   second population
        :return:    Pearson correlation coefficient.
        """
        _sum1, _sum2, _sum3 = cls.correl_base(x, y)
        return _sum1 / (sqrt(_sum2) * sqrt(_sum3))

    @staticmethod
    def factorial_approximation(n):
        """
        Srinivasa Ramanujan factorial estimation. Good for larger values of n.

        :param n:   parameter
        :return:    ln(n!)
        """
        if n < 2.0:
            raise Exception('Parameter n is too small.')
        a = n * log(n) - n
        b = log(n * (1.0 + 4.0 * n * (1.0 + 2.0 * n))) / 6.0
        return a + b + log(pi) / 2.0

    @classmethod
    def poisson_probability(cls, mean, observed):
        """
        Calculates an approximation of the Poisson probability.

        :param mean:        lambda, the average number of occurrences
        :param observed:    the actual number of occurrences observed
        :return:    ln(Poisson probability) - the natural log of the Poisson probability.
        """
        a = observed * log(mean)
        b = -mean
        return a + b - cls.factorial_approximation(observed)

    @classmethod
    def kurtosis(cls, x=array('d')):
        """
        In probability theory and statistics, kurtosis (from the Greek word κυρτός, kyrtos or kurtos, meaning curved,
        arching) is any measure of the "peakedness" of the probability distribution of a real-valued random variable.

        :param x:   data population
        :return:    kurtosis.
        """
        n = len(x)
        if n < 4:
            raise Exception('Data set length too small.')
        # standard deviation
        sigma = cls.stdev(x)
        if sigma == 0:
            return float('nan')
        # average
        mean = cls.avg(x)
        sum = 0.0
        for item in x:
            v = (item - mean) / sigma
            vv = v * v
            sum += (vv * vv)
        # constants
        k1 = 1.0 * n * (n + 1) / (n - 1) / (n - 2) / (n - 3)
        k2 = 3.0 * (n - 1) * (n - 1) / (n - 2) / (n - 3)
        # kurtosis
        k1 * sum - k2

    @classmethod
    def skew(cls, x=array('d')):
        """
        In probability theory and statistics, skewness is a measure of the extent to which a probability distribution
        of a real-valued random variable "leans" to one side of the mean. The skewness value can be positive or
        negative, or even undefined.

        :param x:   data population
        :return:    skewness.
        """
        n = len(x)
        if n < 3:
            raise Exception('Data set length too small.')
        # standard deviation
        sigma = cls.stdev(x)
        if sigma == 0:
            return float('nan')
        # average
        mean = cls.avg(x)
        sum = 0.0
        for item in x:
            v = (item - mean) / sigma
            sum += (v * v * v)
        # constants
        k1 = 1.0 * n / (n - 1) / (n - 2)
        return k1 * sum

    @classmethod
    def norminv(cls, x=array('d')):
        """
        Returns the inverse of the standard normal cumulative distribution.

        :param x:   data population
        :return:    inverse of the standard normal cumulative distribution..
        """
        n = len(x)
        if n < 1:
            return float('nan')
        return cls.stdev(x) / sqrt(n)

    @classmethod
    def common(cls, x=array('d'), y=array('d'), xm=0.0, ym=0.0):
        """
        A common function used by slope, intercept, forecast functions

        :param x:   first population
        :param y:   second population
        :param xm:  avg(x)
        :param ym:  avg(y)
        :return:    tuple of sum's.
        """
        sum1 = 0.0
        sum2 = 0.0
        # calculate sum's
        for i in range(len(x)):
            xv = x(i) - xm
            yv = y(i) - ym
            sum1 += (xv * yv)
            sum2 += (xv * xv)
        return sum1, sum2

    @classmethod
    def slope(cls, x=array('d'), y=array('d')):
        """
        Returns the slope of the linear regression line through data points in known_y's and known_x's.
        The slope is the vertical distance divided by the horizontal distance between any two points on the line,
        which is the rate of change along the regression line.

        :param x:   first population
        :param y:   second population
        :return:    slope (b coefficient)
        """
        if len(x) != len(y):
            return float('nan')
        xm = cls.avg(x)
        ym = cls.avg(y)
        sum1, sum2 = cls.common(x, y, xm, ym)
        # the b coefficient
        return sum1 / sum2

    @classmethod
    def intercept(cls, x=array('d'), y=array('d')):
        """
        Calculates the point at which a line will intersect the y-axis by using existing x-values and y-values. The
        intercept point is based on a best-fit regression line plotted through the known x-values and known y-values.
        Use the INTERCEPT function when you want to determine the value of the dependent variable when the independent
        variable is 0(zero).
        For example, you can use the INTERCEPT function to predict a metal's electrical resistance at 0°C when your
        data points were taken at room temperature and higher.

        :param x:   first population
        :param y:   second population
        :return:    intercept
        """
        if len(x) != len(y):
            return float('nan')
        xm = cls.avg(x)
        ym = cls.avg(y)
        sum1, sum2 = cls.common(x, y, xm, ym)
        # the b coefficient
        b = sum1 / sum2
        # intercept
        return ym - xm * b

    @classmethod
    def forecast(cls, p=0.0, x=array('d'), y=array('d')):
        """
        Calculates, or predicts, a future value by using existing values. The predicted value is a y-value for a given
        x-value. The known values are existing x-values and y-values, and the new value is predicted by using linear
        regression. You can use this function to predict future sales, inventory requirements, or consumer trends.

        :param p:   data point
        :param x:   first population
        :param y:   second population
        :return:    forecast
        """
        if len(x) != len(y):
            return float('nan')
        xm = cls.avg(x)
        ym = cls.avg(y)
        sum1, sum2 = cls.common(x, y, xm, ym)
        # the b coefficient
        b = sum1 / sum2
        # intercept
        a = ym - xm * b
        return a + b * p

Flag Counter