#StackBounty: #python #multithreading #matplotlib #tkinter Running a thread with matplotlib cause python crash

Bounty: 100

I have a program that has 2 very simple threads:

  • One for listening to serial port
  • One for a text UI

I also have a matplotlib animation running in my main(). It has a scope class that is the example from matplotlib.

When the program starts to run, it shows the plot and everything is OK. The problem is that as soon as the user enters a key, the program crashes and python exits with a fatal error.

The ui thread has nothing to do with matplotlib and scope class. If I delete the code that creates the plots, the ui thread has no problem and program runs smoothly. I also notice matplotlib on my system uses tkinter for creating windows.

Do you have any hints or experiences with why the matplotlib animation causes problem? Can’t a thread be used with a matplotlib plot? I am running this in a command window in Windows7 with Python 2.7.

Error:

Fatal Python error: GC object already tracked

This application has requested the Runtime to terminate it in an unusual way.
Please contact the application's support team for more information.

Or this error:

TclStackFree: incorrect freePtr. Call out of sequence?

This application has requested the Runtime to terminate it in an unusual way.
Please contact the application's support team for more information.

Code:

import threading
import serial
from matplotlib.lines import Line2D
import matplotlib.pyplot as plt
import matplotlib.animation as animation

class listner(threading.Thread):      
    def __init__(self):
        threading.Thread.__init__(self)
        self.is_running=True
        self.init_port() ## inits a serial port

    def run(self):
        while self.is_running:
            self.sport.read(1)

    def process(self):
        pass


class ui(threading.Thread):
    def __init__(self):
        threading.Thread.__init__(self)     
        self.running = True

    def run(self):
        while self.running:
            user = raw_input()
            if user.upper() == 'S':
                pass  

def main(port):
    listner_thread = None
    listner_thread = listner.listner(port)
    listner_thread.start();

    ui_thread=None
    ui_thread = ui.ui()
    ui_thread.start()          
    run_charts()


def run_charts():
    fig, (ax1, ax2) = plt.subplots(2, 1)
    scope1= Scope(ax1)
    ani1 = animation.FuncAnimation(fig, scope1.update, emit_ch1, interval=10,blit=True)

    scope2 = Scope(ax2)
    ani2 = animation.FuncAnimation(fig, scope2.update, emit_ch2, interval=10,blit=True)

    plt.show()

## taken from [https://matplotlib.org/examples/animation/strip_chart_demo.html][1]   
class Scope(object):
    def __init__(self, ax, maxt=2, dt=0.02):
        self.ax = ax
        self.dt = dt
        self.maxt = maxt
        self.tdata = [0]
        self.ydata = [0]
        self.line = Line2D(self.tdata, self.ydata)
        self.ax.add_line(self.line)
        self.ax.set_ylim(-.1, 1.1)
        self.ax.set_xlim(0, self.maxt)

    def update(self, y):
        t = self.tdata[-1] + self.dt
        self.tdata.append(t)
        self.ydata.append(y)
        self.line.set_data(self.tdata, self.ydata)
        return self.line,



if __name__ == '__main__':
    main('COM11')


Get this bounty!!!

#StackBounty: #python #recommender-system How to create a relational factorisation recommender?

Bounty: 50

I want to create a recommender system with factorisation recommander.
We have scores on hashtags, ads have scores on hashtags, and we even have scores of subscriber on ads.
We want to use this to recommend ads to subscriber.

  • Scores of suscriber on hastags (but not all of them) :

called df_p

    count  hashtag_id  subscriber_id
0      22         321            172
1      44         321            161
2      25         322            172
3       6         323            172
4      32         325            172
5      26         328            161
...   ...         ...            ...
  • Scores of ads on hashtags (but not all of them):

called df_q

      count  eclipse_id  hashtag_id
0         1        6521         321
1         1        6606         321
2         1        6609         321
3         1        6617         321
4         1        6649         321
5         1        6911         321
...      ...        ...         ...
  • Scores of subscriber on ad (still: not all of them):

How can I use GraphLab factorization_recommender to recommand ad to suscriber ?

The output should be (here for a given user 13):

subscriber_id   eclipse_id  score   rank
13  6565    0.059975420017  1
13  6588    0.0389804676959     2
13  9762    0.0159731995118     3
13  9606    0.0159731995118     4
13  9854    0.0159731995118     5
13  9576    0.0159731995118     6
13  9902    0.0155536116738     7
13  9875    0.0155536116738     8
13  6766    0.0126994707082     9
13  9870    0.0125380719963     10

On the GraphLab official website, I already found :

user_info = graphlab.SFrame({'user_id': ["0", "1", "2"],
                              'name': ["Alice", "Bob", "Charlie"],
                              'numeric_feature': [0.1, 12, 22]})
item_info = graphlab.SFrame({'item_id': ["a", "b", "c", d"],
                              'name': ["item1", "item2", "item3", "item4"],
                              'dict_feature': [{'a' : 23}, {'a' : 13},
                                               {'b' : 1},
                                               {'a' : 23, 'b' : 32}]})
m2 = graphlab.factorization_recommender.create(sf, target='rating',
                                                user_data=user_info,
                                                item_data=item_info)

But it seems not to apply to my case as far as here we have scores on user and scores on item. My project is more relational as described above.


I already tried like wihout the relational features :

products = pd.DataFrame(r_cols)
# We fill NaN cells with 0s
products['eclipse_id'] = products['eclipse_id'].fillna(0).astype(int)
# We transform it to an SFrame
sf = SFrame(data=products)
# We split data to train the model
ratings_base,ratings_test = sf.random_split(.8, seed=0)
# We train the model
# Train Model
personalized_model = graphlab.item_similarity_recommender.create(ratings_base, user_id='subscriber_id', item_id='eclipse_id', target='count', similarity_type='cosine')

Allowing me to recommend (but I don’t know on the basis of what) :

personalized_model.recommend(users=[users[0]])

Giving me back :

subscriber_id   eclipse_id  score   rank
172     11279   5.42914502761        1
172     11275   1.65434508464        2
172     0       0.139058996649       3
172     11268   0.0788922800737      4


Get this bounty!!!

#StackBounty: #python #python-2.7 #image #opencv Simple AR Tag classifier

Bounty: 50

I have been working on creating a simple AR Tag classifier, for detecting a simple AR Tag glyph. Spitting out an image that looks like this:

enter image description here

With the red circle indicating the corner closest to the orientation square.

My program is structured as follows:

AR.py (where all the magic happens)

import cv2
import numpy as np

from artag import ARtag


def display_scaled_image(name, image, scale):
    """Function to display a scaled cv2 image
    :param name:
        Window name
    :type name:
        basestring
    :param image:
        Image as numpy array
    :type image:
        numpy.ndarray
    :param scale:
        Scale factor applied to image
    :type scale:
        float
    """
    height, width = image.shape[:2]
    cv2.imshow(name, cv2.resize(image,
            (int(scale * width), int(scale * height)),
            interpolation=cv2.INTER_CUBIC))


def mask_black(img):
    """Function to mask for black boxes in an BGR image
    :param img:
        Numpy image to process
    :type img:
        numpy.ndarray
    :return:
        the masked image
    :rtype:
        numpy.ndarray
    """
    return cv2.adaptiveThreshold(img, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C,
                        cv2.THRESH_BINARY, 511, 10)


def find_tag(img):
    """
    Identify AR tags in an image

    :param img:
        The image to be processed, may be pre-converted to greyscale
    :type img:
        numpy.ndarray
    :return:
        Tuple[List[artag], numpy.ndarray]
    """
    # Convert the image to greyscale if it is not
    if len(img.shape) > 2:
        img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    dims = img.shape[:2]
    # Normalize image size
    img, _ = rescale_img(img)
    kernel = np.ones((3, 3), np.uint8)
    img = mask_black(img)
    img = cv2.morphologyEx(img, cv2.MORPH_OPEN, kernel)

    _, contours, hierarchy = cv2.findContours(img, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

    # Clean up contours to regular shapes
    culled_conts = []
    for c, h in zip(contours, hierarchy[0]):
        c = cv2.convexHull(c)
        epsilon = 0.01 * cv2.arcLength(c, True)
        approx = cv2.approxPolyDP(c, epsilon, True)
        culled_conts.append((approx, h))

    img, scale = rescale_img(img, dims)

    # Identify nested shapes as possible AR tags
    ar_tags = []
    for c, h in culled_conts:
        # 4 sided tags
        if len(c) == 4:
            # The tag contains a smaller contour
            if h[2] != -1:
                tag = ARtag(c, culled_conts[h[2]][0], scale)
                if tag.valid():
                    ar_tags.append(tag)

    img = cv2.merge((img, img, img))

    return ar_tags, img


def rescale_img(img, dims=None, width=1024):
    """Rescale image, either to the dimensions provided or to the width
    provided, returning the image and it's dimensions

    :param img:
        The input image
    :type img:
        numpy.ndarray
    :param dims:
        Provide this parameter to fit the image to a specific size
    :type dims:
        Tuple[int, int]
    :param width:
        The width to resize the image to, (overridden by dims) deafualts to 1024
    :type width:
        int
    :return:
        Tuple of image and its dimensions
    :rtype:
        Tuple[numpy.ndarray, Tuple[int, int]]
    """
    grey = img
    if dims is None:
        r = float(width) / grey.shape[1]
        dims = (width, int(grey.shape[0] * r))
    else:
        dims = dims[::-1]
    scale = (dims[1] / float(grey.shape[0]), dims[0] / float(grey.shape[1]))
    return cv2.resize(grey, dims, interpolation=cv2.INTER_AREA), scale


if __name__ == "__main__":
    # Display results for all test images
    import os
    for f in list(os.walk("Test imges"))[0][2]:
        if "jpg" in f:
            in_img = cv2.imread("Test imges/" + f)
            grey = cv2.cvtColor(in_img, cv2.COLOR_BGR2GRAY)
            ar_tags, proc_img = find_tag(grey)
            for i in ar_tags:
                proc_img = i.draw(proc_img)
                in_img = i.draw(in_img)
            display_scaled_image("Image - {}".format(f), proc_img, .25)
            display_scaled_image("Orig Image - {}".format(f), in_img, .25)
            cv2.waitKey(0)
            cv2.destroyAllWindows()

artag.py

import math

import cv2


class ARtag(object):
    def __init__(self, outer, inner, scale):
        """Create a ARTag object, based on its inner contour,
        outer contour and the scale of the image processed as
        related to the image that was processed to extract them

        :param outer:
            The outer contour (Assumed to have 4 sides)
        :param inner:
            The outer contour (Assumed to have at least 4 sides)
        :param scale:
            The scale of the image that was processed to create this tag
        :type scale:
            Tuple[int, int]
        """
        self.outer_cont = outer
        for i, c in enumerate(self.outer_cont):
            self.outer_cont[i][0][0] = c[0][0] * scale[1]
            self.outer_cont[i][0][1] = c[0][1] * scale[0]
        self.inner_cont = inner
        for i, c in enumerate(self.inner_cont):
            self.inner_cont[i][0][0] = c[0][0] * scale[1]
            self.inner_cont[i][0][1] = c[0][1] * scale[0]

        self.outer_tup = [tuple(i[0]) for i in outer]
        self.inner_tup = [tuple(i[0]) for i in inner]

        x, y = zip(*self.inner_tup)
        x = sum(x) / len(x)
        y = sum(y) / len(y)

        self.outer_tup = sorted(self.outer_tup, key=lambda p: math.sqrt((p[0] - x)**2 + (p[1] - y)**2))

        # Points of interest labeled clockwise from marked corner
        self.p0 = self.outer_tup[0]
        self.p1 = self.outer_tup[1]
        self.p2 = self.outer_tup[3]
        # Furthest point must be the 3rd point
        self.p3 = self.outer_tup[2]

    def draw(self, img):
        """Draw the tag on a 3 channel image

        :param img:
            The 3 channel image
        :type img:
            numpy.ndarray
        :return:
            The input image
        :rtype:
            numpy.ndarray
        """
        img = cv2.drawContours(img, [self.outer_cont], -1, (0, 255, 0), 10)
        img = cv2.drawContours(img, [self.inner_cont], -1, (255, 0, 0), 10)
        img = cv2.circle(img, tuple(self.p0), 20, (0, 0, 255), 10)
        return img

    def ratio(self, img):
        """Ratio of the size of the Tag vs the input image size

        :param img:
            The input image
        :type img:
            numpy.ndarray
        :return:
            The processed image
        :rtype:
            numpy.ndarray
        """
        return cv2.contourArea(self.outer_cont) / float(img.shape[0] * img.shape[1])

    def in_outer(self):
        return cv2.contourArea(self.inner_cont) / cv2.contourArea(self.outer_cont)

    @staticmethod
    def dist(p0, p1):
        """ Return the distance between two tuples in the format (x, y)

        :param p0:
            The first point
        :type p0:
            Tuple[int, int]
        :param p1:
            The second point
        :type p1:
            Tuple[int, int]
        :return:
            The distance
        :rtype:
            float
        """
        return math.sqrt((p0[0] - p1[0])**2 + (p0[1] - p1[1])**2)

    def valid(self):
        """Check the validity of this artag
        Check the following items:
         - Any line distance is more than 1.8 times the average length
         - Any line distance is less than .2 times the average length
         - The inner contour takes up more than 10% of the tag
         - The inner contour takes up less than 1% of the tag

        :return:
            The validity of the tag
        :rtype:
            bool
        """
        lengths = [
            self.dist(self.p0, self.p1),
            self.dist(self.p1, self.p2),
            self.dist(self.p2, self.p3),
            self.dist(self.p3, self.p0)
        ]
        # check aspect ratio
        if any([i > 1.8 * sum(lengths) / len(lengths) for i in lengths]):
            return False
        if any([i < .2 * sum(lengths) / len(lengths) for i in lengths]):
            return False
        # Check area ratios
        if cv2.contourArea(self.inner_cont) / cv2.contourArea(self.outer_cont) > 0.1:
            return False
        if cv2.contourArea(self.inner_cont) / cv2.contourArea(self.outer_cont) < 0.01:
            return False
        return True

It seems to work great, except for image 11 seems to fail to detect the Tag, this has somthing to do with the nesting of the contours. What I would like to do to fix this image, would be to do somthing like find large 4-sided contours that have smaller 4 sided contours inside, that are not nested and have 4 sides.

The problem is the structure of the hierarchy object returned by opencv. It has the indexes of the contours, but if you modify the list it makes this useless. I would love to have a data structure for the contours, that have the data baked in, but the problem occurs if I start removing contours, becouse they are invalid or somthing I get problems referring to contours that are nonexistent.

Any input on solving this problem elegantly would be a great in addition to a general review of the code.

My code is hosted on github here: AR-Testing

You can download all the code with the test images here: AR-Testing.zip


Get this bounty!!!

#StackBounty: #python #recommender-system How to create a relational factorisation recommander?

Bounty: 50

I want to create a recommender system with factorisation recommander.
We have scores on hashtags, ads have scores on hashtags, and we even have scores of subscriber on ads.
We want to use this to recommend ads to subscriber.

  • Scores of suscriber on hastags (but not all of them) :

called df_p

    count  hashtag_id  subscriber_id
0      22         321            172
1      44         321            161
2      25         322            172
3       6         323            172
4      32         325            172
5      26         328            161
...   ...         ...            ...
  • Scores of ads on hashtags (but not all of them):

called df_q

      count  eclipse_id  hashtag_id
0         1        6521         321
1         1        6606         321
2         1        6609         321
3         1        6617         321
4         1        6649         321
5         1        6911         321
...      ...        ...         ...
  • Scores of subscriber on ad (still: not all of them):

How can I use GraphLab factorization_recommender to recommand ad to suscriber ?

The output should be (here for a given user 13):

subscriber_id   eclipse_id  score   rank
13  6565    0.059975420017  1
13  6588    0.0389804676959     2
13  9762    0.0159731995118     3
13  9606    0.0159731995118     4
13  9854    0.0159731995118     5
13  9576    0.0159731995118     6
13  9902    0.0155536116738     7
13  9875    0.0155536116738     8
13  6766    0.0126994707082     9
13  9870    0.0125380719963     10

On the GraphLab official website, I already found :

user_info = graphlab.SFrame({'user_id': ["0", "1", "2"],
                              'name': ["Alice", "Bob", "Charlie"],
                              'numeric_feature': [0.1, 12, 22]})
item_info = graphlab.SFrame({'item_id': ["a", "b", "c", d"],
                              'name': ["item1", "item2", "item3", "item4"],
                              'dict_feature': [{'a' : 23}, {'a' : 13},
                                               {'b' : 1},
                                               {'a' : 23, 'b' : 32}]})
m2 = graphlab.factorization_recommender.create(sf, target='rating',
                                                user_data=user_info,
                                                item_data=item_info)

But it seems not to apply to my case as far as here we have scores on user and scores on item. My project is more relational as described above.


I already tried like wihout the relational features :

products = pd.DataFrame(r_cols)
# We fill NaN cells with 0s
products['eclipse_id'] = products['eclipse_id'].fillna(0).astype(int)
# We transform it to an SFrame
sf = SFrame(data=products)
# We split data to train the model
ratings_base,ratings_test = sf.random_split(.8, seed=0)
# We train the model
# Train Model
personalized_model = graphlab.item_similarity_recommender.create(ratings_base, user_id='subscriber_id', item_id='eclipse_id', target='count', similarity_type='cosine')

Allowing me to recommend (but I don’t know on the basis of what) :

personalized_model.recommend(users=[users[0]])

Giving me back :

subscriber_id   eclipse_id  score   rank
172     11279   5.42914502761        1
172     11275   1.65434508464        2
172     0       0.139058996649       3
172     11268   0.0788922800737      4


Get this bounty!!!

#StackBounty: #python #visualization #correlation How to more simply see overlapping data for dozens of variables?

Bounty: 50

I’m trying to think of the best way to see how multiple variables (about 40) related to a very large userbase can be seen to interact with one another.

As an analogy, imagine I had survey data of superheros liked by thousands of school children, such that Adam Agner only likes Batman, Brian Agosto likes Superman and Batman, Cian Ailiff likes Wonderwoman, Superman and the Flash, etc. The index would be the list of children’s names (tens of thousands), and the variables would be each superhero (from a list of 50) each with a True or False value.

Rather than just the amount of children who like each superhero, if I can see the overlap information in some easy way, I might find that an unusually large number only like Batman, or that most of those who like Superman are likely to also like the Flash.

The easiest would be to do it visually, but Venn diagrams wouldn’t be practical with a large number (50 here) of variables. So it would seem like a correlation matrix with a heat-map base would be the way to go, with something like this:

Source: http://www.sthda.com/english/wiki/correlation-matrix-a-quick-start-guide-to-analyze-format-and-visualize-a-correlation-matrix-using-r-software

I can imagine a heat map with superheros plotted on both axes could work for seeing interesting matches of one variable against all other ones, but it would still require some less ideal extra steps to see when three or more superheros matched, like seeing the colour overlap of superhero1 with superhero2 & superhero3 both be high, and that’s kind of guesswork, because the actual children who like all 3 could still be low.

At the moment, the best solution I can think of is to try to reduce my variables down into categories, so the example could be Marvel, DC, male, female, but that loses some potential data that could be helpful from overlaps within those categories.

Maybe if I did something like the image above, the circle size would be the number for that overlap, and the colour could be number matches with other variables (without listing them, but I could further investigate). That kind of coding is a little out of my comfort zone, but I could try.

Ideas appreciated! I’d ideally do this in matplotlib or some other means within python, but if I have to use Matlab or another tool then I will consider that. Ideas or suggestions appreciated! Hopefully this was clear, thank you!


Get this bounty!!!

#StackBounty: #python #visualization #beginner How to more simply see overlapping data for dozens of variables?

Bounty: 50

I’m trying to think of the best way to see how multiple variables (about 40) related to a very large userbase can be seen to interact with one another.

As an analogy, imagine I had survey data of superheros liked by thousands of school children, such that Adam Agner only likes Batman, Brian Agosto likes Superman and Batman, Cian Ailiff likes Wonderwoman, Superman and the Flash, etc. The index would be the list of children’s names (tens of thousands), and the variables would be each superhero (from a list of 50) each with a True or False value.

Rather than just the amount of children who like each superhero, if I can see the overlap information in some easy way, I might find that an unusually large number only like Batman, or that most of those who like Superman are likely to also like the Flash.

The easiest would be to do it visually, but Venn diagrams wouldn’t be practical with a large number (50 here) of variables. So it would seem like a correlation matrix with a heat-map base would be the way to go, with something like this:

Source: http://www.sthda.com/english/wiki/correlation-matrix-a-quick-start-guide-to-analyze-format-and-visualize-a-correlation-matrix-using-r-software

I can imagine a heat map with superheros plotted on both axes could work for seeing interesting matches of one variable against all other ones, but it would still require some less ideal extra steps to see when three or more superheros matched, like seeing the colour overlap of superhero1 with superhero2 & superhero3 both be high, and that’s kind of guesswork, because the actual children who like all 3 could still be low.

At the moment, the best solution I can think of is to try to reduce my variables down into categories, so the example could be Marvel, DC, male, female, but that loses some potential data that could be helpful from overlaps within those categories.

Maybe if I did something like the image above, the circle size would be the number for that overlap, and the colour could be number matches with other variables (without listing them, but I could further investigate). That kind of coding is a little out of my comfort zone, but I could try.

Ideas appreciated! I’m most experienced with Python & pandas (it’s how I merge the data), but if I have to use Matlab or another tool then I will consider that. Ideas or suggestions appreciated! Hopefully this was clear, thank you!


Get this bounty!!!

#StackBounty: #python #machine-learning #tensorflow #neural-network #keras Get audiences insights using Keras and TensorFlow

Bounty: 50

Recently I’ve discovered Keras and TensorFlow and I’m trying to get into ML. I have manually classified train and test data from my users DB like so:

9 features and a label, the features are events in my system like “user added a profile picture” or “user paid X for a service” and the label is positive or negative R.O.I (1 or 0)

Sample:
enter image description here

I have used the following code to classify the users:

import numpy as np
from keras.layers import Dense
from keras.models import Sequential

train_data = np.loadtxt("train.csv", delimiter=",", skiprows=1)
test_data = np.loadtxt("test.csv", delimiter=",", skiprows=1)

X_train = train_data[:, 0:9]
Y_train = train_data[:, 9]

X_test = test_data[:, 0:9]
Y_test = test_data[:, 9]

model = Sequential()
model.add(Dense(8, input_dim=9, activation='relu'))
model.add(Dense(6, activation='relu'))
model.add(Dense(3, activation='relu'))
model.add(Dense(1, activation='sigmoid'))

# Compile model
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

# Fit the model
model.fit(X_train, Y_train, epochs=12000, batch_size=10)

# evaluate the model
scores = model.evaluate(X_test, Y_test)
print("nnnResults: %s: %.2f%%" % (model.metrics_names[1], scores[1]*100))

And got a 89% accuracy. That worked great in order to label a user as a valued customer.

Q : How can I extract the features that contributed for the possitive R.O.I so I can boost their focus in the UX?

Or : What is the approach to find the best combined segment of audiences?


Get this bounty!!!

#StackBounty: #python #object-oriented #python-3.x #inheritance #optional Implementing a Maybe for multidimensional data

Bounty: 300

The scenario is we’re working with a REST endpoint that gives us nice JSON objects. We’re using requests, and everything works wonderfully. But one day you notice that data you though was always being pulled back isn’t there for some models! You’re getting index, key and attribute errors left, right and centre. What used to be obj.phones[0] is now:

(DefaultAttr(obj, None).phones or [None])[0]

Or worse, if obj.phones[0] could actually be None!

And so I decided to build a simple Maybe like group of classes. The interface in Haskell seemed really clean and simple. However there’s no case statement in Python, and not one that’s based on type. so I decided to use Maybe.get(default: T) -> T and Just._value to get the value instead. Since Just._value isn’t implemented on either Nothing or Maybe, I decided to make the equality check if the LHS is an instance of the RHS, and so Just(1) == Just would be True.

This however isn’t good enough for the JSON object, and so I subclassed Maybe to create a MaybeNav, that creates a MaybeNav when you get items or attributes. And so if you don’t care if obj.phones[0] is there or None, you can use MaybeNav(obj).phones[0].get(). Which is much cleaner, and simpler to read.

I’ve added type hints to the code, so that it’s a little simpler to reason with. And I’ve added some docstrings, I’d appreciate any way to improve these, as I don’t normally use them. And so my skills with them are likely to be very poor.

Due to using f-strings, the code only works in Python 3.6. Finally any and all help is welcome.

mayby.py

from typing import TypeVar, Generic

T = TypeVar('T')
class Maybe(Generic[T]):
    """Simple Maybe abstract base class"""
    def __init__(self, *args, **kwargs):
        """
        Error as this should be overwridden

        The code for this interface is implemented in Just and Nothing.
        This is to prevent implementation errors. Such as overwritting
        __getattr__ in a child class causing an infinate loop.
        """
        raise TypeError("'Maybe' needs to be instansiated by child constructor")

    def get(self, default: T = None) -> T:
        """Get the value in Just, or return default if Nothing"""
        raise NotImplementedError("'get' should be changed in a sublcass")


class Constant(type):
    def __call__(self, *args, **kwargs):
        """Simple metaclass to create a constant class"""
        # Constants can't be called.
        raise TypeError(f"'{self.__name__}' object is not callable")

    def __repr__(self) -> str:
        # Display the constant, rather than the class location in memory.
        return f'{self.__name__}'


class JustBase:
    def __init__(self, value: T):
        """Younger sibling class of Maybe for Just classes"""
        self.__value = value

    def __repr__(self) -> str:
        return f'{type(self).__name__}({self._value!r})'

    def __eq__(self, other: object) -> bool:
        """
        Check if this is an instance of other

        This makes checking if the class is a Just simpler.
        As it's a common operation.
        """
        return isinstance(other, type) and isinstance(self, other)

    def get(self, default: T = None) -> T:
        """Get the value in Just, or return default if Nothing"""
        return self._value

    @property
    def _value(self):
        return self.__value


def build_maybes(Maybe, just_name, nothing_name):
    """Build a Just and Nothing inheriting from Maybe"""
    Just = type(just_name, (JustBase, Maybe), {})
    class MaybeConstant(Constant, Maybe):
        def get(self, default: T = None) -> T:
            """Get the value in Just, or return default if Nothing"""
            return default
    Nothing = MaybeConstant(nothing_name, (object,), {})
    return Just, Nothing


class MaybeNav(Maybe[T]):
    """Maybe for navigating objects"""
    # __getitem__ and __getattr__ actually return MaybeNav[T].
    def __getitem__(self, item: object) -> Maybe[T]:
        if self == NothingNav:
            return NothingNav
        val = self._value
        try:
            val = val[item]
        except Exception:
            return NothingNav
        else:
            return JustNav(val)

    def __getattr__(self, item: str) -> Maybe[T]:
        obj = object()
        val = getattr(self.get(obj), item, obj)
        if val is obj:
            return NothingNav
        return JustNav(val)


Just, Nothing = build_maybes(Maybe, 'Just', 'Nothing')
JustNav, NothingNav = build_maybes(MaybeNav, 'JustNav', 'NothingNav')

# Don't delete T, so subclasses can use the same generic type
del TypeVar, Generic, Constant, JustBase

An example of using this can be:

import math
from collections import namedtuple
from typing import Iterable

from maybe import Maybe, Just, Nothing, MaybeNav, JustNav

def safe_log(number: float) -> Maybe[float]:
    if number > 0:
        return Just(math.log(number))
    else:
        return Nothing


def user_values(obj: object) -> Iterable[MaybeNav[object]]:
    obj = JustNav(obj)
    return [
        obj.name,
        obj.first_name,
        obj.last_name,
        obj.phones[0]
    ]


v = safe_log(1000)
print(v == Just, v)
v = safe_log(-1000)
print(v == Just, v)

User = namedtuple('User', 'name first_name last_name phones')
vals = user_values(User('Peilonrayz', 'Peilonrayz', None, []))
print(vals)
print([val.get(None) for val in vals])

I find maybe.py a little hard to read, mostly as I got the inheritance to happen in build_maybes. And so the following is a class diagram of the user defined objects in maybe.py. One thing to note is the dotted arrows from Nothing and NothingNav to MaybeConstant are to show they are using a metaclass, rather than box standard inheritance. The box with some classes in it is to emphasise they are created in build_maybes.

Class diagram


Get this bounty!!!

#StackBounty: #python #rabbitmq #celery start celery worker and enable it for broadcast queue

Bounty: 50

I’m trying to start celery worker so it only listens to single queue. This is not a problem, I can do this that way:

python -m celery worker -A my_module -Q my_queue -c 1

But now I also want this my_queue queue to be a broadcast queue, so I do this in my celeryconfig:

from kombu.common import Broadcast
CELERY_QUEUES = (Broadcast('my_queue'),)

But as soon as I do this I cannot start my worker anymore, I get error from rabbitmq:

amqp.exceptions.PreconditionFailed: Exchange.declare: (406) PRECONDITION_FAILED - inequivalent arg 'type' for exchange 'my_queue' in vhost 'myvhost': received 'fanout' but current is 'direct'

If I start worker without -Q (but leaving Broadcast in celeryconfig.py as described above) and I list rabbitmq queues I can see broadcast queue is created and named like this:

bcast.43fecba7-786a-461c-a322-620039b29b8b

And similarly if I define this queue within worker (using -Q as mentioned above) or as simple Queue in celeryconfig.py like this:

from kombu import Queue
CELERY_QUEUES = (Queue('my_queue'),)

I can see this queue in rabbitmq like this:

my_queue

It apperas it does not matter what I put into Broadcast call when defining the queue – this seems to be internal celery name, not passed to rabbitmq.

So I’m guessing when worker is starting then my_queue is created and once that’s done it cannot be made Broadcast.

I can have a worker that listens to any queue (not only to my_queue) which I would start by removing the -Q argument. But it would be nice to be able to have a single process that only listens to that particular queue since my tasks I throw in there are fast and I’d like to bring latency down as much as possible.

— Edit 1 —
Spent some time with this problem and it seems bcast queue mentioned above does not appear consistently. After reseting rabbitmq and running celery without -Q option bcast queue did not appear…


Get this bounty!!!

#StackBounty: #python #sockets #ssl Python/sockets/ssl EOF occurred in violation of protocol

Bounty: 150

I would like to authenticate the server at client’s side in my echo client/server program. I’m using python 2.7.12 and the ssl module on

Distributor ID: Ubuntu
Description:    Ubuntu 14.04.5 LTS
Release:        14.04
Codename:       trusty

I’ve generated client’s and server’s certificates and keys using the openssl commands:

openssl req -new -x509 -days 365 -nodes -out client.pem -keyout client.key
openssl req -new -x509 -days 365 -nodes -out server.pem -keyout server.key

Versions of openssl library itself and openssl used by python are the same:

openssl version -a
OpenSSL 1.0.1f 6 Jan 2014
built on: Fri Sep 23 12:19:57 UTC 2016
platform: debian-amd64
options:  bn(64,64) rc4(16x,int) des(idx,cisc,16,int) blowfish(idx) 
compiler: cc -fPIC -DOPENSSL_PIC -DOPENSSL_THREADS -D_REENTRANT -DDSO_DLFCN -DHAVE_DLFCN_H -m64 -DL_ENDIAN -DTERMIO -g -O2 -fstack-protector --param=ssp-buffer-size=4 -Wformat -Werror=format-security -D_FORTIFY_SOURCE=2 -Wl,-Bsymbolic-functions -Wl,-z,relro -Wa,--noexecstack -Wall -DMD32_REG_T=int -DOPENSSL_IA32_SSE2 -DOPENSSL_BN_ASM_MONT -DOPENSSL_BN_ASM_MONT5 -DOPENSSL_BN_ASM_GF2m -DSHA1_ASM -DSHA256_ASM -DSHA512_ASM -DMD5_ASM -DAES_ASM -DVPAES_ASM -DBSAES_ASM -DWHIRLPOOL_ASM -DGHASH_ASM
OPENSSLDIR: "/usr/lib/ssl"

python -c "import ssl; print ssl.OPENSSL_VERSION"
OpenSSL 1.0.1f 6 Jan 2014

However, the code below shows some errors, at server’s side: EOF occurred in violation of protocol (_ssl.c:1645) (but the server still works), and at client’s side:

Traceback (most recent call last):
  File "/http_ssl_client.py", line 36, in <module>
    if not cert or ('commonName', 'test') not in cert['subject'][4]: raise Exception("Invalid SSL cert for host %s. Check if this is a man-in-themiddle attack!" )
Exception: Invalid SSL cert for host %s. Check if this is a man-in-themiddle attack!
{'notBefore': u'Jun  3 11:54:21 2017 GMT', 'serialNumber': u'BBDCBEED69655B6E', 'notAfter': 'Jun  3 11:54:21 2018 GMT', 'version': 3L, 'subject': ((('countryName', u'pl'),), (('stateOrProvinceName', u'test'),), (('localityName', u'test'),), (('organizationName', u'test'),), (('organizationalUnitName', u'test'),), (('commonName', u'test'),), (('emailAddress', u'test'),)), 'issuer': ((('countryName', u'pl'),), (('stateOrProvinceName', u'test'),), (('localityName', u'test'),), (('organizationName', u'test'),), (('organizationalUnitName', u'test'),), (('commonName', u'test'),), (('emailAddress', u'test'),))}

Server’s code:

#!/bin/usr/env python
import socket
import ssl

def main():
    HOST = '127.0.0.1'
    PORT = 1234

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    sock.bind((HOST, PORT))
    sock.listen(5)

    while True:
        conn = None
        client_sock, addr = sock.accept()
        try:
            ssl_client = ssl.wrap_socket(client_sock, server_side=True, certfile="server.pem", keyfile="server.key", ssl_version=ssl.PROTOCOL_TLSv1_2)
            data =  ssl_client.read(1024)
            print data
            ssl_client.write(data)
        except ssl.SSLError as e:
            print(e)
        finally:
            if conn:
                conn.close()
if __name__ == '__main__':
    main()

Client:

#!/bin/usr/env python
import socket
import ssl

if __name__ == '__main__':

    HOST = '127.0.0.1'
    PORT = 1234

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.connect((HOST, PORT))

    context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
    context.verify_mode = ssl.CERT_REQUIRED
    context.load_verify_locations('server.pem')

    if ssl.HAS_SNI:
        secure_sock = context.wrap_socket(sock, server_hostname=HOST)
    else:
        secure_sock = context.wrap_socket(sock)

    cert = secure_sock.getpeercert()
    print cert

    if not cert or ('commonName', 'test') not in cert['subject'][4]: raise Exception("Error" )

    secure_sock.write('hello')

    print secure_sock.read(1024)

    secure_sock.close()
    sock.close()

All files are in the same directory.


Get this bounty!!!