#StackBounty: #python #performance #benchmarking #pypy Accurately testing Pypy vs CPython performance

Bounty: 50

The Problem Description:

I have this custom “checksum” function:

NORMALIZER = 0x10000


def get_checksum(part1, part2, salt="trailing"):
    """Returns a checksum of two strings."""

    combined_string = part1 + part2 + " " + salt if part2 != "***" else part1
    ords = [ord(x) for x in combined_string]

    checksum = ords[0]  # initial value

    # TODO: document the logic behind the checksum calculations
    iterator = zip(ords[1:], ords)
    checksum += sum(x + 2 * y if counter % 2 else x * y
                    for counter, (x, y) in enumerate(iterator))
    checksum %= NORMALIZER

    return checksum

Which I want to test on both Python3.6 and PyPy performance-wise. I’d like to see if the function would perform better on PyPy, but I’m not completely sure, what is the most reliable and clean way to do it.

What I’ve tried and the Question:

Currently, I’m using timeit for both:

$ python3.6 -mtimeit -s "from test import get_checksum" "get_checksum('test1' * 100000, 'test2' * 100000)"
10 loops, best of 3: 329 msec per loop

$ pypy -mtimeit -s "from test import get_checksum" "get_checksum('test1' * 100000, 'test2' * 100000)"
10 loops, best of 3: 104 msec per loop

My concern is I’m not absolutely sure if timeit is the right tool for the job on PyPy because of the potential JIT warmup overhead.

Plus, the PyPy itself reports the following before reporting the test results:

WARNING: timeit is a very unreliable tool. use perf or something else for real measurements
pypy -m pip install perf
pypy -m perf timeit -s 'from test import get_checksum' "get_checksum('test1' * 1000000, 'test2' * 1000000)"

What would be the best and most accurate approach to test the same exact function performance across these and potentially other Python implementations?


Get this bounty!!!

#StackBounty: #python #numpy #matplotlib #fractals Plotting the Mandelbrot set at different zoom levels

Bounty: 50

I’m interested in making an animated movie of a zoom in on a part of the Mandelbrot set. My code works well for a few zooms, but upon trying to zoom in quite far, I find that the fractal becomes “smoothed out”. Am I missing something that prevents me from seeing the fractal structure at higher zoom levels? Am I hitting machine precision in the computations? Here’s what I’m talking about:

Zoom level 3:
zoom level 3
Zoom level 9:
zoom level 9
Zoom level 20:
zoom level 20

The first plot looks good, the second is okay, and the third is not fractal at all.

If there are any other deficiencies or improvements I’d be glad to hear about them as well.

Here’s my code:

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from matplotlib import animation
import time

# some interesting places in the set
# http://www.nahee.com/Derbyshire/manguide.html

N = 500
nIts = 25
nZooms = 50
x0=0
y0=-1

movie = np.zeros([N,N,nZooms])

x = np.linspace(-2,1,N)
y = np.linspace(-1,1,N)
X,Y = np.meshgrid(x,y)
c = X + 1j*Y
z = 0*c
for i in range(nIts):
    z = z**2 + c

mask = np.abs(z) < 1
z[z>1]=0
z[np.isnan(z)]=0
movie[:,:,0] = mask


# plotting stuff
for j in range(1,nZooms):
    h=1./(2**j)
    print "Plot number ", j
    x = x0+h*np.linspace(-1,1,N)
    y = y0+h*np.linspace(-1,1,N)
    X,Y = np.meshgrid(x,y)
    c = X + 1j*Y
    z = 0*c
    for i in range(nIts):
        z = z**2 + c

    mask = np.abs(z) < 1
    z[z>1]=0
    z[np.isnan(z)]=0
    movie[:,:,j] = mask

fig = plt.figure()

for j in range(nZooms):
    name = "image%d.png" % j
    plt.imshow(movie[:,:,j], cmap = 'RdBu')
    plt.gray()
    plt.axis('equal')
    plt.axis('off')
    # plt.show()
    fig.savefig(name)
    time.sleep(1)


Get this bounty!!!

#StackBounty: #python Plotting the Mandelbrot set at different zoom levels

Bounty: 50

I’m interested in making an animated movie of a zoom in on a part of the Mandelbrot set. My code works well for a few zooms, but upon trying to zoom in quite far, I find that the fractal becomes “smoothed out”. Am I missing something that prevents me from seeing the fractal structure at higher zoom levels? Am I hitting machine precision in the computations? Here’s what I’m talking about:

Zoom level 3:
zoom level 3
Zoom level 9:
zoom level 9
Zoom level 20:
zoom level 20

The first plot looks good, the second is okay, and the third is not fractal at all.

If there are any other deficiencies or improvements I’d be glad to hear about them as well.

Here’s my code:

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from matplotlib import animation
import time

# some interesting places in the set
# http://www.nahee.com/Derbyshire/manguide.html

N = 500
nIts = 25
nZooms = 50
x0=0
y0=-1

movie = np.zeros([N,N,nZooms])

x = np.linspace(-2,1,N)
y = np.linspace(-1,1,N)
X,Y = np.meshgrid(x,y)
c = X + 1j*Y
z = 0*c
for i in range(nIts):
    z = z**2 + c

mask = np.abs(z) < 1
z[z>1]=0
z[np.isnan(z)]=0
movie[:,:,0] = mask


# plotting stuff
for j in range(1,nZooms):
    h=1./(2**j)
    print "Plot number ", j
    x = x0+h*np.linspace(-1,1,N)
    y = y0+h*np.linspace(-1,1,N)
    X,Y = np.meshgrid(x,y)
    c = X + 1j*Y
    z = 0*c
    for i in range(nIts):
        z = z**2 + c

    mask = np.abs(z) < 1
    z[z>1]=0
    z[np.isnan(z)]=0
    movie[:,:,j] = mask

fig = plt.figure()

for j in range(nZooms):
    name = "image%d.png" % j
    plt.imshow(movie[:,:,j], cmap = 'RdBu')
    plt.gray()
    plt.axis('equal')
    plt.axis('off')
    # plt.show()
    fig.savefig(name)
    time.sleep(1)


Get this bounty!!!

#StackBounty: #org-babel #python Get pandas data-frame as a table in org-babel

Bounty: 50

In the following example, I create a very simple pandas DataFrame and export results verbatim:

#+BEGIN_SRC python :exports both :results verbatim
import pandas as pd
test = pd.DataFrame({'A': [1000, 1000], 'B' : [60, 100]})
return test
#+END_SRC

#+RESULTS:
:       A    B
: 0  1000   60
: 1  1000  100

If I try to have the output as a table, things get messed up:

#+BEGIN_SRC python :exports both :results table
import pandas as pd
test = pd.DataFrame({'A': [1000, 1000], 'B' : [60, 100]})
return test
#+END_SRC

#+RESULTS:
| A    B |

Is there any way to fix this?

Expected output:

|   |    A |   B |
| 0 | 1000 |  60 |
| 1 | 1000 | 100 |


Get this bounty!!!

#StackBounty: #python #google-cloud-platform #google-speech-api Google speech service – not returning transcription

Bounty: 50

I am using the sample code provided here and have implemented with the following:

# [START import_libraries]
import argparse
import base64
import json
import time
from oauth2client.service_account import ServiceAccountCredentials
import googleapiclient.discovery
import googleapiclient as gac
# [END import_libraries]


# [START authenticating]


# Application default credentials provided by env variable
# GOOGLE_APPLICATION_CREDENTIALS
def get_speech_service(credentials):
    return googleapiclient.discovery.build('speech', 'v1beta1',credentials = credentials)



def main(speech_file):
    """Transcribe the given audio file asynchronously.
    Args:
        speech_file: the name of the audio file.
    """
    # [START construct_request]
    with open(speech_file, 'rb') as speech:
        # Base64 encode the binary audio file for inclusion in the request.
        speech_content = base64.b64encode(speech.read())

#    print speech_content

    scopes = ['https://www.googleapis.com/auth/cloud-platform']

    credentials = ServiceAccountCredentials.from_json_keyfile_name(
        '/Users/user/Documents/google_cloud/myjson.json', scopes)

    service = get_speech_service(credentials)
    service_request = service.speech().asyncrecognize(
        body={
            'config': {
                # There are a bunch of config options you can specify. See
                # https://cloud.google.com/speech/reference/rest/v1beta1/RecognitionConfig for the full list.
                'encoding': 'LINEAR16',  # raw 16-bit signed LE samples
                'sampleRate': 16000,  # 16 khz
                # See http://g.co/cloud/speech/docs/languages for a list of
                # supported languages.
                'languageCode': 'en-US',  # a BCP-47 language tag
            },
            'audio': {
                'content': speech_content.decode('UTF-8')
                }
            })
    # [END construct_request]
    # [START send_request]
    response = service_request.execute()
    print(json.dumps(response))
    # [END send_request]

    name = response['name']
    # Construct a GetOperation request.
    service_request = service.operations().get(name=name)

    while True:
        # Give the server a few seconds to process.
        print('Waiting for server processing...')
        time.sleep(1)
        # Get the long running operation with response.
        response = service_request.execute()

        if 'done' in response and response['done']:
            break

    # First print the raw json response
    print(json.dumps(response['response'], indent=2))

    # Now print the actual transcriptions
    out = []
    for result in response['response'].get('results', []):
        print 'poo'
        print('Result:')
        for alternative in result['alternatives']:
            print(u'  Alternative: {}'.format(alternative['transcript']))
        out.append(result)
    return response

r = main("/Users/user/Downloads/brooklyn.flac")

Yet my print is the following:

{"name": "3202776140236290963"}
Waiting for server processing...
Waiting for server processing...
{
  "@type": "type.googleapis.com/google.cloud.speech.v1beta1.AsyncRecognizeResponse"
}

And my returned object is:

{u'done': True,
 u'metadata': {u'@type': u'type.googleapis.com/google.cloud.speech.v1beta1.AsyncRecognizeMetadata',
  u'lastUpdateTime': u'2017-03-25T15:54:46.136925Z',
  u'progressPercent': 100,
  u'startTime': u'2017-03-25T15:54:44.514614Z'},
 u'name': u'2024312474309214820',
 u'response': {u'@type': u'type.googleapis.com/google.cloud.speech.v1beta1.AsyncRecognizeResponse'}}

On my console screen I see the requests coming through via:
enter image description here

Unsure why I am not getting the proper transcription back from the sample file.

Any input is appreciated!


Get this bounty!!!

#StackBounty: #python #performance #python-3.x #sorting #heap Sorting almost sorted array with a minimum heap

Bounty: 50

I’m currently looking into the quickest way to sort an almost sorted array:

Given an array of $n$ elements, where each element is at most $k$ away
from its target position, devise an algorithm that sorts in $O(n log k)$
time.

I’ve implemented a sorting function which “slides” through an input list, pushes the element from a sliding window into a min heap (using the heapq built-in module), pops the minimum element which is collected in the result list:

from typing import List
from heapq import heappush, heappop


def sort_almost_sorted(a: List[int], k: int) -> List[int]:
    if not a:
        return []

    length = len(a)
    if k >= length:
        return sorted(a)  # apply built-in "timsort", designed to be quick on almost sorted lists

    result = []
    min_heap = []  # maintain a min heap for a sliding window
    for slice_start in range(0, length, k + 1):  # sliding window
        # push the next slice to min heap
        for index in range(slice_start, min(slice_start + k + 1, length)):
            heappush(min_heap, a[index])

        result.append(heappop(min_heap))

    # put the rest of the heap into the result
    for _ in range(len(min_heap)):
        result.append(heappop(min_heap))

    return result

It works on my sample inputs.

Do you think I’m using the minimum heap appropriately and this implementation is $O(n log k)$? What would you improve code-quality or code-organization wise?


Get this bounty!!!

#StackBounty: #python #json #webhooks #api-ai Python Webhook: Passing through a URL + payload

Bounty: 50

I’m a beginner with Python and trying to build a service that takes information from api.ai, passes it to an API, then returns a confirmation message from the JSON it returns.

app.py:

def processRequest(req):
    if req.get("result").get("action") != "bookMyConference":
        return {}

    joinme_query = scheduleNewConference(req)
    if joinme_query is None:
        return {}

    #oauth
    orequest = req.get("originalRequest") # work down the tree
    odata = orequest.get("data") # work down the tree
    user = odata.get("user") # work down the tree
    access_token = user.get("access_token")

    #data
    result = req.get("result") # work down the tree
    parameters = result.get("parameters") # work down the tree
    startdate = parameters.get("start-date")

    payload = {
        "start-date": startdate
    }

    # POST info to join.me
    baseurl = "https://api.join.me/v1/meetings"
    p = Request(baseurl)
    p.add_header('Content-Type', 'application/json; charset=utf-8')
    p.add_header('Authorization', 'Bearer ' + access_token) #from oauth
    jsondata = json.dumps(payload)
    jsondataasbytes = jsondata.encode('utf-8')   # needs to be bytes
    result = urlopen(p, jsondataasbytes).read()
    data = json.loads(result)
    res = makeWebhookResult(data)
    return res

Edit 4: Here’s the error I’m getting in my Heroku logs:

2017-03-21T19:06:09.383612+00:00 app[web.1]: HTTPError: HTTP Error
400: Bad Request


Get this bounty!!!

#StackBounty: #python #python-2.7 #debugging #gdb #deadlock Debugging running python app using gdb in python mode does not work

Bounty: 50

I have a multithreaded python application running using virtualenv and I can not properly attach to it using gdb. I want to debug the deadlock and I think this is the only way I can do this with because the app is too huge and I can not reproduce the issue in develop environment.
I have installed python2.7-dbg and then trying to run debugging with gdb /home/PATHTOVENV/venv/bin/python 30183. Here is the output:

GNU gdb (Ubuntu 7.7.1-0ubuntu5~14.04.2) 7.7.1
Copyright (C) 2014 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.  Type "show copying"
and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
<http://www.gnu.org/software/gdb/documentation/>.
For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from /home/PATHTOVENV/venv/bin/python...(no debugging symbols found)...done.
Attaching to program: /home/PATHTOVENV/venv/bin/python, process 30183
Reading symbols from /lib/x86_64-linux-gnu/libpthread.so.0...Reading symbols from /usr/lib/debug//lib/x86_64-linux-gnu/libpthread-2.19.so...done.
done.
[New LWP 30326]
[New LWP 30325]
[New LWP 30324]
[New LWP 30323]
[New LWP 30322]
[New LWP 30321]
[New LWP 30320]
[New LWP 30319]
[New LWP 30318]
[New LWP 30317]
[New LWP 30316]
[New LWP 30315]
[New LWP 30314]
[New LWP 30313]
[New LWP 30312]
[New LWP 30311]
[New LWP 30310]
[New LWP 30309]
[New LWP 30308]
[New LWP 30307]
[New LWP 30306]
[New LWP 30305]
[New LWP 30304]
[New LWP 30302]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Loaded symbols for /lib/x86_64-linux-gnu/libpthread.so.0
Reading symbols from /lib/x86_64-linux-gnu/libc.so.6...Reading symbols from /usr/lib/debug//lib/x86_64-linux-gnu/libc-2.19.so...done.
done.
Loaded symbols for /lib/x86_64-linux-gnu/libc.so.6
Reading symbols from /lib/x86_64-linux-gnu/libdl.so.2...Reading symbols from /usr/lib/debug//lib/x86_64-linux-gnu/libdl-2.19.so...done.
done.
Loaded symbols for /lib/x86_64-linux-gnu/libdl.so.2
Reading symbols from /lib/x86_64-linux-gnu/libutil.so.1...Reading symbols from /usr/lib/debug//lib/x86_64-linux-gnu/libutil-2.19.so...done.
done.
Loaded symbols for /lib/x86_64-linux-gnu/libutil.so.1
Reading symbols from /lib/x86_64-linux-gnu/libz.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libz.so.1
Reading symbols from /lib/x86_64-linux-gnu/libm.so.6...Reading symbols from /usr/lib/debug//lib/x86_64-linux-gnu/libm-2.19.so...done.
done.
Loaded symbols for /lib/x86_64-linux-gnu/libm.so.6
Reading symbols from /lib64/ld-linux-x86-64.so.2...Reading symbols from /usr/lib/debug//lib/x86_64-linux-gnu/ld-2.19.so...done.
done.
Loaded symbols for /lib64/ld-linux-x86-64.so.2
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/lxml/etree.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/lxml/etree.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libxslt.so.1...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libxslt.so.1
Reading symbols from /usr/lib/x86_64-linux-gnu/libexslt.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libexslt.so.0
Reading symbols from /usr/lib/x86_64-linux-gnu/libxml2.so.2...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libxml2.so.2
Reading symbols from /lib/x86_64-linux-gnu/libgcrypt.so.11...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libgcrypt.so.11
Reading symbols from /lib/x86_64-linux-gnu/liblzma.so.5...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/liblzma.so.5
Reading symbols from /lib/x86_64-linux-gnu/libgpg-error.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libgpg-error.so.0
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_hashlib.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/_hashlib.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_hashlib.x86_64-linux-gnu.so
Reading symbols from /lib/x86_64-linux-gnu/libcrypto.so.1.0.0...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libcrypto.so.1.0.0
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/datetime.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/datetime.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/datetime.x86_64-linux-gnu.so
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_ctypes.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/_ctypes.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_ctypes.x86_64-linux-gnu.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libffi.so.6...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libffi.so.6
Reading symbols from /lib/x86_64-linux-gnu/libuuid.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libuuid.so.1
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_ssl.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/_ssl.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_ssl.x86_64-linux-gnu.so
Reading symbols from /lib/x86_64-linux-gnu/libssl.so.1.0.0...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libssl.so.1.0.0
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/bson/_cbson.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/bson/_cbson.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/pymongo/_cmessage.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/pymongo/_cmessage.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/greenlet.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/greenlet.so
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_csv.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/_csv.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_csv.x86_64-linux-gnu.so
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_json.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/_json.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_json.x86_64-linux-gnu.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/cchardet/_cchardet.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/cchardet/_cchardet.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libstdc++.so.6...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libstdc++.so.6
Reading symbols from /lib/x86_64-linux-gnu/libgcc_s.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libgcc_s.so.1
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/_yaml.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/_yaml.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libyaml-0.so.2...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libyaml-0.so.2
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_elementtree.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/_elementtree.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_elementtree.x86_64-linux-gnu.so
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/pyexpat.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/pyexpat.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/pyexpat.x86_64-linux-gnu.so
Reading symbols from /lib/x86_64-linux-gnu/libexpat.so.1...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libexpat.so.1
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/datrie.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/datrie.so
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/parser.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/parser.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/parser.x86_64-linux-gnu.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/simplejson/_speedups.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/simplejson/_speedups.so
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_multiprocessing.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/_multiprocessing.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_multiprocessing.x86_64-linux-gnu.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/thrift/protocol/fastbinary.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/thrift/protocol/fastbinary.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/hiredis/hiredis.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/hiredis/hiredis.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/tornado/speedups.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/tornado/speedups.so
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_curses.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/_curses.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_curses.x86_64-linux-gnu.so
Reading symbols from /lib/x86_64-linux-gnu/libncursesw.so.5...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libncursesw.so.5
Reading symbols from /lib/x86_64-linux-gnu/libtinfo.so.5...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libtinfo.so.5
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/zope/interface/_zope_interface_coptimizations.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/zope/interface/_zope_interface_coptimizations.so
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/termios.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/termios.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/termios.x86_64-linux-gnu.so
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/resource.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/resource.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/resource.x86_64-linux-gnu.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/_psutil_linux.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/_psutil_linux.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/_psutil_posix.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/_psutil_posix.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/_pydevd_bundle/pydevd_cython.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/_pydevd_bundle/pydevd_cython.so
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_sqlite3.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/_sqlite3.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_sqlite3.x86_64-linux-gnu.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libsqlite3.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libsqlite3.so.0
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_lsprof.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/_lsprof.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/_lsprof.x86_64-linux-gnu.so
Reading symbols from /lib/x86_64-linux-gnu/libnss_files.so.2...Reading symbols from /usr/lib/debug//lib/x86_64-linux-gnu/libnss_files-2.19.so...done.
done.
Loaded symbols for /lib/x86_64-linux-gnu/libnss_files.so.2
Reading symbols from /lib/x86_64-linux-gnu/libnss_dns.so.2...Reading symbols from /usr/lib/debug//lib/x86_64-linux-gnu/libnss_dns-2.19.so...done.
done.
Loaded symbols for /lib/x86_64-linux-gnu/libnss_dns.so.2
Reading symbols from /lib/x86_64-linux-gnu/libresolv.so.2...Reading symbols from /usr/lib/debug//lib/x86_64-linux-gnu/libresolv-2.19.so...done.
done.
Loaded symbols for /lib/x86_64-linux-gnu/libresolv.so.2
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/dawg.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/dawg.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/PIL/_imaging.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/PIL/_imaging.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libjpeg.so.8...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libjpeg.so.8
Reading symbols from /usr/lib/x86_64-linux-gnu/libtiff.so.5...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libtiff.so.5
Reading symbols from /usr/lib/x86_64-linux-gnu/libjbig.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libjbig.so.0
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/mmap.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/mmap.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/mmap.x86_64-linux-gnu.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/maxminddb/extension.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/maxminddb/extension.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libmaxminddb.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libmaxminddb.so.0
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/core/multiarray.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/core/multiarray.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/core/umath.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/core/umath.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/core/_dotblas.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/core/_dotblas.so
Reading symbols from /usr/lib/libcblas.so.3...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/libcblas.so.3
Reading symbols from /usr/lib/libatlas.so.3...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/libatlas.so.3
Reading symbols from /usr/lib/x86_64-linux-gnu/libgfortran.so.3...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libgfortran.so.3
Reading symbols from /usr/lib/x86_64-linux-gnu/libquadmath.so.0...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libquadmath.so.0
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/core/scalarmath.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/core/scalarmath.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/lib/_compiled_base.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/lib/_compiled_base.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/linalg/lapack_lite.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/linalg/lapack_lite.so
Reading symbols from /usr/lib/liblapack.so.3...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/liblapack.so.3
Reading symbols from /usr/lib/libf77blas.so.3...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/libf77blas.so.3
Reading symbols from /usr/lib/libblas.so.3...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/libblas.so.3
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/linalg/_umath_linalg.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/linalg/_umath_linalg.so
Reading symbols from /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/future_builtins.x86_64-linux-gnu.so...Reading symbols from /usr/lib/debug/usr/lib/python2.7/lib-dynload/future_builtins.x86_64-linux-gnu.so...done.
done.
Loaded symbols for /home/PATHTOVENV/venv/lib/python2.7/lib-dynload/future_builtins.x86_64-linux-gnu.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/fft/fftpack_lite.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/fft/fftpack_lite.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/random/mtrand.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/numpy/random/mtrand.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/_path.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/_path.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/ft2font.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/ft2font.so
Reading symbols from /usr/lib/x86_64-linux-gnu/libfreetype.so.6...(no debugging symbols found)...done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/libfreetype.so.6
Reading symbols from /lib/x86_64-linux-gnu/libpng12.so.0...(no debugging symbols found)...done.
Loaded symbols for /lib/x86_64-linux-gnu/libpng12.so.0
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/_png.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/_png.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/_cntr.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/_cntr.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/_image.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/_image.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/_tri.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/_tri.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/_qhull.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/_qhull.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/backends/_backend_agg.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/matplotlib/backends/_backend_agg.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/PIL/_imagingft.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/PIL/_imagingft.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/Crypto/Util/_counter.so...(no debugging symbols found)...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/Crypto/Util/_counter.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/Crypto/Cipher/_AES.so...(no debugging symbols found)...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/Crypto/Cipher/_AES.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/Crypto/Util/strxor.so...(no debugging symbols found)...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/Crypto/Util/strxor.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/_geohash.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/_geohash.so
Reading symbols from /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/bx/intervals/intersection.so...done.
Loaded symbols for /home/PATHTOVENV/venv/local/lib/python2.7/site-packages/bx/intervals/intersection.so
Reading symbols from /usr/lib/x86_64-linux-gnu/gconv/CP1251.so...Reading symbols from /usr/lib/debug//usr/lib/x86_64-linux-gnu/gconv/CP1251.so...done.
done.
Loaded symbols for /usr/lib/x86_64-linux-gnu/gconv/CP1251.so
sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
85  ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S: No such file or directory.
(gdb) info thread
  Id   Target Id         Frame 
  25   Thread 0x7f08551db700 (LWP 30302) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  24   Thread 0x7f08541d9700 (LWP 30304) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  23   Thread 0x7f08539d8700 (LWP 30305) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  22   Thread 0x7f08531d7700 (LWP 30306) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  21   Thread 0x7f08529d6700 (LWP 30307) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  20   Thread 0x7f08521d5700 (LWP 30308) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  19   Thread 0x7f08519d4700 (LWP 30309) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  18   Thread 0x7f08511d3700 (LWP 30310) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  17   Thread 0x7f08509d2700 (LWP 30311) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  16   Thread 0x7f08501d1700 (LWP 30312) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  15   Thread 0x7f084f9d0700 (LWP 30313) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  14   Thread 0x7f084f1cf700 (LWP 30314) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  13   Thread 0x7f084e9ce700 (LWP 30315) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  12   Thread 0x7f084e1cd700 (LWP 30316) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  11   Thread 0x7f084d9cc700 (LWP 30317) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  10   Thread 0x7f084d1cb700 (LWP 30318) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  9    Thread 0x7f084c9ca700 (LWP 30319) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  8    Thread 0x7f084c1c9700 (LWP 30320) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  7    Thread 0x7f084b9c8700 (LWP 30321) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  6    Thread 0x7f084b1c7700 (LWP 30322) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  5    Thread 0x7f084a9c6700 (LWP 30323) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  4    Thread 0x7f084a1c5700 (LWP 30324) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  3    Thread 0x7f08499c4700 (LWP 30325) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
  2    Thread 0x7f08491c3700 (LWP 30326) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85
* 1    Thread 0x7f0903832740 (LWP 30183) "python" sem_wait () at ../nptl/sysdeps/unix/sysv/linux/x86_64/sem_wait.S:85

The gdb sees all the threads and their sem_wait state but I can not use python helpers for GDB:

(gdb) py-list
Undefined command: "py-list".  Try "help".

OS Ubuntu 14.04, python 2.7.
I have tried different hacks I had googled (like linking python-dbg.py into virtualenv bin directory) but with no profit, may be I have do it wrong way…


Get this bounty!!!

#StackBounty: #python #regression #convnet #keras #audio-recognition Optimizing CNN network

Bounty: 50

I am currently trying to recreate the result of this paper, in which they do feature extraction from a “spectogram” of log-melfilter energies..

enter image description here

Since the paper doesn’t state what kind of feature I am seeking, i am currently trying to extract features, and match them to MFCC features. The paper states a technique called LWS (Limited weight sharing) in which the spectogram frequency axis will be divided into section, and each section don’t share their weight with others.

So i’ve divided the my input image into 13 section to receive 1 output features from a (6,3,3) input image. 6 for the number of rows, 3 as each column represent [static delta delta_delta] data of the given log melfilter energi, and the last 3 is the color channels.

If i’d used 13 filterbanks, and made the plot, will the result of this be that each (1,3,3) matrix would result in one feature, but that seemed a bit too good to be true, so i decided to use 78 filterbanks and divide it into 13 section which should result in one feature can be extracted from a matrix of size (6,3,3)

I am training the network with this model structure:

def create_model(init_mode='normal',activation_mode='softsign',optimizer_mode="Adamax", activation_mode_conv = 'softsign'):
    model = Sequential()


    model.add(ZeroPadding2D((6,4),input_shape=(6,3,3)))
    model.add(Convolution2D(32,3,3 , activation=activation_mode_conv))
    print model.output_shape
    model.add(Convolution2D(32, 3,3, activation=activation_mode_conv))
    print model.output_shape
    model.add(MaxPooling2D(pool_size=(2,2),strides=(2,1)))
    print model.output_shape
    model.add(Convolution2D(64, 3,3 , activation=activation_mode_conv))
    print model.output_shape
    model.add(Convolution2D(64, 3,3 , activation=activation_mode_conv))
    print model.output_shape
    model.add(MaxPooling2D(pool_size=(2,2),strides=(2,1)))
    model.add(Flatten())
    print model.output_shape
    model.add(Dense(output_dim=32, input_dim=64, init=init_mode,activation=activation_mode))
    model.add(Dense(output_dim=13, input_dim=50, init=init_mode,activation=activation_mode))
    model.add(Dense(output_dim=1, input_dim=13, init=init_mode,activation=activation_mode))
    model.add(Dense(output_dim=1,  init=init_mode, activation=activation_mode))
    #print model.summary()
    model.compile(loss='mean_squared_error',optimizer=optimizer_mode)

    return model

This model keeps for some reason providing me very bad results..
I seem to keep getting an loss of 216, which is nearly 3 times the data range…

I did a grid seach to find out which parameter (activation function, init_mode, epochs and batch_size would be best, which are those chosen in the function above (eventhough there wasn’t much change in the outcome..)

What can i do to get better results?
Is the CNN network poorly designed?


Get this bounty!!!

#StackBounty: #python #python-2.7 #socket #tcp #network-file-transfer Protocol implementation, TCP, sending images through sockets (soc…

Bounty: 50

I would like to ask about your opinion about my code. The idea is simple: I designed my own protocol, where client asks server about the image, and the server sends the image, following the below steps (this is the actual protocol I wanted to implement):

CLIENT                                      SERVER
                      GETrn   
       ----------------------------------->
                      OKrn
       <----------------------------------- 
                   GET_SIZErn
       ----------------------------------->
                    SIZE 1024rn
       <-----------------------------------
                   GET_IMGrn
       ----------------------------------->
    IMG_DATA an when image sending is over, EOIMGrn 
       <-----------------------------------
                   DONErn
       ----------------------------------->
            server disconnects the client   

I improved the code as suggested by @TobErnack, but still, would like to know your opinion of my improvements. Here’s the link to the original question and code: Implementing my own network protocol using TCP, sending images through sockets (sockets, python)

Everything works (it seems so) but I would like to ask, what else I could possibly improve here.

  1. Does the implementation reflects in 100% what I wanted to achieve in
    my protocol flow?
  2. Are the steps implemented as I wanted them to be in my protocol?

client.py

#!/usr/bin/env python

import socket
import sys

HOST = '127.0.0.1'
PORT = 6666

sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_address = (HOST, PORT)
sock.connect(server_address)
fname = 'fromserver.png'


def recvall(sock, msgLen):
    msg = ""
    bytesRcvd = 0

    while bytesRcvd < msgLen:

        chunk = sock.recv(msgLen - bytesRcvd)

        if chunk == "": break

        bytesRcvd += len(chunk)
        msg += chunk

        if "rn" in msg: break
    return msg


try:

    sock.sendall("GETrn")
    data = recvall(sock, 4096)

    if data:
        txt = data.strip()
        print '--%s--' % txt

        if txt == 'OK':

            sock.sendall("GET_SIZErn")
            data = recvall(sock, 4096)

            if data:
                txt = data.strip()
                print '--%s--' % txt

                if txt.startswith('SIZE'):

                    tmp = txt.split()
                    size = int(tmp[1])

                    print '--%s--' % size

                    sock.sendall("GET_IMGrn")

                    myfile = open(fname, 'wb')

                    amount_received = 0
                    while amount_received < size:
                        data = recvall(sock, 4096)
                        if not data:
                            break
                        amount_received += len(data)
                        print amount_received

                        txt = data.strip('rn')

                        if 'EOIMG' in str(txt):
                            print 'Image received successfully'
                            myfile.write(data)
                            myfile.close()
                            sock.sendall("DONErn")
                        else:
                            myfile.write(data)
finally:
    sock.close()

server.py

# !/usr/bin/env python

import random
import socket, select
from time import gmtime, strftime

image = 'tux.png'

HOST = '127.0.0.1'
PORT = 6666

connected_clients_sockets = []

server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)

connected_clients_sockets.append(server_socket)


def recvall(sock, msgLen):
    msg = ""
    bytesRcvd = 0

    while bytesRcvd < msgLen:

        chunk = sock.recv(msgLen - bytesRcvd)

        if chunk == "": break

        bytesRcvd += len(chunk)
        msg += chunk

        if "rn" in msg: break
    return msg


while True:

    read_sockets, write_sockets, error_sockets = select.select(connected_clients_sockets, [], [])

    for sock in read_sockets:

        if sock == server_socket:

            sockfd, client_address = server_socket.accept()
            connected_clients_sockets.append(sockfd)

        else:
            try:
                data = recvall(sock, 4096)

                if data:

                    txt = data.strip()
                    print '--%s--' % txt

                    if txt == 'GET':
                        sock.sendall('OKrn')

                    elif txt == 'GET_SIZE':

                        with open('tux.png', 'rb') as f1:
                            file_size = len(f1.read())
                            f1.seek(0)

                        print '--%s--' % file_size

                        file_size = '%s' % file_size
                        sock.sendall('SIZE %srn' % file_size)

                    elif txt == 'GET_IMG':
                        with open(image, 'rb') as fp:
                            image_data = fp.read()

                        msg = '%sEOIMGrn' % image_data
                        sock.sendall(msg)

                    elif txt == 'DONE':
                        sock.close()
                        connected_clients_sockets.remove(sock)

            except:
                sock.close()
                connected_clients_sockets.remove(sock)
                continue
server_socket.close()


Get this bounty!!!