Python Distributed Computing (works)

Question

Estoy usando un hilo antiguo para publicar un nuevo código que intenta resolver el mismo problema. ¿Qué constituye un pepinillo seguro? this?

sock.py

from socket import socket 
from socket import AF_INET 
from socket import SOCK_STREAM 
from socket import gethostbyname 
from socket import gethostname 

class SocketServer: 
    def __init__(self, port): 
    self.sock = socket(AF_INET, SOCK_STREAM) 
    self.port = port 
    def listen(self, data): 
    self.sock.bind(("127.0.0.1", self.port)) 
    self.sock.listen(len(data)) 
    while data: 
     s = self.sock.accept()[0] 
     siz, dat = data.pop() 
     s.send(siz) 
     s.send(dat) 
     s.close() 

class Socket: 
    def __init__(self, host, port): 
    self.sock = socket(AF_INET, SOCK_STREAM) 
    self.sock.connect((host, port)) 
    def recv(self, size): 
    return self.sock.recv(size) 

pack.py

#http://stackoverflow.com/questions/6234586/we-need-to-pickle-any-sort-of-callable 
from marshal import dumps as marshal_dumps 
from pickle import dumps as pickle_dumps 
from struct import pack as struct_pack 

class packer: 
    def __init__(self): 
    self.f = [] 
    def pack(self, what): 
    if type(what) is type(lambda:None): 
     self.f = [] 
     self.f.append(marshal_dumps(what.func_code)) 
     self.f.append(pickle_dumps(what.func_name)) 
     self.f.append(pickle_dumps(what.func_defaults)) 
     self.f.append(pickle_dumps(what.func_closure)) 
     self.f = pickle_dumps(self.f) 
     return (struct_pack('Q', len(self.f)), self.f) 

unpack.py

from types import FunctionType 
from pickle import loads as pickle_loads 
from marshal import loads as marshal_loads 
from struct import unpack as struct_unpack 
from struct import calcsize 

#http://stackoverflow.com/questions/6234586/we-need-to-pickle-any-sort-of-callable 

class unpacker: 
    def __init__(self): 
    self.f = [] 
    self.fcompiled = lambda:None 
    self.sizeofsize = calcsize('Q') 
    def unpack(self, sock): 
    size = struct_unpack('Q', sock.recv(self.sizeofsize))[0] 
    self.f = pickle_loads(sock.recv(size)) 
    a = marshal_loads(self.f[0]) 
    b = globals() ## 
    c = pickle_loads(self.f[1]) 
    d = pickle_loads(self.f[2]) 
    e = pickle_loads(self.f[3]) 
    self.fcompiled = FunctionType(a, b, c, d, e) 
    return self.fcompiled 

test.py

from unpack import unpacker 
from pack import packer 
from sock import SocketServer 
from sock import Socket 
from threading import Thread 
from time import sleep 

count = 2 
port = 4446 

def f(): 
    print 42 

def server(): 
    ss = SocketServer(port) 
    pack = packer() 
    functions = [pack.pack(f) for nothing in range(count)] 
    ss.listen(functions) 

if __name__ == "__main__": 
    Thread(target=server).start() 
    sleep(1) 
    unpack = unpacker() 
    for nothing in range(count): 
    print unpack.unpack(Socket("127.0.0.1", port)) 

de salida:

<function f at 0x12917d0> 
<function f at 0x12915f0> 

Answer 1

ValueError: insecure string pickle se genera cuando su salmuera está dañada. ¿Está seguro de que está recibiendo todo el objeto en escabeche en un sock.recv() (unpack.py)?

Editar: para evitar esto para cualquier tamaño que podría hacer (la clase Socket tendría que mantener a recv ser llamado con un argumento tamaño del búfer (es decir

class Socket: 
    def recv(self, bufsize): 
     return self.sock.recv(bufsize) 

)):

import struct 

struct.pack('Q', len(pickled_list)) 
# Send it, and then send the pickled list. 

En el programa del receptor:

import struct 

length = struct.unpack('Q', sock.recv(struct.calcsize('Q')))[0] 
pickled_list = sock.recv(length) 

'Q' es un unsigned long long. Para otras cosas de estructura, vea the struct module documentation

Answer 2

no creo que los objetos de proceso están diseñados para ser enviados a través de la red. Mire la línea 256 en multiprocesamiento/process.py.

# We subclass bytes to avoid accidental transmission of auth keys over network. 

Parece que hay una buena razón para mí. Si quieres hacer informática distribuida, tal vez deberías consultar un library designed for that.