Why is my python socket connection not receiving all of the data, sometimes...?

Question

I have made a simple test program for connecting multiple clients to a server. The server just sends the client its time repeatedly, with some additional encryption. The problem is that sometimes the client does not receive the entire message, which causes an error when pickle tries to unpickle an incomplete message. However, it functions perfectly on localhost.

server.py

import datetime
import logging
import socket
import threading
import traceback

import rsa.randnum

from blob_api import *

logging.basicConfig(format='%(levelname)s - %(message)s', level=logging.DEBUG)

SERVER_IP = "192.168.101.1"
SERVER_PORT = 8108


class __Server(socket.socket):
    def __init__(self):
        super().__init__()
        self.public_key, self.private_key = rsa.newkeys(2048)
        try:
            self.bind((SERVER_IP, SERVER_PORT))
        except socket.error as _:
            logging.error(traceback.format_exc())
        self.listen()
        logging.info(f"Server started: ('{SERVER_IP}', {SERVER_PORT}).")
        while True:
            client_socket, client_address = self.accept()
            logging.info(f"Connected to client: {client_address}, waiting for public key...")
            threading.Thread(target=self.client_thread, args=(client_socket, client_address,)).start()

    def client_thread(self, client_socket: socket.socket, client_address):
        client_public_key: rsa.PublicKey = pickle.loads(client_socket.recv(2048))
        client_socket.sendall(pickle.dumps(self.public_key))
        logging.info(f"Established connection to client: {client_address}.")
        n = 0
        while True:
            n += 1
            try:
                header = client_socket.recv(HEADER_SIZE)
                print(f"Header recv {n}: " + header.decode())
                if not len(header):
                    break
                packet = client_socket.recv(int(header.decode()))
                data = decrypt(packet, self.private_key)
                client_socket.sendall(create_message(encrypt(f"Server {n}: {datetime.datetime.now()}", client_public_key)))
            except:
                logging.error(traceback.format_exc())
                print(packet)   
                break
        logging.warning(f"Lost connection to client: {client_address}")
        client_socket.close()


__Server()

client.py

import datetime
import socket
import sys
import threading
import logging
import traceback

from blob_api import *

logging.basicConfig(format='%(levelname)s - %(message)s', level=logging.DEBUG)

SERVER_IP = "SERVER_PUBLIC_IP"
SERVER_PORT = 8108


class Connection(socket.socket):
    def __init__(self):
        super().__init__()
        self.public_key, self.private_key = rsa.newkeys(2048)
        try:
            self.connect((SERVER_IP, SERVER_PORT))
        except ConnectionRefusedError as e:
            logging.warning("Could not connect to server " + str(e))
            sys.exit()
        self.sendall(pickle.dumps(self.public_key))
        self.server_public_key = pickle.loads(self.recv(2048))
        threading.Thread(target=self.__handle_requests).start()

    def __handle_requests(self):
        logging.info("Established connection to server.")
        n = 0
        while True:
            n += 1
            try:
                message = create_message(encrypt(f"Client: {datetime.datetime.now()}", self.server_public_key))
                print(message)
                self.sendall(message)
                header = self.recv(HEADER_SIZE)
                print(f"Header recv {n}: " + header.decode())
                if not len(header):
                    break
                packet = self.recv(int(header.decode()))
                data = decrypt(packet, self.private_key)
                print(data)
            except:
                logging.error(traceback.format_exc())
                break
        logging.warning("Lost connection to server.")
        self.close()

Connection()

blob_api.py

import pickle

import obj_encrypt
import rsa
import rsa.randnum

HEADER_SIZE = 8

def encrypt(data: any, public_key: rsa.PublicKey) -> bytes:
    secret = obj_encrypt.Secret(key=str(rsa.randnum.read_random_int(32)))
    return pickle.dumps({
        "data": secret.encrypt(data),
        "secret": rsa.encrypt(pickle.dumps(secret), public_key)
    })


def decrypt(packet: bytes, private_key: rsa.PrivateKey) -> any:
    packet = pickle.loads(packet)
    return pickle.loads(rsa.decrypt(packet["secret"], private_key)).decrypt(packet["data"])


def create_message(message: bytes) -> bytes:
    return f"{len(message):<8}".encode() + message

I have tried a connection from a client on a laptop on a mobile phone hotspot to a remote server. The client connects successfully and the message is received and printed on the client side for a short time, but then at a random point in time (~5-60 seconds), either the client or server do not receive the full message. The header of the message is read first, which determines the length of the rest of the message, then the exact length of the message can be read.

This is the end of the output on the client side.

Header recv 44: 349
Server 44: 2024-03-13 12:29:40.200717
Len: 346
b"346     \x80\x04\x95O\x01\x00\x00\x00\x00\x00\x00}\x94(\x8c\x04data\x94C1\x00\x9f\xc0\xe9]\xea\x9c\tg#\xa0\x83;\x9d\x1a\xda\xe3\xb5\xa9u\x7f\x15\xff\xeah\x0c\xf8f\xbe\xc14\x8c\xa2\xc1T^\xf9Z]\x16\xf2\x96,3\xd6\\\xb0\x04[\x94\x8c\x06secret\x94B\x00\x01\x00\x00a`\x05#\xf7\x08}\x17Ay\xe2\x96\xc8\xab:V@%\x94?j-\xd3F\xd3Q\xc1\xabl\xf3\x94\x9b\xc3U\xc2\xe7fD\n\x17\xad2\x0c\xea\x87m\xb3\xfb\xffq\xbb'\xdd\x07\x0f\r\x95\x14]3\xdfuh\xf3\x03i\x9f\x94R\x92>\xce\xa5\x8d\xb0';\x1a\x91)kJ\x9f?\xbe\xec\xc4\x85\\\xed\xb2\x88]k\x7f\xfc~\x0e9\xcd\xa2P:\x86Nq,J\xe3ol\xe3\x00\x94\xf8^+j\xb2\x8d3\xb1\x84\t\x80a\x16\x98\x84\x0fM\xe9\xd7\xa9OZ\xcf\xab&\xe1\xab\xa5\x1b\xc9\r,\xaf\r\x1f0,\xc5\xd9\x11L\xe6>\xcb\x12\xe6\x9e\xbc\xde\xb48\x1f\xa9\xd2g\xb2BZ\x98\xa4\x00\x85\xa1\xc9\xba\x87\r\xe0\x16\xc2\x9d\x00\xc3lz\xc9\x0f\xb8\x1fE\xf0[\xd3N\xd4\xa1\x91e_IS\x17C\x014V\xce@\xa3\x04.\xbc\xd0\x05\xb3\xf1\xc6\x97\x04_\xa6W\xe0\x17\x82\xa0\\\x92P3g\xd9\x1e.\x82{?'4\xd2\x19l\xce\xb0\xb5\xb0\xc3M\x8d\x14%V\x94u."
Header recv 45: 349
Server 45: 2024-03-13 12:29:40.292893
Len: 346
b'346     \x80\x04\x95O\x01\x00\x00\x00\x00\x00\x00}\x94(\x8c\x04data\x94C1\x89\xbe"\xdb\xd5=\x19\x01\xc8\xc3\xba\x81\x96\x07L)\xfa,\'(\xaa\x80"\xadt\xbck\x8fZY\x8c\xd2\x84\x0e\xfc\xdb$\x1dY\xf3\x98\x8e\x88pvC\xd9\xb2\x8b\x94\x8c\x06secret\x94B\x00\x01\x00\x00B\xcez\x93\xa8\xbbf\n\xc5\xcbi6\xfe}\xff\xb1\x8e\xc8\x98\xbb\x8bbp\\g\x0f\x15y\xf6\xf0_\x0e\x82\x8b\x1e\xb3\xb3\xdf\x17\xc6\x97\xd9\x02\x88\x00l\xaa={gF\x92\xec;\xdbB\x08\xbd\x18I\x84*\xb9\xb8P\xbf\r|\xb2\xbeP\x19\xb1+\xb2\xaa[\xd8\xc7\x1e\xb4K(\x05\xbd_\x90\x99\xdd.\xfb^\x17\xac\xa7\xd69\x13\x8c\x1e\x8e\xb4I\xbc\xb2)c\xc5\x97\xb5\xb3e2\x91\xb2\xd8\xe3\xc1\xben\x0c\xdcd\xd0\xf3\x84\xfa[\xfc6\xbd_\x12M\xafS\xcc\xe4H\x8bQQ\xac\x07\xff\xe1\xd1\x19IW\xf6\x94\xf3\xd13\x08F\xe2\x07 U<\xcb\xecD\xcd\x84\x00#\xe4\xc2\xa5(\x80\x12\x80\x1dm\xab\x8b\x19N\xa1;/\x13\x7f\x05\xde\xc4+\xdeKQ\x01\'\xcc\xe6;>s\x01\xf9\xdd\x80\xdc\xb0A\xee\xba\x85c/\x7f\x04z\xe0H\xb6l\x90\x9a\x91\xa5\xed\xe28\xa8T\xf4\x99l\xd6\xf2\x85\xcb\x05\xee\x19,o\xa7\xe1w\x80\x87.\x02@n#\xac\xadQ\xa2\xb6\x94u.'
Header recv 46:
WARNING - Lost connection to server.

This is the end of the output on the server side.

Header recv 44: 346     
Len: 349
Header recv 45: 346     
Len: 349
Header recv 46: 346     
ERROR - Traceback (most recent call last):
  File "/path_to/server.py", line 46, in client_thread
    data = decrypt(packet, self.private_key)
  File "/path_to/blob_api.py", line 18, in decrypt
    packet = pickle.loads(packet)
_pickle.UnpicklingError: pickle data was truncated
b'\x80\x04\x95O\x01\x00\x00\x00\x00\x00\x00}\x94(\x8c\x04data\x94C1\x89\xbe"\xdb\xd5=\x19\x01\xc8\xc3\xba\x81\x96\x07L)\xfa,\'(\xaa\x80"\xadt\xbck\x8fZY\x8c\xd2\x84\x0e\xfc\xdb$\x1dY\xf3\x98\x8e\x88pvC\xd9\xb2\x8b\x94\x8c\x06secret\x94B\x00\x01\x00\x00B\xcez\x93\xa8\xbbf\n\xc5\xcbi6\xfe}\xff\xb1\x8e\xc8\x98\xbb\x8bbp\\g\x0f\x15y\xf6\xf0_\x0e\x82\x8b\x1e\xb3\xb3\xdf\x17\xc6\x97\xd9\x02\x88\x00l\xaa={g'

The client sent a server a message with the size of 346 bytes, the server only received 137. However, you can see that it worked 45 times before the error. I think it has something to do with a connection with a mobile hotspot because it works fine on localhost but I'm unsure.

Any help would be appreciated.

Answer 1

Thank you to Wonka for linking me to another question. I believe the problem was that the packets were fragmented. To fix it, I simply changed the packet = recv() function to the recvall() function in the linked question.