Boost Serialization to char buffer and Deserialization from data

Question

I am making a communication module between processes using boost::interprocess::message_queue Because message queue takes array buffer, I would to serialize a packet into an array buffer.

inline void message_queue_t<VoidPointer>::send
   (const void *buffer, size_type buffer_size, unsigned int priority)

for example,

  header header(5, 2);
  char buffer[64] = {};
  uint32_t size = header.save(buffer);

  queue.send(buffer, sizeof(size), 0);   // queue is boost::interprocess::message_queue

Here's a progress:

#ifndef IPC_PACKET_HEADER_HPP
#define IPC_PACKET_HEADER_HPP

#include <boost/archive/binary_iarchive.hpp>
#include <boost/archive/binary_oarchive.hpp>
#include <boost/iostreams/stream.hpp>
#include <iostream>
#include <string>

class header {
public: 
  header(uint32_t id, uint32_t size) :
    id_(id),
    size_(size) {
  }

  ~header() = default;

  uint32_t save(char* buffer) {
    boost::iostreams::basic_array_sink<char> sink((char *)buffer, 64);  
    boost::iostreams::stream<boost::iostreams::basic_array_sink<char>> os(sink);
    boost::archive::binary_oarchive oa(os);
    oa & *(this);

    return 0; // size that are copied to the buffer?
  }

  void load(const void* data) {
    // boost::archive::binary_iarchive ia(iss);
    // ia & *(this);
  }

private:
  friend class boost::serialization::access;
  
  uint32_t id_;
  uint32_t size_;

template<class Archive>
  void serialize(Archive& ar, const unsigned int version) {
    ar & id_;
    ar & size_;
  }
};

#endif

Two questions.

1. When serialized using save() function, I need to know the size that are serialized so that I can pass it to the send function of boost::interprocess::message_queue. How do we get this size?
2. I am running out of idea how to make load function. It suppose to get a byte data and should load them itself. Can you help with this?

I would appreciate any inputs.

Answer 1

Dynamic Buffer

I would suggest using a dynamically sized container:

void save(std::vector<char>& buffer) {
    bio::stream os(bio::back_inserter(buffer));
    boost::archive::binary_oarchive(os) << *this;
}

Now you will just have buffer.size() reflecting the bytes serialized:

header example {42,9};

std::vector<char> dynamic;
example.save(dynamic);
fmt::print("dynamic({}), {::02x}\n", dynamic.size(), dynamic);

Static Buffer

Of course, if you insist you can make use of a statically sized container.

uint32_t save(std::span<char> buffer) {
    bio::stream os(bio::array_sink(buffer.data(), buffer.size()));
    boost::archive::binary_oarchive(os) << *this;
    return os.tellp();
}

Here, the returned value is the resulting seek position into the put buffer of the stream.

CAVEAT If your buffer is too small it will silently create an incomplete archive

Loading

Regardless of the chosen approach, loading looks the same:

void load(std::span<char const> data) {
    bio::stream is(bio::array_source(data.data(), data.size()));
    boost::archive::binary_iarchive(is) >> *this;
}

Live Demo

Demonstrating all the approaches and also optimizing archive size using some flags:

Live On Coliru

#include <boost/archive/binary_iarchive.hpp>
#include <boost/archive/binary_oarchive.hpp>

#include <boost/iostreams/device/back_inserter.hpp>
#include <boost/iostreams/stream.hpp>
#include <fmt/ranges.h>
#include <iostream>
#include <span>

namespace bio = boost::iostreams;

template <int ArchiveFlags = 0> class header {
  public:
    header(uint32_t id = -1, uint32_t size = -1) : id_(id), size_(size) {}

    ~header() = default;

    uint32_t save(std::span<char> buffer) {
        bio::stream os(bio::array_sink(buffer.data(), buffer.size()));
        boost::archive::binary_oarchive(os, ArchiveFlags) << *this;
        return os.tellp();
    }

    void save(std::vector<char>& buffer) {
        bio::stream os(bio::back_inserter(buffer));
        boost::archive::binary_oarchive(os, ArchiveFlags) << *this;
    }

    void load(std::span<char const> data) {
        bio::stream is(bio::array_source(data.data(), data.size()));
        boost::archive::binary_iarchive(is, ArchiveFlags) >> *this;
    }

    bool operator==(header const&) const = default;

  private:
    friend class boost::serialization::access;

    uint32_t id_;
    uint32_t size_;

    template <class Archive> void serialize(Archive& ar, unsigned) { ar& id_& size_; }
};

template <int Flags = 0> void demo() {
    using T = header<Flags>;

    T example{42, 9};

    std::vector<char> dynamic;
    example.save(dynamic);
    fmt::print("dynamic({}), {::02x}\n", dynamic.size(), dynamic);

    {
        T roundtrip;
        roundtrip.load(dynamic);
        fmt::print("roundtrip: {}\n", roundtrip == example);
    }

    std::array<char, 64> fixed;
    auto n = example.save(fixed);
    fmt::print("fixed({}), {::02x}\n", n, std::span(fixed).subspan(0, n));

    {
        T roundtrip;
        roundtrip.load(fixed); // remaining bytes ignored
        fmt::print("sloppy roundtrip: {}\n", roundtrip == example);
    }

    {
        T roundtrip;
        roundtrip.load(std::span(fixed).subspan(0, n)); // trimmed remaining bytes
        fmt::print("trimmed roundtrip: {}\n", roundtrip == example);
    }
}

int main() {
    fmt::print("\n------ Normal archive flags\n");
    demo(); // normal

    fmt::print("\n------ Size-optimized archive flags\n");
    demo<boost::archive::no_header     //
         | boost::archive::no_codecvt  //
         | boost::archive::no_tracking //
         >();                          // optimized
}

Prints the informative and expected:

------ Normal archive flags
dynamic(53), [16, 00, 00, 00, 00, 00, 00, 00, 73, 65, 72, 69, 61, 6c, 69, 7a, 61, 74, 69, 6f, 6e, 3a, 3a, 61, 72, 63, 68, 69, 76, 65, 14, 00, 04, 08, 04, 08, 01, 00, 00, 00, 00, 00, 00, 00, 00, 2a, 00, 00, 00, 09, 00, 00, 00]
roundtrip: true
fixed(53), [16, 00, 00, 00, 00, 00, 00, 00, 73, 65, 72, 69, 61, 6c, 69, 7a, 61, 74, 69, 6f, 6e, 3a, 3a, 61, 72, 63, 68, 69, 76, 65, 14, 00, 04, 08, 04, 08, 01, 00, 00, 00, 00, 00, 00, 00, 00, 2a, 00, 00, 00, 09, 00, 00, 00]
sloppy roundtrip: true
trimmed roundtrip: true

------ Size-optimized archive flags
dynamic(13), [00, 00, 00, 00, 00, 2a, 00, 00, 00, 09, 00, 00, 00]
roundtrip: true
fixed(13), [00, 00, 00, 00, 00, 2a, 00, 00, 00, 09, 00, 00, 00]
sloppy roundtrip: true
trimmed roundtrip: true

BONUS

Since the Boost Serialization buys you no functionality here (no object tracking, object graph recursion, not even portability) consider just using bitwise serialization here:

Live On Coliru

#include <cassert>
#include <fmt/ranges.h>
#include <iostream>
#include <span>

namespace MyMessages {
    struct header {
        uint32_t id_;
        uint32_t size_;

        auto operator<=>(header const&) const = default;
    };

    struct some_other_message {
        header   header_;
        uint32_t len_;
        uint8_t  text_[32];

        auto operator<=>(some_other_message const&) const = default;
    };

    using std::span; // or e.g. boost::span

    template <typename T> static inline auto save(T const& msg, span<char> out) {
        static_assert(std::is_trivial_v<T> && std::is_standard_layout_v<T>);

        assert(out.size() >= sizeof(T));
        memcpy(out.data(), &msg, sizeof(T));
        return out.subspan(sizeof(T));
    }

    template <typename T> static inline auto load(T& msg, span<char const> in) {
        static_assert(std::is_trivial_v<T> && std::is_standard_layout_v<T>);

        assert(in.size() >= sizeof(T));
        memcpy(&msg, in.data(), sizeof(T));
        return in.subspan(sizeof(T));
    }
} // namespace MyMessages


int main() {
    using MyMessages::span;
    MyMessages::some_other_message example{{42, 9}, 12, "Hello world!"};

    std::array<char, 64> buf;

    {
        auto remain = save(example, buf);
        auto n      = remain.data() - buf.data();
        fmt::print("fixed({}), {::02x}\n", n, span(buf).subspan(0, n));
    }

    {
        MyMessages::some_other_message roundtrip;

        auto remain   = load(roundtrip, buf);
        auto consumed = remain.data() - buf.data();
        fmt::print("roundtrip({}): {}\n", consumed, roundtrip == example);
    }

    {
        MyMessages::header just_header;

        auto remain   = load(just_header, buf);
        auto consumed = remain.data() - buf.data();
        fmt::print("partial deserialization({}): {}\n", consumed, just_header == example.header_);
    }
}

Note how it doesn't use Boost Serialization, Boost Iostreams, or any boost at all, and the header serializes into 8 bytes intead of 53 using a serialization archive:

fixed(44), [2a, 00, 00, 00, 09, 00, 00, 00, 0c, 00, 00, 00, 48, 65, 6c, 6c, 6f, 20, 77, 6f, 72, 6c, 64, 21, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00, 00]
roundtrip(44): true
partial deserialization(8): true