add support for requests with both MultipartFormDataItems and Content Providers (#1454)
* add support for requests with both MultipartFormDataItems and ContentProviders
* rework implementation
* use const auto & and fix offset calculation
* fix zero items
* snake case variables
* clang-format
* commonize get_multipart_content_provider, add Put() with MultipartFormDataProviderItems
* fix linker multiple definition error
* add test MultipartFormDataTest.DataProviderItems
diff --git a/httplib.h b/httplib.h
index 751aa92..1d0e628 100644
--- a/httplib.h
+++ b/httplib.h
@@ -369,6 +369,14 @@
using ContentProviderResourceReleaser = std::function<void(bool success)>;
+struct MultipartFormDataProvider {
+ std::string name;
+ ContentProviderWithoutLength provider;
+ std::string filename;
+ std::string content_type;
+};
+using MultipartFormDataProviderItems = std::vector<MultipartFormDataProvider>;
+
using ContentReceiverWithProgress =
std::function<bool(const char *data, size_t data_length, uint64_t offset,
uint64_t total_length)>;
@@ -934,6 +942,9 @@
const MultipartFormDataItems &items);
Result Post(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items, const std::string &boundary);
+ Result Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items);
Result Put(const std::string &path);
Result Put(const std::string &path, const char *body, size_t content_length,
@@ -963,6 +974,9 @@
const MultipartFormDataItems &items);
Result Put(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items, const std::string &boundary);
+ Result Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items);
Result Patch(const std::string &path);
Result Patch(const std::string &path, const char *body, size_t content_length,
@@ -1201,6 +1215,9 @@
ContentProvider content_provider,
ContentProviderWithoutLength content_provider_without_length,
const std::string &content_type);
+ ContentProviderWithoutLength get_multipart_content_provider(
+ const std::string &boundary, const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items);
std::string adjust_host_string(const std::string &host) const;
@@ -1296,6 +1313,10 @@
const MultipartFormDataItems &items);
Result Post(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items, const std::string &boundary);
+ Result Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items);
+
Result Put(const std::string &path);
Result Put(const std::string &path, const char *body, size_t content_length,
const std::string &content_type);
@@ -1324,6 +1345,10 @@
const MultipartFormDataItems &items);
Result Put(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items, const std::string &boundary);
+ Result Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items);
+
Result Patch(const std::string &path);
Result Patch(const std::string &path, const char *body, size_t content_length,
const std::string &content_type);
@@ -2854,8 +2879,7 @@
}
inline bool get_ip_and_port(const struct sockaddr_storage &addr,
- socklen_t addr_len, std::string &ip,
- int &port) {
+ socklen_t addr_len, std::string &ip, int &port) {
if (addr.ss_family == AF_INET) {
port = ntohs(reinterpret_cast<const struct sockaddr_in *>(&addr)->sin_port);
} else if (addr.ss_family == AF_INET6) {
@@ -4129,29 +4153,48 @@
return valid;
}
+template <typename T>
+inline std::string
+serialize_multipart_formdata_item_begin(const T &item,
+ const std::string &boundary) {
+ std::string body = "--" + boundary + "\r\n";
+ body += "Content-Disposition: form-data; name=\"" + item.name + "\"";
+ if (!item.filename.empty()) {
+ body += "; filename=\"" + item.filename + "\"";
+ }
+ body += "\r\n";
+ if (!item.content_type.empty()) {
+ body += "Content-Type: " + item.content_type + "\r\n";
+ }
+ body += "\r\n";
+
+ return body;
+}
+
+inline std::string serialize_multipart_formdata_item_end() { return "\r\n"; }
+
+inline std::string
+serialize_multipart_formdata_finish(const std::string &boundary) {
+ return "--" + boundary + "--\r\n";
+}
+
+inline std::string
+serialize_multipart_formdata_get_content_type(const std::string &boundary) {
+ return "multipart/form-data; boundary=" + boundary;
+}
+
inline std::string
serialize_multipart_formdata(const MultipartFormDataItems &items,
- const std::string &boundary,
- std::string &content_type) {
+ const std::string &boundary, bool finish = true) {
std::string body;
for (const auto &item : items) {
- body += "--" + boundary + "\r\n";
- body += "Content-Disposition: form-data; name=\"" + item.name + "\"";
- if (!item.filename.empty()) {
- body += "; filename=\"" + item.filename + "\"";
- }
- body += "\r\n";
- if (!item.content_type.empty()) {
- body += "Content-Type: " + item.content_type + "\r\n";
- }
- body += "\r\n";
- body += item.content + "\r\n";
+ body += serialize_multipart_formdata_item_begin(item, boundary);
+ body += item.content + serialize_multipart_formdata_item_end();
}
- body += "--" + boundary + "--\r\n";
+ if (finish) body += serialize_multipart_formdata_finish(boundary);
- content_type = "multipart/form-data; boundary=" + boundary;
return body;
}
@@ -4536,8 +4579,8 @@
*reinterpret_cast<struct sockaddr_storage *>(rp->ai_addr);
std::string ip;
int dummy = -1;
- if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage),
- ip, dummy)) {
+ if (detail::get_ip_and_port(addr, sizeof(struct sockaddr_storage), ip,
+ dummy)) {
addrs.push_back(ip);
}
}
@@ -6647,6 +6690,49 @@
return true;
}
+inline ContentProviderWithoutLength ClientImpl::get_multipart_content_provider(
+ const std::string &boundary, const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items) {
+ size_t cur_item = 0, cur_start = 0;
+ // cur_item and cur_start are copied to within the std::function and maintain
+ // state between successive calls
+ return [&, cur_item, cur_start](size_t offset,
+ DataSink &sink) mutable -> bool {
+ if (!offset && items.size()) {
+ sink.os << detail::serialize_multipart_formdata(items, boundary, false);
+ return true;
+ } else if (cur_item < provider_items.size()) {
+ if (!cur_start) {
+ const auto &begin = detail::serialize_multipart_formdata_item_begin(
+ provider_items[cur_item], boundary);
+ offset += begin.size();
+ cur_start = offset;
+ sink.os << begin;
+ }
+
+ DataSink cur_sink;
+ bool has_data = true;
+ cur_sink.write = sink.write;
+ cur_sink.done = [&]() { has_data = false; };
+ cur_sink.is_writable = sink.is_writable;
+
+ if (!provider_items[cur_item].provider(offset - cur_start, cur_sink))
+ return false;
+
+ if (!has_data) {
+ sink.os << detail::serialize_multipart_formdata_item_end();
+ cur_item++;
+ cur_start = 0;
+ }
+ return true;
+ } else {
+ sink.os << detail::serialize_multipart_formdata_finish(boundary);
+ sink.done();
+ return true;
+ }
+ };
+}
+
inline bool
ClientImpl::process_socket(const Socket &socket,
std::function<bool(Stream &strm)> callback) {
@@ -6869,9 +6955,10 @@
inline Result ClientImpl::Post(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items) {
- std::string content_type;
- const auto &body = detail::serialize_multipart_formdata(
- items, detail::make_multipart_data_boundary(), content_type);
+ const auto &boundary = detail::make_multipart_data_boundary();
+ const auto &content_type =
+ detail::serialize_multipart_formdata_get_content_type(boundary);
+ const auto &body = detail::serialize_multipart_formdata(items, boundary);
return Post(path, headers, body, content_type.c_str());
}
@@ -6882,12 +6969,25 @@
return Result{nullptr, Error::UnsupportedMultipartBoundaryChars};
}
- std::string content_type;
- const auto &body =
- detail::serialize_multipart_formdata(items, boundary, content_type);
+ const auto &content_type =
+ detail::serialize_multipart_formdata_get_content_type(boundary);
+ const auto &body = detail::serialize_multipart_formdata(items, boundary);
return Post(path, headers, body, content_type.c_str());
}
+inline Result
+ClientImpl::Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items) {
+ const auto &boundary = detail::make_multipart_data_boundary();
+ const auto &content_type =
+ detail::serialize_multipart_formdata_get_content_type(boundary);
+ return send_with_content_provider(
+ "POST", path, headers, nullptr, 0, nullptr,
+ get_multipart_content_provider(boundary, items, provider_items),
+ content_type);
+}
+
inline Result ClientImpl::Put(const std::string &path) {
return Put(path, std::string(), std::string());
}
@@ -6964,9 +7064,10 @@
inline Result ClientImpl::Put(const std::string &path, const Headers &headers,
const MultipartFormDataItems &items) {
- std::string content_type;
- const auto &body = detail::serialize_multipart_formdata(
- items, detail::make_multipart_data_boundary(), content_type);
+ const auto &boundary = detail::make_multipart_data_boundary();
+ const auto &content_type =
+ detail::serialize_multipart_formdata_get_content_type(boundary);
+ const auto &body = detail::serialize_multipart_formdata(items, boundary);
return Put(path, headers, body, content_type);
}
@@ -6977,12 +7078,24 @@
return Result{nullptr, Error::UnsupportedMultipartBoundaryChars};
}
- std::string content_type;
- const auto &body =
- detail::serialize_multipart_formdata(items, boundary, content_type);
+ const auto &content_type =
+ detail::serialize_multipart_formdata_get_content_type(boundary);
+ const auto &body = detail::serialize_multipart_formdata(items, boundary);
return Put(path, headers, body, content_type);
}
+inline Result
+ClientImpl::Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items) {
+ const auto &boundary = detail::make_multipart_data_boundary();
+ const auto &content_type =
+ detail::serialize_multipart_formdata_get_content_type(boundary);
+ return send_with_content_provider(
+ "PUT", path, headers, nullptr, 0, nullptr,
+ get_multipart_content_provider(boundary, items, provider_items),
+ content_type);
+}
inline Result ClientImpl::Patch(const std::string &path) {
return Patch(path, std::string(), std::string());
}
@@ -7443,7 +7556,7 @@
}
inline void SSLSocketStream::get_local_ip_and_port(std::string &ip,
- int &port) const {
+ int &port) const {
detail::get_local_ip_and_port(sock_, ip, port);
}
@@ -8147,6 +8260,12 @@
const std::string &boundary) {
return cli_->Post(path, headers, items, boundary);
}
+inline Result
+Client::Post(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items) {
+ return cli_->Post(path, headers, items, provider_items);
+}
inline Result Client::Put(const std::string &path) { return cli_->Put(path); }
inline Result Client::Put(const std::string &path, const char *body,
size_t content_length,
@@ -8210,6 +8329,12 @@
const std::string &boundary) {
return cli_->Put(path, headers, items, boundary);
}
+inline Result
+Client::Put(const std::string &path, const Headers &headers,
+ const MultipartFormDataItems &items,
+ const MultipartFormDataProviderItems &provider_items) {
+ return cli_->Put(path, headers, items, provider_items);
+}
inline Result Client::Patch(const std::string &path) {
return cli_->Patch(path);
}
diff --git a/test/test.cc b/test/test.cc
index 88937ce..e592ed1 100644
--- a/test/test.cc
+++ b/test/test.cc
@@ -5146,6 +5146,204 @@
t.join();
}
+TEST(MultipartFormDataTest, DataProviderItems) {
+
+ std::random_device seed_gen;
+ std::mt19937 random(seed_gen());
+
+ std::string rand1;
+ rand1.resize(1000);
+ std::generate(rand1.begin(), rand1.end(), [&]() { return random(); });
+
+ std::string rand2;
+ rand2.resize(3000);
+ std::generate(rand2.begin(), rand2.end(), [&]() { return random(); });
+
+ SSLServer svr(SERVER_CERT_FILE, SERVER_PRIVATE_KEY_FILE);
+
+ svr.Post("/post-none", [&](const Request &req, Response & /*res*/,
+ const ContentReader &content_reader) {
+ ASSERT_FALSE(req.is_multipart_form_data());
+
+ std::string body;
+ content_reader([&](const char *data, size_t data_length) {
+ body.append(data, data_length);
+ return true;
+ });
+
+ EXPECT_EQ(body, "");
+ });
+
+ svr.Post("/post-items", [&](const Request &req, Response & /*res*/,
+ const ContentReader &content_reader) {
+ ASSERT_TRUE(req.is_multipart_form_data());
+ MultipartFormDataItems files;
+ content_reader(
+ [&](const MultipartFormData &file) {
+ files.push_back(file);
+ return true;
+ },
+ [&](const char *data, size_t data_length) {
+ files.back().content.append(data, data_length);
+ return true;
+ });
+
+ ASSERT_TRUE(files.size() == 2);
+
+ EXPECT_EQ(std::string(files[0].name), "name1");
+ EXPECT_EQ(files[0].content, "Testing123");
+ EXPECT_EQ(files[0].filename, "filename1");
+ EXPECT_EQ(files[0].content_type, "application/octet-stream");
+
+ EXPECT_EQ(files[1].name, "name2");
+ EXPECT_EQ(files[1].content, "Testing456");
+ EXPECT_EQ(files[1].filename, "");
+ EXPECT_EQ(files[1].content_type, "");
+ });
+
+ svr.Post("/post-providers", [&](const Request &req, Response & /*res*/,
+ const ContentReader &content_reader) {
+ ASSERT_TRUE(req.is_multipart_form_data());
+ MultipartFormDataItems files;
+ content_reader(
+ [&](const MultipartFormData &file) {
+ files.push_back(file);
+ return true;
+ },
+ [&](const char *data, size_t data_length) {
+ files.back().content.append(data, data_length);
+ return true;
+ });
+
+ ASSERT_TRUE(files.size() == 2);
+
+ EXPECT_EQ(files[0].name, "name3");
+ EXPECT_EQ(files[0].content, rand1);
+ EXPECT_EQ(files[0].filename, "filename3");
+ EXPECT_EQ(files[0].content_type, "");
+
+ EXPECT_EQ(files[1].name, "name4");
+ EXPECT_EQ(files[1].content, rand2);
+ EXPECT_EQ(files[1].filename, "filename4");
+ EXPECT_EQ(files[1].content_type, "");
+ });
+
+ svr.Post("/post-both", [&](const Request &req, Response & /*res*/,
+ const ContentReader &content_reader) {
+ ASSERT_TRUE(req.is_multipart_form_data());
+ MultipartFormDataItems files;
+ content_reader(
+ [&](const MultipartFormData &file) {
+ files.push_back(file);
+ return true;
+ },
+ [&](const char *data, size_t data_length) {
+ files.back().content.append(data, data_length);
+ return true;
+ });
+
+ ASSERT_TRUE(files.size() == 4);
+
+ EXPECT_EQ(std::string(files[0].name), "name1");
+ EXPECT_EQ(files[0].content, "Testing123");
+ EXPECT_EQ(files[0].filename, "filename1");
+ EXPECT_EQ(files[0].content_type, "application/octet-stream");
+
+ EXPECT_EQ(files[1].name, "name2");
+ EXPECT_EQ(files[1].content, "Testing456");
+ EXPECT_EQ(files[1].filename, "");
+ EXPECT_EQ(files[1].content_type, "");
+
+ EXPECT_EQ(files[2].name, "name3");
+ EXPECT_EQ(files[2].content, rand1);
+ EXPECT_EQ(files[2].filename, "filename3");
+ EXPECT_EQ(files[2].content_type, "");
+
+ EXPECT_EQ(files[3].name, "name4");
+ EXPECT_EQ(files[3].content, rand2);
+ EXPECT_EQ(files[3].filename, "filename4");
+ EXPECT_EQ(files[3].content_type, "");
+ });
+
+ auto t = std::thread([&]() { svr.listen("localhost", 8080); });
+ while (!svr.is_running()) {
+ std::this_thread::sleep_for(std::chrono::milliseconds(1));
+ }
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+
+ {
+ Client cli("https://localhost:8080");
+ cli.enable_server_certificate_verification(false);
+
+ MultipartFormDataItems items{
+ {"name1", "Testing123", "filename1", "application/octet-stream"},
+ {"name2", "Testing456", "", ""}, // not a file
+ };
+
+ {
+ auto res = cli.Post("/post-none", {}, {}, {});
+ ASSERT_TRUE(res);
+ ASSERT_EQ(200, res->status);
+ }
+
+ MultipartFormDataProviderItems providers;
+
+ {
+ auto res =
+ cli.Post("/post-items", {}, items, providers); // empty providers
+ ASSERT_TRUE(res);
+ ASSERT_EQ(200, res->status);
+ }
+
+ providers.push_back({"name3",
+ [&](size_t offset, httplib::DataSink &sink) -> bool {
+ // test the offset is given correctly at each step
+ if (!offset)
+ sink.os.write(rand1.data(), 30);
+ else if (offset == 30)
+ sink.os.write(rand1.data() + 30, 300);
+ else if (offset == 330)
+ sink.os.write(rand1.data() + 330, 670);
+ else if (offset == rand1.size())
+ sink.done();
+ return true;
+ },
+ "filename3",
+ {}});
+
+ providers.push_back({"name4",
+ [&](size_t offset, httplib::DataSink &sink) -> bool {
+ // test the offset is given correctly at each step
+ if (!offset)
+ sink.os.write(rand2.data(), 2000);
+ else if (offset == 2000)
+ sink.os.write(rand2.data() + 2000, 1);
+ else if (offset == 2001)
+ sink.os.write(rand2.data() + 2001, 999);
+ else if (offset == rand2.size())
+ sink.done();
+ return true;
+ },
+ "filename4",
+ {}});
+
+ {
+ auto res = cli.Post("/post-providers", {}, {}, providers);
+ ASSERT_TRUE(res);
+ ASSERT_EQ(200, res->status);
+ }
+
+ {
+ auto res = cli.Post("/post-both", {}, items, providers);
+ ASSERT_TRUE(res);
+ ASSERT_EQ(200, res->status);
+ }
+ }
+
+ svr.stop();
+ t.join();
+}
+
TEST(MultipartFormDataTest, WithPreamble) {
Server svr;
svr.Post("/post", [&](const Request & /*req*/, Response &res) {
