Skip to content

Commit 9ab1643

Browse files
umenneldeanberris
authored andcommitted
Introduced new chunked transfer encoding parser to remove chunk markers in streaming clients.
1 parent 73d4024 commit 9ab1643

File tree

7 files changed

+126
-22
lines changed

7 files changed

+126
-22
lines changed

boost/network/protocol/http/client/async_impl.hpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,15 @@ struct async_client
4242

4343
async_client(bool cache_resolved, bool follow_redirect,
4444
bool always_verify_peer, int timeout,
45+
bool remove_chunk_markers,
4546
std::shared_ptr<boost::asio::io_service> service,
4647
optional<string_type> certificate_filename,
4748
optional<string_type> verify_path,
4849
optional<string_type> certificate_file,
4950
optional<string_type> private_key_file,
5051
optional<string_type> ciphers,
5152
optional<string_type> sni_hostname, long ssl_options)
52-
: connection_base(cache_resolved, follow_redirect, timeout),
53+
: connection_base(cache_resolved, follow_redirect, timeout, remove_chunk_markers),
5354
service_ptr(service.get() ? service
5455
: std::make_shared<boost::asio::io_service>()),
5556
service_(*service_ptr),

boost/network/protocol/http/client/connection/async_base.hpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ struct async_connection_base {
4343
// tag.
4444
static connection_ptr new_connection(
4545
resolve_function resolve, resolver_type &resolver, bool follow_redirect,
46-
bool always_verify_peer, bool https, int timeout,
46+
bool always_verify_peer, bool https, int timeout, bool remove_chunk_markers,
4747
optional<string_type> certificate_filename = optional<string_type>(),
4848
optional<string_type> const &verify_path = optional<string_type>(),
4949
optional<string_type> certificate_file = optional<string_type>(),
@@ -59,7 +59,7 @@ struct async_connection_base {
5959
certificate_filename, verify_path, certificate_file, private_key_file,
6060
ciphers, sni_hostname, ssl_options);
6161
auto temp = std::make_shared<async_connection>(
62-
resolver, resolve, follow_redirect, timeout, std::move(delegate));
62+
resolver, resolve, follow_redirect, timeout, remove_chunk_markers, std::move(delegate));
6363
BOOST_ASSERT(temp != nullptr);
6464
return temp;
6565
}

boost/network/protocol/http/client/connection/async_normal.hpp

Lines changed: 92 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
#include <iterator>
1313
#include <cstdint>
14+
#include <iostream>
1415
#include <boost/algorithm/string/trim.hpp>
1516
#include <boost/asio/steady_timer.hpp>
1617
#include <boost/asio/placeholders.hpp>
@@ -37,6 +38,77 @@ namespace network {
3738
namespace http {
3839
namespace impl {
3940

41+
template <class Tag>
42+
struct chunk_encoding_parser {
43+
44+
chunk_encoding_parser() : state(state_t::header), chunk_size(0) {}
45+
46+
enum state_t { header, header_end, data, data_end };
47+
48+
state_t state;
49+
size_t chunk_size;
50+
std::array<typename char_<Tag>::type, 1024> buffer;
51+
52+
void update_chunk_size(boost::iterator_range<typename std::array<typename char_<Tag>::type, 1024>::const_iterator> const& range) {
53+
if (range.empty())
54+
return;
55+
std::stringstream ss;
56+
ss << std::hex << range;
57+
size_t size;
58+
ss >> size;
59+
chunk_size = (chunk_size << (range.size()*4)) + size;
60+
}
61+
62+
boost::iterator_range<typename std::array<typename char_<Tag>::type, 1024>::const_iterator> operator()(boost::iterator_range<typename std::array<typename char_<Tag>::type, 1024>::const_iterator> const& range) {
63+
auto iter = boost::begin(range);
64+
auto begin = iter;
65+
auto pos = boost::begin(buffer);
66+
67+
while (iter != boost::end(range))
68+
switch(state) {
69+
case state_t::header:
70+
iter = std::find(iter, boost::end(range), '\r');
71+
update_chunk_size(boost::make_iterator_range(begin, iter));
72+
if (iter != boost::end(range)) {
73+
state = state_t::header_end;
74+
++iter;
75+
}
76+
break;
77+
78+
case state_t::header_end:
79+
BOOST_ASSERT(*iter == '\n');
80+
++iter;
81+
state = state_t::data;
82+
break;
83+
84+
case state_t::data:
85+
if (chunk_size == 0) {
86+
BOOST_ASSERT(*iter == '\r');
87+
++iter;
88+
state = state_t::data_end;
89+
} else {
90+
auto len = std::min(chunk_size, (size_t)std::distance(iter, boost::end(range)));
91+
begin = iter;
92+
iter = std::next(iter, len);
93+
pos = std::copy(begin, iter, pos);
94+
chunk_size -= len;
95+
}
96+
break;
97+
98+
case state_t::data_end:
99+
BOOST_ASSERT (*iter == '\n');
100+
++iter;
101+
begin = iter;
102+
state = state_t::header;
103+
break;
104+
105+
default:
106+
BOOST_ASSERT(false && "Bug, report this to the developers!");
107+
}
108+
return boost::make_iterator_range(boost::begin(buffer), pos);
109+
}
110+
};
111+
40112
template <class Tag, unsigned version_major, unsigned version_minor>
41113
struct async_connection_base;
42114

@@ -73,8 +145,10 @@ struct http_async_connection
73145

74146
http_async_connection(resolver_type& resolver, resolve_function resolve,
75147
bool follow_redirect, int timeout,
148+
bool remove_chunk_markers,
76149
connection_delegate_ptr delegate)
77150
: timeout_(timeout),
151+
remove_chunk_markers_(remove_chunk_markers),
78152
timer_(resolver.get_io_service()),
79153
is_timedout_(false),
80154
follow_redirect_(follow_redirect),
@@ -397,13 +471,23 @@ struct http_async_connection
397471
callback(make_iterator_range(begin, end), ec);
398472
} else {
399473
string_type body_string;
400-
std::swap(body_string, this->partial_parsed);
401-
auto it = this->part.begin();
402-
std::advance(it, bytes_transferred);
403-
body_string.append(this->part.begin(), it);
404-
if (this->is_chunk_encoding) {
405-
this->body_promise.set_value(parse_chunk_encoding(body_string));
474+
if (this->is_chunk_encoding && remove_chunk_markers_) {
475+
for (size_t i = 0; i < this->partial_parsed.size(); i += 1024) {
476+
auto range = parse_chunk_encoding(
477+
boost::make_iterator_range(this->partial_parsed.data() + i,
478+
this->partial_parsed.data() + std::min(i+1024, this->partial_parsed.size())));
479+
body_string.append(boost::begin(range), boost::end(range));
480+
}
481+
this->partial_parsed.clear();
482+
auto range = parse_chunk_encoding(boost::make_iterator_range(this->part.begin(),
483+
this->part.begin() + bytes_transferred));
484+
body_string.append(boost::begin(range), boost::end(range));
485+
this->body_promise.set_value(body_string);
406486
} else {
487+
std::swap(body_string, this->partial_parsed);
488+
auto it = this->part.begin();
489+
std::advance(it, bytes_transferred);
490+
body_string.append(this->part.begin(), it);
407491
this->body_promise.set_value(body_string);
408492
}
409493
}
@@ -520,6 +604,7 @@ struct http_async_connection
520604
}
521605

522606
int timeout_;
607+
bool remove_chunk_markers_;
523608
boost::asio::steady_timer timer_;
524609
bool is_timedout_;
525610
bool follow_redirect_;
@@ -529,6 +614,7 @@ struct http_async_connection
529614
connection_delegate_ptr delegate_;
530615
boost::asio::streambuf command_streambuf;
531616
string_type method;
617+
chunk_encoding_parser<Tag> parse_chunk_encoding;
532618
};
533619

534620
} // namespace impl

boost/network/protocol/http/client/facade.hpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,8 @@ class basic_client_facade {
312312
options.openssl_verify_path(), options.openssl_certificate_file(),
313313
options.openssl_private_key_file(), options.openssl_ciphers(),
314314
options.openssl_sni_hostname(), options.openssl_options(),
315-
options.io_service(), options.timeout()));
315+
options.io_service(), options.timeout(),
316+
options.remove_chunk_markers()));
316317
}
317318
};
318319

boost/network/protocol/http/client/options.hpp

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,8 @@ class client_options {
3434
openssl_options_(0),
3535
io_service_(),
3636
always_verify_peer_(true),
37-
timeout_(0) {}
37+
timeout_(0),
38+
remove_chunk_markers_(false) {}
3839

3940
client_options(client_options const& other)
4041
: cache_resolved_(other.cache_resolved_),
@@ -48,7 +49,8 @@ class client_options {
4849
openssl_options_(other.openssl_options_),
4950
io_service_(other.io_service_),
5051
always_verify_peer_(other.always_verify_peer_),
51-
timeout_(other.timeout_) {}
52+
timeout_(other.timeout_),
53+
remove_chunk_markers_(other.remove_chunk_markers) {}
5254

5355
client_options& operator=(client_options other) {
5456
other.swap(*this);
@@ -69,6 +71,7 @@ class client_options {
6971
swap(io_service_, other.io_service_);
7072
swap(always_verify_peer_, other.always_verify_peer_);
7173
swap(timeout_, other.timeout_);
74+
swap(remove_chunk_markers_, other.remove_chunk_markers_);
7275
}
7376

7477
/// Specify whether the client should cache resolved endpoints.
@@ -154,6 +157,12 @@ class client_options {
154157
return *this;
155158
}
156159

160+
/// Set an overall timeout for HTTP requests.
161+
client_options& remove_chunk_markers(bool v) {
162+
remove_chunk_markers_ = v;
163+
return *this;
164+
}
165+
157166
bool cache_resolved() const { return cache_resolved_; }
158167

159168
bool follow_redirects() const { return follow_redirects_; }
@@ -190,6 +199,8 @@ class client_options {
190199

191200
int timeout() const { return timeout_; }
192201

202+
bool remove_chunk_markers() const { return remove_chunk_markers_; }
203+
193204
private:
194205
bool cache_resolved_;
195206
bool follow_redirects_;
@@ -203,6 +214,7 @@ class client_options {
203214
std::shared_ptr<boost::asio::io_service> io_service_;
204215
bool always_verify_peer_;
205216
int timeout_;
217+
bool remove_chunk_markers_;
206218
};
207219

208220
template <class Tag>

boost/network/protocol/http/client/pimpl.hpp

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,12 @@ struct basic_client_impl
7474
optional<string_type> const& private_key_file,
7575
optional<string_type> const& ciphers,
7676
optional<string_type> const& sni_hostname, long ssl_options,
77-
std::shared_ptr<boost::asio::io_service> service, int timeout)
78-
: base_type(cache_resolved, follow_redirect, always_verify_peer, timeout,
79-
service, certificate_filename, verify_path, certificate_file,
80-
private_key_file, ciphers, sni_hostname, ssl_options) {}
77+
std::shared_ptr<boost::asio::io_service> service, int timeout,
78+
bool remove_chunk_markers)
79+
: base_type(cache_resolved, follow_redirect, always_verify_peer, timeout,
80+
remove_chunk_markers, service, certificate_filename, verify_path,
81+
certificate_file, private_key_file, ciphers, sni_hostname,
82+
ssl_options) {}
8183

8284
~basic_client_impl() = default;
8385
};

boost/network/protocol/http/policies/async_connection.hpp

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ struct async_connection_policy : resolver_policy<Tag>::type {
4040
struct connection_impl {
4141
connection_impl(
4242
bool follow_redirect, bool always_verify_peer, resolve_function resolve,
43-
resolver_type& resolver, bool https, int timeout,
43+
resolver_type& resolver, bool https, int timeout, bool remove_chunk_markers,
4444
optional<string_type> /*unused*/ const& certificate_filename,
4545
optional<string_type> const& verify_path,
4646
optional<string_type> const& certificate_file,
@@ -49,8 +49,8 @@ struct async_connection_policy : resolver_policy<Tag>::type {
4949
optional<string_type> const& sni_hostname, long ssl_options) {
5050
pimpl = impl::async_connection_base<Tag, version_major, version_minor>::
5151
new_connection(resolve, resolver, follow_redirect, always_verify_peer,
52-
https, timeout, certificate_filename, verify_path,
53-
certificate_file, private_key_file, ciphers,
52+
https, timeout, remove_chunk_markers, certificate_filename,
53+
verify_path, certificate_file, private_key_file, ciphers,
5454
sni_hostname, ssl_options);
5555
}
5656

@@ -87,21 +87,23 @@ struct async_connection_policy : resolver_policy<Tag>::type {
8787
std::uint16_t port, resolve_completion_function once_resolved) {
8888
this->resolve(resolver, host, port, once_resolved);
8989
},
90-
resolver, boost::iequals(protocol_, string_type("https")), timeout_,
90+
resolver, boost::iequals(protocol_, string_type("https")), timeout_, remove_chunk_markers_,
9191
certificate_filename, verify_path, certificate_file, private_key_file,
9292
ciphers, sni_hostname, ssl_options);
9393
}
9494

9595
void cleanup() {}
9696

9797
async_connection_policy(bool cache_resolved, bool follow_redirect,
98-
int timeout)
98+
int timeout, bool remove_chunk_markers)
9999
: resolver_base(cache_resolved),
100100
follow_redirect_(follow_redirect),
101-
timeout_(timeout) {}
101+
timeout_(timeout),
102+
remove_chunk_markers_(remove_chunk_markers) {}
102103

103104
bool follow_redirect_;
104105
int timeout_;
106+
bool remove_chunk_markers_;
105107
};
106108

107109
} // namespace http

0 commit comments

Comments
 (0)