@@ -33,14 +33,64 @@ internal partial class Http2Connection : IHttp2StreamLifetimeHandler, IHttpHeade
33
33
private const PseudoHeaderFields _mandatoryRequestPseudoHeaderFields =
34
34
PseudoHeaderFields . Method | PseudoHeaderFields . Path | PseudoHeaderFields . Scheme ;
35
35
36
+ private const string MaximumEnhanceYourCalmCountProperty = "Microsoft.AspNetCore.Server.Kestrel.Http2.MaxEnhanceYourCalmCount" ;
37
+ private const string MaximumFlowControlQueueSizeProperty = "Microsoft.AspNetCore.Server.Kestrel.Http2.MaxConnectionFlowControlQueueSize" ;
38
+
39
+ private static readonly int _enhanceYourCalmMaximumCount = GetMaximumEnhanceYourCalmCount ( ) ;
40
+
41
+ private static int GetMaximumEnhanceYourCalmCount ( )
42
+ {
43
+ var data = AppContext . GetData ( MaximumEnhanceYourCalmCountProperty ) ;
44
+ if ( data is int count )
45
+ {
46
+ return count ;
47
+ }
48
+ if ( data is string countStr && int . TryParse ( countStr , out var parsed ) )
49
+ {
50
+ return parsed ;
51
+ }
52
+
53
+ return 20 ; // Empirically derived
54
+ }
55
+
56
+ // Accumulate _enhanceYourCalmCount over the course of EnhanceYourCalmTickWindowCount ticks.
57
+ // This should make bursts less likely to trigger disconnects.
58
+ private const int EnhanceYourCalmTickWindowCount = 5 ;
59
+
60
+ private static bool IsEnhanceYourCalmEnabled => _enhanceYourCalmMaximumCount > 0 ;
61
+
62
+ private static readonly int ? ConfiguredMaximumFlowControlQueueSize = GetConfiguredMaximumFlowControlQueueSize ( ) ;
63
+
64
+ private static int ? GetConfiguredMaximumFlowControlQueueSize ( )
65
+ {
66
+ var data = AppContext . GetData ( MaximumFlowControlQueueSizeProperty ) ;
67
+
68
+ if ( data is int count )
69
+ {
70
+ return count ;
71
+ }
72
+
73
+ if ( data is string countStr && int . TryParse ( countStr , out var parsed ) )
74
+ {
75
+ return parsed ;
76
+ }
77
+
78
+ return null ;
79
+ }
80
+
81
+ private readonly int _maximumFlowControlQueueSize ;
82
+
83
+ private bool IsMaximumFlowControlQueueSizeEnabled => _maximumFlowControlQueueSize > 0 ;
84
+
36
85
private readonly HttpConnectionContext _context ;
37
86
private readonly Http2FrameWriter _frameWriter ;
38
87
private readonly Pipe _input ;
39
88
private readonly Task _inputTask ;
40
89
private readonly int _minAllocBufferSize ;
41
90
private readonly HPackDecoder _hpackDecoder ;
42
91
private readonly InputFlowControl _inputFlowControl ;
43
- private readonly OutputFlowControl _outputFlowControl = new OutputFlowControl ( new MultipleAwaitableProvider ( ) , Http2PeerSettings . DefaultInitialWindowSize ) ;
92
+ private readonly OutputFlowControl _outputFlowControl ;
93
+ private readonly AwaitableProvider _outputFlowControlAwaitableProvider ; // Keep our own reference so we can track queue size
44
94
45
95
private readonly Http2PeerSettings _serverSettings = new Http2PeerSettings ( ) ;
46
96
private readonly Http2PeerSettings _clientSettings = new Http2PeerSettings ( ) ;
@@ -59,6 +109,9 @@ internal partial class Http2Connection : IHttp2StreamLifetimeHandler, IHttpHeade
59
109
private int _clientActiveStreamCount ;
60
110
private int _serverActiveStreamCount ;
61
111
112
+ private int _enhanceYourCalmCount ;
113
+ private int _tickCount ;
114
+
62
115
// The following are the only fields that can be modified outside of the ProcessRequestsAsync loop.
63
116
private readonly ConcurrentQueue < Http2Stream > _completedStreams = new ConcurrentQueue < Http2Stream > ( ) ;
64
117
private readonly StreamCloseAwaitable _streamCompletionAwaitable = new StreamCloseAwaitable ( ) ;
@@ -88,6 +141,9 @@ public Http2Connection(HttpConnectionContext context)
88
141
// Capture the ExecutionContext before dispatching HTTP/2 middleware. Will be restored by streams when processing request
89
142
_context . InitialExecutionContext = ExecutionContext . Capture ( ) ;
90
143
144
+ _outputFlowControlAwaitableProvider = new MultipleAwaitableProvider ( ) ;
145
+ _outputFlowControl = new OutputFlowControl ( _outputFlowControlAwaitableProvider , Http2PeerSettings . DefaultInitialWindowSize ) ;
146
+
91
147
_frameWriter = new Http2FrameWriter (
92
148
context . Transport . Output ,
93
149
context . ConnectionContext ,
@@ -129,6 +185,17 @@ public Http2Connection(HttpConnectionContext context)
129
185
_serverSettings . MaxHeaderListSize = ( uint ) httpLimits . MaxRequestHeadersTotalSize ;
130
186
_serverSettings . InitialWindowSize = ( uint ) http2Limits . InitialStreamWindowSize ;
131
187
188
+ _maximumFlowControlQueueSize = ConfiguredMaximumFlowControlQueueSize is null
189
+ ? 4 * http2Limits . MaxStreamsPerConnection // 4 is a magic number to give us some padding above the expected maximum size
190
+ : ( int ) ConfiguredMaximumFlowControlQueueSize ;
191
+
192
+ var minimumMaximumFlowControlQueueSize = 2 * http2Limits . MaxStreamsPerConnection ; // Double to match 7.0 and 8.0
193
+ if ( IsMaximumFlowControlQueueSizeEnabled && _maximumFlowControlQueueSize < minimumMaximumFlowControlQueueSize )
194
+ {
195
+ Log . Http2FlowControlQueueMaximumTooLow ( context . ConnectionId , minimumMaximumFlowControlQueueSize , _maximumFlowControlQueueSize ) ;
196
+ _maximumFlowControlQueueSize = minimumMaximumFlowControlQueueSize ;
197
+ }
198
+
132
199
// Start pool off at a smaller size if the max number of streams is less than the InitialStreamPoolSize
133
200
StreamPool = new PooledStreamStack < Http2Stream > ( Math . Min ( InitialStreamPoolSize , http2Limits . MaxStreamsPerConnection ) ) ;
134
201
@@ -352,13 +419,20 @@ public async Task ProcessRequestsAsync<TContext>(IHttpApplication<TContext> appl
352
419
stream . Abort ( new IOException ( CoreStrings . Http2StreamAborted , connectionError ) ) ;
353
420
}
354
421
355
- // Use the server _serverActiveStreamCount to drain all requests on the server side.
356
- // Can't use _clientActiveStreamCount now as we now decrement that count earlier/
357
- // Can't use _streams.Count as we wait for RST/END_STREAM before removing the stream from the dictionary
358
- while ( _serverActiveStreamCount > 0 )
422
+ // For some reason, this loop doesn't terminate when we're trying to abort.
423
+ // Since we're making a narrow fix for a patch, we'll bypass it in such scenarios.
424
+ // TODO: This is probably a bug - something in here should probably detect aborted
425
+ // connections and short-circuit.
426
+ if ( ! ( IsEnhanceYourCalmEnabled || IsMaximumFlowControlQueueSizeEnabled ) || error is not Http2ConnectionErrorException )
359
427
{
360
- await _streamCompletionAwaitable ;
361
- UpdateCompletedStreams ( ) ;
428
+ // Use the server _serverActiveStreamCount to drain all requests on the server side.
429
+ // Can't use _clientActiveStreamCount now as we now decrement that count earlier/
430
+ // Can't use _streams.Count as we wait for RST/END_STREAM before removing the stream from the dictionary
431
+ while ( _serverActiveStreamCount > 0 )
432
+ {
433
+ await _streamCompletionAwaitable ;
434
+ UpdateCompletedStreams ( ) ;
435
+ }
362
436
}
363
437
364
438
while ( StreamPool . TryPop ( out var pooledStream ) )
@@ -1053,6 +1127,20 @@ private void StartStream()
1053
1127
throw new Http2StreamErrorException ( _currentHeadersStream . StreamId , CoreStrings . Http2ErrorMaxStreams , Http2ErrorCode . REFUSED_STREAM ) ;
1054
1128
}
1055
1129
1130
+ if ( IsMaximumFlowControlQueueSizeEnabled && _outputFlowControlAwaitableProvider . ActiveCount > _maximumFlowControlQueueSize )
1131
+ {
1132
+ Log . Http2FlowControlQueueOperationsExceeded ( _context . ConnectionId , _maximumFlowControlQueueSize ) ;
1133
+
1134
+ // Now that we've logged a useful message, we can put vague text in the exception
1135
+ // messages in case they somehow make it back to the client (not expected)
1136
+
1137
+ // This will close the socket - we want to do that right away
1138
+ Abort ( new ConnectionAbortedException ( "HTTP/2 connection exceeded the outgoing flow control maximum queue size." ) ) ;
1139
+
1140
+ // Throwing an exception as well will help us clean up on our end more quickly by (e.g.) skipping processing of already-buffered input
1141
+ throw new Http2ConnectionErrorException ( CoreStrings . Http2ConnectionFaulted , Http2ErrorCode . INTERNAL_ERROR ) ;
1142
+ }
1143
+
1056
1144
// We don't use the _serverActiveRequestCount here as during shutdown, it and the dictionary counts get out of sync.
1057
1145
// The streams still exist in the dictionary until the client responds with a RST or END_STREAM.
1058
1146
// Also, we care about the dictionary size for too much memory consumption.
@@ -1061,6 +1149,20 @@ private void StartStream()
1061
1149
// Server is getting hit hard with connection resets.
1062
1150
// Tell client to calm down.
1063
1151
// TODO consider making when to send ENHANCE_YOUR_CALM configurable?
1152
+
1153
+ if ( IsEnhanceYourCalmEnabled && Interlocked . Increment ( ref _enhanceYourCalmCount ) > EnhanceYourCalmTickWindowCount * _enhanceYourCalmMaximumCount )
1154
+ {
1155
+ Log . Http2TooManyEnhanceYourCalms ( _context . ConnectionId , _enhanceYourCalmMaximumCount ) ;
1156
+
1157
+ // Now that we've logged a useful message, we can put vague text in the exception
1158
+ // messages in case they somehow make it back to the client (not expected)
1159
+
1160
+ // This will close the socket - we want to do that right away
1161
+ Abort ( new ConnectionAbortedException ( CoreStrings . Http2ConnectionFaulted ) ) ;
1162
+ // Throwing an exception as well will help us clean up on our end more quickly by (e.g.) skipping processing of already-buffered input
1163
+ throw new Http2ConnectionErrorException ( CoreStrings . Http2ConnectionFaulted , Http2ErrorCode . ENHANCE_YOUR_CALM ) ;
1164
+ }
1165
+
1064
1166
throw new Http2StreamErrorException ( _currentHeadersStream . StreamId , CoreStrings . Http2TellClientToCalmDown , Http2ErrorCode . ENHANCE_YOUR_CALM ) ;
1065
1167
}
1066
1168
}
@@ -1123,6 +1225,12 @@ private void AbortStream(int streamId, IOException error)
1123
1225
void IRequestProcessor . Tick ( DateTimeOffset now )
1124
1226
{
1125
1227
Input . CancelPendingRead ( ) ;
1228
+ // We count EYCs over a window of a given length to avoid flagging short-lived bursts.
1229
+ // At the end of each window, reset the count.
1230
+ if ( IsEnhanceYourCalmEnabled && ++ _tickCount % EnhanceYourCalmTickWindowCount == 0 )
1231
+ {
1232
+ Interlocked . Exchange ( ref _enhanceYourCalmCount , 0 ) ;
1233
+ }
1126
1234
}
1127
1235
1128
1236
void IHttp2StreamLifetimeHandler . OnStreamCompleted ( Http2Stream stream )
0 commit comments