Coverage for manila/api/v2/limits.py: 94%

1# Copyright 2011 OpenStack LLC. 

2# All Rights Reserved. 

3# 

4# Licensed under the Apache License, Version 2.0 (the "License"); you may 

5# not use this file except in compliance with the License. You may obtain 

6# a copy of the License at 

7# 

8# http://www.apache.org/licenses/LICENSE-2.0 

9# 

10# Unless required by applicable law or agreed to in writing, software 

11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 

12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 

13# License for the specific language governing permissions and limitations 

14# under the License. 

15 

16""" 

17Module dedicated functions/classes dealing with rate limiting requests. 

18""" 

19 

20import collections 

21import copy 

22from http import client as http_client 

23import math 

24import re 

25import time 

26 

27from oslo_serialization import jsonutils 

28from oslo_utils import importutils 

29import webob.dec 

30import webob.exc 

31 

32from manila.api.openstack import wsgi 

33from manila.api.views import limits as limits_views 

34from manila.i18n import _ 

35from manila import quota 

36from manila.wsgi import common as base_wsgi 

37 

38QUOTAS = quota.QUOTAS 

39 

40 

41# Convenience constants for the limits dictionary passed to Limiter(). 

42PER_SECOND = 1 

43PER_MINUTE = 60 

44PER_HOUR = 60 * 60 

45PER_DAY = 60 * 60 * 24 

46 

47 

48class LimitsController(wsgi.Controller): 

49 """Controller for accessing limits in the OpenStack API.""" 

50 

51 def index(self, req): 

52 """Return all global and rate limit information.""" 

53 context = req.environ['manila.context'] 

54 quotas = QUOTAS.get_project_quotas(context, context.project_id, 

55 usages=True) 

56 abs_limits = {'in_use': {}, 'limit': {}} 

57 for k, v in quotas.items(): 

58 abs_limits['limit'][k] = v['limit'] 

59 abs_limits['in_use'][k] = v['in_use'] 

60 rate_limits = req.environ.get("manila.limits", []) 

61 

62 builder = self._get_view_builder(req) 

63 return builder.build(req, rate_limits, abs_limits) 

64 

65 def _get_view_builder(self, req): 

66 return limits_views.ViewBuilder() 

67 

68 

69def create_resource(): 

70 return wsgi.Resource(LimitsController()) 

71 

72 

73class Limit(object): 

74 """Stores information about a limit for HTTP requests.""" 

75 

76 UNITS = { 

77 1: "SECOND", 

78 60: "MINUTE", 

79 60 * 60: "HOUR", 

80 60 * 60 * 24: "DAY", 

81 } 

82 

83 UNIT_MAP = {v: k for k, v in UNITS.items()} 

84 

85 def __init__(self, verb, uri, regex, value, unit): 

86 """Initialize a new `Limit`. 

87 

88 @param verb: HTTP verb (POST, PUT, etc.) 

89 @param uri: Human-readable URI 

90 @param regex: Regular expression format for this limit 

91 @param value: Integer number of requests which can be made 

92 @param unit: Unit of measure for the value parameter 

93 """ 

94 self.verb = verb 

95 self.uri = uri 

96 self.regex = regex 

97 self.value = int(value) 

98 self.unit = unit 

99 self.unit_string = self.display_unit().lower() 

100 self.remaining = int(value) 

101 

102 if value <= 0: 102 ↛ 103line 102 didn't jump to line 103 because the condition on line 102 was never true

103 raise ValueError("Limit value must be > 0") 

104 

105 self.last_request = None 

106 self.next_request = None 

107 

108 self.water_level = 0 

109 self.capacity = self.unit 

110 self.request_value = float(self.capacity) / float(self.value) 

111 msg = (_("Only %(value)s %(verb)s request(s) can be " 

112 "made to %(uri)s every %(unit_string)s.") % 

113 {'value': self.value, 'verb': self.verb, 

114 'uri': self.uri, 'unit_string': self.unit_string}) 

115 self.error_message = msg 

116 

117 def __call__(self, verb, url): 

118 """Represents a call to this limit from a relevant request. 

119 

120 @param verb: string http verb (POST, GET, etc.) 

121 @param url: string URL 

122 """ 

123 if self.verb != verb or not re.match(self.regex, url): 

124 return 

125 

126 now = self._get_time() 

127 

128 if self.last_request is None: 

129 self.last_request = now 

130 

131 leak_value = now - self.last_request 

132 

133 self.water_level -= leak_value 

134 self.water_level = max(self.water_level, 0) 

135 self.water_level += self.request_value 

136 

137 difference = self.water_level - self.capacity 

138 

139 self.last_request = now 

140 

141 if difference > 0: 

142 self.water_level -= self.request_value 

143 self.next_request = now + difference 

144 return difference 

145 

146 cap = self.capacity 

147 water = self.water_level 

148 val = self.value 

149 

150 self.remaining = math.floor(((cap - water) / cap) * val) 

151 self.next_request = now 

152 

153 def _get_time(self): 

154 """Retrieve the current time. Broken out for testability.""" 

155 return time.time() 

156 

157 def display_unit(self): 

158 """Display the string name of the unit.""" 

159 return self.UNITS.get(self.unit, "UNKNOWN") 

160 

161 def display(self): 

162 """Return a useful representation of this class.""" 

163 return { 

164 "verb": self.verb, 

165 "URI": self.uri, 

166 "regex": self.regex, 

167 "value": self.value, 

168 "remaining": int(self.remaining), 

169 "unit": self.display_unit(), 

170 "resetTime": int(self.next_request or self._get_time()), 

171 } 

172 

173 

174# "Limit" format is a dictionary with the HTTP verb, human-readable URI, 

175# a regular-expression to match, value and unit of measure (PER_DAY, etc.) 

176 

177DEFAULT_LIMITS = [ 

178 Limit("POST", "*", ".*", 10, PER_MINUTE), 

179 Limit("POST", "*/servers", "^/servers", 50, PER_DAY), 

180 Limit("PUT", "*", ".*", 10, PER_MINUTE), 

181 Limit("GET", "*changes-since*", ".*changes-since.*", 3, PER_MINUTE), 

182 Limit("DELETE", "*", ".*", 100, PER_MINUTE), 

183] 

184 

185 

186class RateLimitingMiddleware(base_wsgi.Middleware): 

187 """Rate-limits requests passing through this middleware. 

188 

189 All limit information is stored in memory for this implementation. 

190 """ 

191 

192 def __init__(self, application, limits=None, limiter=None, **kwargs): 

193 """Initialize new `RateLimitingMiddleware`. 

194 

195 `RateLimitingMiddleware` wraps the given WSGI application and 

196 sets up the given limits. 

197 

198 @param application: WSGI application to wrap 

199 @param limits: String describing limits 

200 @param limiter: String identifying class for representing limits 

201 

202 Other parameters are passed to the constructor for the limiter. 

203 """ 

204 base_wsgi.Middleware.__init__(self, application) 

205 

206 # Select the limiter class 

207 if limiter is None: 207 ↛ 208line 207 didn't jump to line 208 because the condition on line 207 was never true

208 limiter = Limiter 

209 else: 

210 limiter = importutils.import_class(limiter) 

211 

212 # Parse the limits, if any are provided 

213 if limits is not None: 213 ↛ 216line 213 didn't jump to line 216 because the condition on line 213 was always true

214 limits = limiter.parse_limits(limits) 

215 

216 self._limiter = limiter(limits or DEFAULT_LIMITS, **kwargs) 

217 

218 @webob.dec.wsgify(RequestClass=wsgi.Request) 

219 def __call__(self, req): 

220 """Represents a single call through this middleware. 

221 

222 We should record the request if we have a limit relevant to 

223 it. If no limit is relevant to the request, ignore it. 

224 

225 If the request should be rate limited, return a fault telling 

226 the user they are over the limit and need to retry later. 

227 """ 

228 verb = req.method 

229 url = req.url 

230 context = req.environ.get("manila.context") 

231 

232 if context: 232 ↛ 233line 232 didn't jump to line 233 because the condition on line 232 was never true

233 username = context.user_id 

234 else: 

235 username = None 

236 

237 delay, error = self._limiter.check_for_delay(verb, url, username) 

238 

239 if delay: 

240 msg = _("This request was rate-limited.") 

241 retry = time.time() + delay 

242 return wsgi.OverLimitFault(msg, error, retry) 

243 

244 req.environ["manila.limits"] = self._limiter.get_limits(username) 

245 

246 return self.application 

247 

248 

249class Limiter(object): 

250 """Rate-limit checking class which handles limits in memory.""" 

251 

252 def __init__(self, limits, **kwargs): 

253 """Initialize the new `Limiter`. 

254 

255 @param limits: List of `Limit` objects 

256 """ 

257 self.limits = copy.deepcopy(limits) 

258 self.levels = collections.defaultdict(lambda: copy.deepcopy(limits)) 

259 

260 # Pick up any per-user limit information 

261 for key, value in kwargs.items(): 

262 if key.startswith('user:'): 262 ↛ 261line 262 didn't jump to line 261 because the condition on line 262 was always true

263 username = key[5:] 

264 self.levels[username] = self.parse_limits(value) 

265 

266 def get_limits(self, username=None): 

267 """Return the limits for a given user.""" 

268 return [limit.display() for limit in self.levels[username]] 

269 

270 def check_for_delay(self, verb, url, username=None): 

271 """Check the given verb/user/user triplet for limit. 

272 

273 @return: Tuple of delay (in seconds) and error message (or None, None) 

274 """ 

275 delays = [] 

276 

277 for limit in self.levels[username]: 

278 delay = limit(verb, url) 

279 if delay: 

280 delays.append((delay, limit.error_message)) 

281 

282 if delays: 

283 delays.sort() 

284 return delays[0] 

285 

286 return None, None 

287 

288 # Note: This method gets called before the class is instantiated, 

289 # so this must be either a static method or a class method. It is 

290 # used to develop a list of limits to feed to the constructor. We 

291 # put this in the class so that subclasses can override the 

292 # default limit parsing. 

293 @staticmethod 

294 def parse_limits(limits): 

295 """Convert a string into a list of Limit instances. 

296 

297 This implementation expects a semicolon-separated sequence of 

298 parenthesized groups, where each group contains a 

299 comma-separated sequence consisting of HTTP method, 

300 user-readable URI, a URI reg-exp, an integer number of 

301 requests which can be made, and a unit of measure. Valid 

302 values for the latter are "SECOND", "MINUTE", "HOUR", and 

303 "DAY". 

304 

305 @return: List of Limit instances. 

306 """ 

307 

308 # Handle empty limit strings 

309 limits = limits.strip() 

310 if not limits: 

311 return [] 

312 

313 # Split up the limits by semicolon 

314 result = [] 

315 for group in limits.split(';'): 

316 group = group.strip() 

317 if group[:1] != '(' or group[-1:] != ')': 

318 raise ValueError("Limit rules must be surrounded by " 

319 "parentheses") 

320 group = group[1:-1] 

321 

322 # Extract the Limit arguments 

323 args = [a.strip() for a in group.split(',')] 

324 if len(args) != 5: 

325 raise ValueError("Limit rules must contain the following " 

326 "arguments: verb, uri, regex, value, unit") 

327 

328 # Pull out the arguments 

329 verb, uri, regex, value, unit = args 

330 

331 # Upper-case the verb 

332 verb = verb.upper() 

333 

334 # Convert value--raises ValueError if it's not integer 

335 value = int(value) 

336 

337 # Convert unit 

338 unit = unit.upper() 

339 if unit not in Limit.UNIT_MAP: 

340 raise ValueError("Invalid units specified") 

341 unit = Limit.UNIT_MAP[unit] 

342 

343 # Build a limit 

344 result.append(Limit(verb, uri, regex, value, unit)) 

345 

346 return result 

347 

348 

349class WsgiLimiter(object): 

350 """Rate-limit checking from a WSGI application. 

351 

352 Uses an in-memory `Limiter`. 

353 To use, POST ``/<username>`` with JSON data such as:: 

354 

355 { 

356 "verb" : GET, 

357 "path" : "/servers" 

358 } 

359 

360 and receive a 204 No Content, or a 403 Forbidden with an X-Wait-Seconds 

361 header containing the number of seconds to wait before the action would 

362 succeed. 

363 """ 

364 

365 def __init__(self, limits=None): 

366 """Initialize the new `WsgiLimiter`. 

367 

368 @param limits: List of `Limit` objects 

369 """ 

370 self._limiter = Limiter(limits or DEFAULT_LIMITS) 

371 

372 @webob.dec.wsgify(RequestClass=wsgi.Request) 

373 def __call__(self, request): 

374 """Handles a call to this application. 

375 

376 Returns 204 if the request is acceptable to the limiter, else 

377 a 403 is returned with a relevant header indicating when the 

378 request *will* succeed. 

379 """ 

380 if request.method != "POST": 

381 raise webob.exc.HTTPMethodNotAllowed() 

382 

383 try: 

384 info = dict(jsonutils.loads(request.body)) 

385 except ValueError: 

386 raise webob.exc.HTTPBadRequest() 

387 

388 username = request.path_info_pop() 

389 verb = info.get("verb") 

390 path = info.get("path") 

391 

392 delay, error = self._limiter.check_for_delay(verb, path, username) 

393 

394 if delay: 

395 headers = {"X-Wait-Seconds": "%.2f" % delay} 

396 return webob.exc.HTTPForbidden(headers=headers, explanation=error) 

397 else: 

398 return webob.exc.HTTPNoContent() 

399 

400 

401class WsgiLimiterProxy(object): 

402 """Rate-limit requests based on answers from a remote source.""" 

403 

404 def __init__(self, limiter_address): 

405 """Initialize the new `WsgiLimiterProxy`. 

406 

407 @param limiter_address: IP/port combination of where to request limit 

408 """ 

409 self.limiter_address = limiter_address 

410 

411 def check_for_delay(self, verb, path, username=None): 

412 body = jsonutils.dumps({"verb": verb, "path": path}) 

413 headers = {"Content-Type": "application/json"} 

414 

415 conn = http_client.HTTPConnection(self.limiter_address) 

416 

417 if username: 417 ↛ 418line 417 didn't jump to line 418 because the condition on line 417 was never true

418 conn.request("POST", "/%s" % (username), body, headers) 

419 else: 

420 conn.request("POST", "/", body, headers) 

421 

422 resp = conn.getresponse() 

423 

424 if resp.status >= 200 and resp.status < 300: 

425 # there's nothing to rate-limit 

426 return None, None 

427 

428 return resp.getheader("X-Wait-Seconds"), resp.read() or None 

429 

430 # Note: This method gets called before the class is instantiated, 

431 # so this must be either a static method or a class method. It is 

432 # used to develop a list of limits to feed to the constructor. 

433 # This implementation returns an empty list, since all limit 

434 # decisions are made by a remote server. 

435 @staticmethod 

436 def parse_limits(limits): 

437 """Ignore a limits string. 

438 

439 This simply doesn't apply for the limit proxy. 

440 

441 @return: Empty list. 

442 """ 

443 

444 return []