Merge pull request #1938 from pbiering/add-propfind_max_resource_size

Add propfind max resource size support for PROPFIND and PUT

CHANGELOG.md

@@ -10,6 +10,8 @@
 * Adjust: [logging] header/content debug log indended by space to be skipped by logwatch
 * Improve: remove unnecessary open+read for mtime+size cache
 * Extend: add selected XML query properties to request result log line for improved timing analysis incl. logwatch support
+* Add: [server] max_resource_size option
+* Add: support PROPFIND/max-resource-size by max_resource_size (capped to 80% of max_content_length)
 
 ## 3.5.9
 * Extend: [auth] add support for type http_remote_user

DOCUMENTATION.md

@@ -797,9 +797,21 @@ Default: `8`
 
 The maximum size of the request body. (bytes)
 
-Default: `100000000`
+Default: `100000000` (100 Mbyte)
 
-In case of using a reverse proxy in front of check also there related option
+In case of using a reverse proxy in front of check also there related option.
+
+##### max_resource_size
+
+_(>= 3.5.10)_
+
+The maximum size of a resource. (bytes)
+
+Default: `10000000` (10 Mbyte)
+
+Limited to 80% of max_content_length to cover plain base64 encoded payload.
+
+Announced to clients requesting "max-resource-size" via PROPFIND.
 
 ##### timeout
 

config

@@ -21,10 +21,15 @@
 # Max parallel connections
 #max_connections = 8
 
-# Max size of request body (bytes)
+# Max size of request body (bytes), default: 100 Mbyte
 # In case of using a reverse proxy in front of check also there related option
 #max_content_length = 100000000
 
+# Max resource size (bytes), default: 10 Mbyte
+# Limited to 80% of max_content_length to cover plain base64 encoded payload
+# Announced to clients requesting "max-resource-size" via PROPFIND
+#max_ressource_size = 10000000
+
 # Socket timeout (seconds)
 #timeout = 30
 

radicale/app/__init__.py

@@ -73,6 +73,7 @@ class Application(ApplicationPartDelete, ApplicationPartHead,
     _auth_delay: float
     _internal_server: bool
     _max_content_length: int
+    _max_resource_size: int
     _auth_realm: str
     _auth_type: str
     _web_type: str
@@ -95,6 +96,14 @@ class Application(ApplicationPartDelete, ApplicationPartHead,
         """
         super().__init__(configuration)
         self._mask_passwords = configuration.get("logging", "mask_passwords")
+        self._max_content_length = configuration.get("server", "max_content_length")
+        self._max_resource_size = configuration.get("server", "max_resource_size")
+        if (self._max_resource_size > (self._max_content_length * 0.8)):
+            max_resource_size_limited = int(self._max_content_length * 0.8)
+            logger.warning("max_resource_size capped to: %d bytes (from %d to 80%% of max_content_length %d)", max_resource_size_limited, self._max_resource_size, self._max_content_length)
+            self._max_resource_size = max_resource_size_limited
+        else:
+            logger.info("max_resource_size set to: %d bytes", self._max_resource_size)
         self._bad_put_request_content = configuration.get("logging", "bad_put_request_content")
         logger.info("log bad put request content: %s", self._bad_put_request_content)
         self._request_header_on_debug = configuration.get("logging", "request_header_on_debug")

radicale/app/base.py

@@ -39,6 +39,7 @@ class ApplicationBase:
     _rights: rights.BaseRights
     _web: web.BaseWeb
     _encoding: str
+    _max_resource_size: int
     _permit_delete_collection: bool
     _permit_overwrite_collection: bool
     _strict_preconditions: bool

radicale/app/propfind.py

@@ -34,7 +34,7 @@ from radicale.log import logger
 def xml_propfind(base_prefix: str, path: str,
                  xml_request: Optional[ET.Element],
                  allowed_items: Iterable[Tuple[types.CollectionOrItem, str]],
-                 user: str, encoding: str) -> Optional[ET.Element]:
+                 user: str, encoding: str, max_resource_size: int) -> Optional[ET.Element]:
     """Read and answer PROPFIND requests.
 
     Read rfc4918-9.1 for info.
@@ -71,14 +71,14 @@ def xml_propfind(base_prefix: str, path: str,
         write = permission == "w"
         multistatus.append(xml_propfind_response(
             base_prefix, path, item, props, user, encoding, write=write,
-            allprop=allprop, propname=propname))
+            allprop=allprop, propname=propname, max_resource_size=max_resource_size))
 
     return multistatus
 
 
 def xml_propfind_response(
         base_prefix: str, path: str, item: types.CollectionOrItem,
-        props: Sequence[str], user: str, encoding: str, write: bool = False,
+        props: Sequence[str], user: str, encoding: str, max_resource_size: int, write: bool = False,
         propname: bool = False, allprop: bool = False) -> ET.Element:
     """Build and return a PROPFIND response."""
     if propname and allprop or (props and (propname or allprop)):
@@ -111,6 +111,9 @@ def xml_propfind_response(
         props.append(xmlutils.make_clark("D:supported-report-set"))
         props.append(xmlutils.make_clark("D:resourcetype"))
         props.append(xmlutils.make_clark("D:owner"))
+        if not allprop:
+            # RFC4791#5.2.5: SHOULD NOT be returned by a PROPFIND DAV:allprop request
+            props.append(xmlutils.make_clark("C:max-resource-size"))
 
         if is_collection and collection.is_principal:
             props.append(xmlutils.make_clark("C:calendar-user-address-set"))
@@ -239,6 +242,9 @@ def xml_propfind_response(
                 child_element.text = xmlutils.make_href(
                     base_prefix, "/%s/" % collection.owner)
                 element.append(child_element)
+        elif tag == xmlutils.make_clark("C:max-resource-size"):
+            # RFC4791#5.2.5
+            element.text = str(max_resource_size)
         elif is_collection:
             if tag == xmlutils.make_clark("D:getcontenttype"):
                 if is_leaf:
@@ -407,7 +413,7 @@ class ApplicationPartPropfind(ApplicationBase):
             headers = {"DAV": httputils.DAV_HEADERS,
                        "Content-Type": "text/xml; charset=%s" % self._encoding}
             xml_answer = xml_propfind(base_prefix, path, xml_content,
-                                      allowed_items, user, self._encoding)
+                                      allowed_items, user, self._encoding, max_resource_size=self._max_resource_size)
             if xml_answer is None:
                 return httputils.NOT_ALLOWED
             return client.MULTI_STATUS, headers, self._xml_response(xml_answer), xmlutils.pretty_xml(xml_content)

radicale/app/put.py

@@ -46,7 +46,7 @@ PRODID = u"-//Radicale//NONSGML Version " + utils.package_version("radicale") +
 
 
 def prepare(vobject_items: List[vobject.base.Component], path: str,
-            content_type: str, permission: bool, parent_permission: bool,
+            content_type: str, permission: bool, parent_permission: bool, max_resource_size: int,
             tag: Optional[str] = None,
             write_whole_collection: Optional[bool] = None) -> Tuple[
                 Iterator[radicale_item.Item],  # items
@@ -103,6 +103,13 @@ def prepare(vobject_items: List[vobject.base.Component], path: str,
                         else:
                             logger.warning("Problem during prepare item with UID '%s' (content suppressed in this loglevel): %s", item.uid, e)
                         raise
+                    size = len(item.serialize())
+                    if (size > max_resource_size):
+                        logger.warning("PUT request contains item with UID %r size %d > limit %d: %r", item.uid, size, max_resource_size, path)
+                        # Use OverflowError as flag for max_resource_size
+                        raise OverflowError
+                    else:
+                        logger.debug("PUT request contains item with UID %r size %d <= limit %d: %r", item.uid, size, max_resource_size, path)
                     items.append(item)
             elif write_whole_collection and tag == "VADDRESSBOOK":
                 for vobject_item in vobject_items:
@@ -121,12 +128,26 @@ def prepare(vobject_items: List[vobject.base.Component], path: str,
                         else:
                             logger.warning("Problem during prepare item with UID '%s' (content suppressed in this loglevel): %s", item.uid, e)
                         raise
+                    size = len(item.serialize())
+                    if (size > max_resource_size):
+                        logger.warning("PUT request contains item with UID %r size %d > limit %d: %r", item.uid, size, max_resource_size, path)
+                        # Use OverflowError as flag for max_resource_size
+                        raise OverflowError
+                    else:
+                        logger.debug("PUT request contains item with UID %r size %d <= limit %d: %r", item.uid, size, max_resource_size, path)
                     items.append(item)
             elif not write_whole_collection:
                 vobject_item, = vobject_items
                 item = radicale_item.Item(collection_path=collection_path,
                                           vobject_item=vobject_item)
                 item.prepare()
+                size = len(item.serialize())
+                if (size > max_resource_size):
+                    logger.warning("PUT request contains item with UID %r size %d above limit %d: %r", item.uid, size, max_resource_size, path)
+                    # Use OverflowError as flag for max_resource_size
+                    raise OverflowError
+                else:
+                    logger.debug("PUT request contains item with UID %r size %d below limit %d: %r", item.uid, size, max_resource_size, path)
                 items.append(item)
 
         if write_whole_collection:
@@ -188,7 +209,8 @@ class ApplicationPartPut(ApplicationBase):
          prepared_props, prepared_exc_info) = prepare(
              vobject_items, path, content_type,
              bool(rights.intersect(access.permissions, "Ww")),
-             bool(rights.intersect(access.parent_permissions, "w")))
+             bool(rights.intersect(access.parent_permissions, "w")),
+             self._max_resource_size)
 
         with self._storage.acquire_lock("w", user, path=path, request="PUT"):
             item = next(iter(self._storage.discover(path)), None)
@@ -252,13 +274,18 @@ class ApplicationPartPut(ApplicationBase):
                      vobject_items, path, content_type,
                      bool(rights.intersect(access.permissions, "Ww")),
                      bool(rights.intersect(access.parent_permissions, "w")),
+                     self._max_resource_size,
                      tag, write_whole_collection)
             props = prepared_props
             if prepared_exc_info:
-                logger.warning(
-                    "Bad PUT request on %r (prepare): %s", path, prepared_exc_info[1],
-                    exc_info=prepared_exc_info)
-                return httputils.BAD_REQUEST
+                # Use OverflowError as flag for max_resource_size
+                if prepared_exc_info[0] == OverflowError:
+                    return httputils.PRECONDITION_FAILED
+                else:
+                    logger.warning(
+                        "Bad PUT request on %r (prepare): %s", path, prepared_exc_info[1],
+                        exc_info=prepared_exc_info)
+                    return httputils.BAD_REQUEST
 
             if write_whole_collection:
                 try:

radicale/config.py

@@ -162,7 +162,11 @@ DEFAULT_CONFIG_SCHEMA: types.CONFIG_SCHEMA = OrderedDict([
             "type": positive_int}),
         ("max_content_length", {
             "value": "100000000",
-            "help": "maximum size of request body in bytes",
+            "help": "maximum size of request body in bytes (default: 100 Mbyte)",
+            "type": positive_int}),
+        ("max_resource_size", {
+            "value": "10000000",
+            "help": "maximum size of resource (default: 10 Mbyte)",
             "type": positive_int}),
         ("timeout", {
             "value": "30",
@@ -600,7 +604,7 @@ This is an automated message. Please do not reply.""",
         ("profiling_per_request_min_duration", {
             "value": "3",
             "help": "log profiling data per request minimum duration (seconds)",
-            "type": int}),
+            "type": positive_int}),
         ("profiling_per_request_header", {
             "value": "False",
             "help": "Log profiling request body (if passing minimum duration)",
@@ -612,11 +616,11 @@ This is an automated message. Please do not reply.""",
         ("profiling_per_request_method_interval", {
             "value": "600",
             "help": "log profiling data per request method interval (seconds)",
-            "type": int}),
+            "type": positive_int}),
         ("profiling_top_x_functions", {
             "value": "10",
             "help": "log profiling top X functions (limit)",
-            "type": int}),
+            "type": positive_int}),
         ("mask_passwords", {
             "value": "True",
             "help": "mask passwords in logs",

radicale/tests/static/event_multiple3.ics

@@ -0,0 +1,40 @@
+BEGIN:VCALENDAR
+PRODID:-//Mozilla.org/NONSGML Mozilla Calendar V1.1//EN
+VERSION:2.0
+BEGIN:VTIMEZONE
+TZID:Europe/Paris
+X-LIC-LOCATION:Europe/Paris
+BEGIN:DAYLIGHT
+TZOFFSETFROM:+0100
+TZOFFSETTO:+0200
+TZNAME:CEST
+DTSTART:19700329T020000
+RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=3
+END:DAYLIGHT
+BEGIN:STANDARD
+TZOFFSETFROM:+0200
+TZOFFSETTO:+0100
+TZNAME:CET
+DTSTART:19701025T030000
+RRULE:FREQ=YEARLY;BYDAY=-1SU;BYMONTH=10
+END:STANDARD
+END:VTIMEZONE
+BEGIN:VEVENT
+UID:event
+SUMMARY:Event
+DTSTART;TZID=Europe/Paris:20130901T190000
+DTEND;TZID=Europe/Paris:20130901T200000
+END:VEVENT
+BEGIN:VTODO
+UID:todo
+DTSTART;TZID=Europe/Paris:20130901T220000
+DURATION:PT1H
+SUMMARY:Todo
+END:VTODO
+BEGIN:VEVENT
+UID:event2
+SUMMARY:Event-with-longer-description
+DTSTART;TZID=Europe/Paris:20130901T190000
+DTEND;TZID=Europe/Paris:20130901T200000
+END:VEVENT
+END:VCALENDAR

radicale/tests/test_base.py

@@ -142,6 +142,22 @@ permissions: RrWw""")
         assert "Event" in answer
         assert "UID:event" in answer
 
+    def test_add_event_exceed_size(self) -> None:
+        """Add an event which is exceeding max-resource-size."""
+        self.configure({"server": {"max_resource_size": 20}})
+        self.mkcalendar("/calendar.ics/")
+        event = get_file_content("event1.ics")
+        path = "/calendar.ics/event1.ics"
+        self.put(path, event, check=412)
+
+    def test_add_events_exceed_size(self) -> None:
+        """Add multipe events where last is exceeding max-resource-size."""
+        self.configure({"server": {"max_resource_size": 603}})
+        self.mkcalendar("/calendar.ics/")
+        event = get_file_content("event_multiple3.ics")
+        path = "/calendar.ics/"
+        self.put(path, event, check=412)
+
     def test_add_event_broken(self) -> None:
         """Add a broken event."""
         self.mkcalendar("/calendar.ics/")
@@ -676,11 +692,13 @@ permissions: RrWw""")
         assert not isinstance(response, int)
         status, prop = response["D:sync-token"]
         assert status == 200 and prop.text
+        assert "C:max-resource-size" not in response
         _, responses = self.propfind("/calendar.ics/event.ics", propfind)
         response = responses["/calendar.ics/event.ics"]
         assert not isinstance(response, int)
         status, prop = response["D:getetag"]
         assert status == 200 and prop.text
+        assert "C:max-resource-size" not in response
 
     def test_propfind_nonexistent(self) -> None:
         """Read a property that does not exist."""
@@ -692,6 +710,40 @@ permissions: RrWw""")
         status, prop = response["ICAL:calendar-color"]
         assert status == 404 and not prop.text
 
+    def test_propfind_max_resource_size(self) -> None:
+        """Read property C:max-resource-size"""
+        self.mkcalendar("/calendar.ics/")
+        event = get_file_content("event1.ics")
+        self.put("/calendar.ics/event.ics", event)
+        _, responses = self.propfind("/calendar.ics/", """\
+<?xml version="1.0"?>
+ <propfind xmlns="DAV:" xmlns:C="urn:ietf:params:xml:ns:caldav">
+   <prop>
+     <C:max-resource-size />
+   </prop>
+ </propfind>""")
+        response = responses["/calendar.ics/"]
+        assert not isinstance(response, int)
+        status, prop = response["C:max-resource-size"]
+        assert status == 200 and prop.text
+
+    def test_propfind_getctag(self) -> None:
+        """Read property CS:getctag"""
+        self.mkcalendar("/calendar.ics/")
+        event = get_file_content("event1.ics")
+        self.put("/calendar.ics/event.ics", event)
+        _, responses = self.propfind("/calendar.ics/", """\
+<?xml version="1.0"?>
+<propfind xmlns="DAV:" xmlns:CS="http://calendarserver.org/ns/">
+  <prop>
+    <CS:getctag />
+  </prop>
+</propfind>""")
+        response = responses["/calendar.ics/"]
+        assert not isinstance(response, int)
+        status, prop = response["CS:getctag"]
+        assert status == 200 and prop.text
+
     def test_proppatch(self) -> None:
         """Set/Remove a property and read it back."""
         self.mkcalendar("/calendar.ics/")