storage.py 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841
  1. # This file is part of Radicale Server - Calendar Server
  2. # Copyright © 2014 Jean-Marc Martins
  3. # Copyright © 2012-2016 Guillaume Ayoub
  4. #
  5. # This library is free software: you can redistribute it and/or modify
  6. # it under the terms of the GNU General Public License as published by
  7. # the Free Software Foundation, either version 3 of the License, or
  8. # (at your option) any later version.
  9. #
  10. # This library is distributed in the hope that it will be useful,
  11. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13. # GNU General Public License for more details.
  14. #
  15. # You should have received a copy of the GNU General Public License
  16. # along with Radicale. If not, see <http://www.gnu.org/licenses/>.
  17. """
  18. Storage backends.
  19. This module loads the storage backend, according to the storage configuration.
  20. Default storage uses one folder per collection and one file per collection
  21. entry.
  22. """
  23. import errno
  24. import json
  25. import os
  26. import posixpath
  27. import shlex
  28. import stat
  29. import subprocess
  30. import threading
  31. import time
  32. import datetime
  33. from contextlib import contextmanager
  34. from hashlib import md5
  35. from importlib import import_module
  36. from itertools import groupby
  37. from random import getrandbits
  38. from tempfile import TemporaryDirectory, NamedTemporaryFile
  39. import vobject
  40. if os.name == "nt":
  41. import ctypes
  42. import ctypes.wintypes
  43. import msvcrt
  44. LOCKFILE_EXCLUSIVE_LOCK = 2
  45. if ctypes.sizeof(ctypes.c_void_p) == 4:
  46. ULONG_PTR = ctypes.c_uint32
  47. else:
  48. ULONG_PTR = ctypes.c_uint64
  49. class Overlapped(ctypes.Structure):
  50. _fields_ = [
  51. ("internal", ULONG_PTR),
  52. ("internal_high", ULONG_PTR),
  53. ("offset", ctypes.wintypes.DWORD),
  54. ("offset_high", ctypes.wintypes.DWORD),
  55. ("h_event", ctypes.wintypes.HANDLE)]
  56. lock_file_ex = ctypes.windll.kernel32.LockFileEx
  57. lock_file_ex.argtypes = [
  58. ctypes.wintypes.HANDLE,
  59. ctypes.wintypes.DWORD,
  60. ctypes.wintypes.DWORD,
  61. ctypes.wintypes.DWORD,
  62. ctypes.wintypes.DWORD,
  63. ctypes.POINTER(Overlapped)]
  64. lock_file_ex.restype = ctypes.wintypes.BOOL
  65. unlock_file_ex = ctypes.windll.kernel32.UnlockFileEx
  66. unlock_file_ex.argtypes = [
  67. ctypes.wintypes.HANDLE,
  68. ctypes.wintypes.DWORD,
  69. ctypes.wintypes.DWORD,
  70. ctypes.wintypes.DWORD,
  71. ctypes.POINTER(Overlapped)]
  72. unlock_file_ex.restype = ctypes.wintypes.BOOL
  73. elif os.name == "posix":
  74. import fcntl
  75. def load(configuration, logger):
  76. """Load the storage manager chosen in configuration."""
  77. storage_type = configuration.get("storage", "type")
  78. if storage_type == "multifilesystem":
  79. collection_class = Collection
  80. else:
  81. collection_class = import_module(storage_type).Collection
  82. class CollectionCopy(collection_class):
  83. """Collection copy, avoids overriding the original class attributes."""
  84. CollectionCopy.configuration = configuration
  85. CollectionCopy.logger = logger
  86. return CollectionCopy
  87. def get_etag(text):
  88. """Etag from collection or item."""
  89. etag = md5()
  90. etag.update(text.encode("utf-8"))
  91. return '"%s"' % etag.hexdigest()
  92. def get_uid(item):
  93. """UID value of an item if defined."""
  94. return hasattr(item, "uid") and item.uid.value
  95. def sanitize_path(path):
  96. """Make path absolute with leading slash to prevent access to other data.
  97. Preserve a potential trailing slash.
  98. """
  99. trailing_slash = "/" if path.endswith("/") else ""
  100. path = posixpath.normpath(path)
  101. new_path = "/"
  102. for part in path.split("/"):
  103. if not part or part in (".", ".."):
  104. continue
  105. new_path = posixpath.join(new_path, part)
  106. trailing_slash = "" if new_path.endswith("/") else trailing_slash
  107. return new_path + trailing_slash
  108. def is_safe_path_component(path):
  109. """Check if path is a single component of a path.
  110. Check that the path is safe to join too.
  111. """
  112. return path and "/" not in path and path not in (".", "..")
  113. def is_safe_filesystem_path_component(path):
  114. """Check if path is a single component of a filesystem path.
  115. Check that the path is safe to join too.
  116. """
  117. return (
  118. path and not os.path.splitdrive(path)[0] and
  119. not os.path.split(path)[0] and path not in (os.curdir, os.pardir) and
  120. not path.startswith(".") and not path.endswith("~"))
  121. def path_to_filesystem(root, *paths):
  122. """Convert path to a local filesystem path relative to base_folder.
  123. `root` must be a secure filesystem path, it will be prepend to the path.
  124. Conversion of `paths` is done in a secure manner, or raises ``ValueError``.
  125. """
  126. paths = [sanitize_path(path).strip("/") for path in paths]
  127. safe_path = root
  128. for path in paths:
  129. if not path:
  130. continue
  131. for part in path.split("/"):
  132. if not is_safe_filesystem_path_component(part):
  133. raise UnsafePathError(part)
  134. safe_path = os.path.join(safe_path, part)
  135. return safe_path
  136. class UnsafePathError(ValueError):
  137. def __init__(self, path):
  138. message = "Can't translate name safely to filesystem: %s" % path
  139. super().__init__(message)
  140. class ComponentExistsError(ValueError):
  141. def __init__(self, path):
  142. message = "Component already exists: %s" % path
  143. super().__init__(message)
  144. class ComponentNotFoundError(ValueError):
  145. def __init__(self, path):
  146. message = "Component doesn't exist: %s" % path
  147. super().__init__(message)
  148. class EtagMismatchError(ValueError):
  149. def __init__(self, etag1, etag2):
  150. message = "ETags don't match: %s != %s" % (etag1, etag2)
  151. super().__init__(message)
  152. class Item:
  153. def __init__(self, collection, item, href, last_modified=None):
  154. self.collection = collection
  155. self.item = item
  156. self.href = href
  157. self.last_modified = last_modified
  158. def __getattr__(self, attr):
  159. return getattr(self.item, attr)
  160. @property
  161. def etag(self):
  162. return get_etag(self.serialize())
  163. class BaseCollection:
  164. # Overriden on copy by the "load" function
  165. configuration = None
  166. logger = None
  167. def __init__(self, path, principal=False):
  168. """Initialize the collection.
  169. ``path`` must be the normalized relative path of the collection, using
  170. the slash as the folder delimiter, with no leading nor trailing slash.
  171. """
  172. raise NotImplementedError
  173. @classmethod
  174. def discover(cls, path, depth="0"):
  175. """Discover a list of collections under the given ``path``.
  176. If ``depth`` is "0", only the actual object under ``path`` is
  177. returned.
  178. If ``depth`` is anything but "0", it is considered as "1" and direct
  179. children are included in the result.
  180. The ``path`` is relative.
  181. The root collection "/" must always exist.
  182. """
  183. raise NotImplementedError
  184. @classmethod
  185. def move(cls, item, to_collection, to_href):
  186. """Move an object.
  187. ``item`` is the item to move.
  188. ``to_collection`` is the target collection.
  189. ``to_href`` is the target name in ``to_collection``. An item with the
  190. same name might already exist.
  191. """
  192. if item.collection.path == to_collection.path and item.href == to_href:
  193. return
  194. to_collection.upload(to_href, item.item)
  195. item.collection.delete(item.href)
  196. @property
  197. def etag(self):
  198. return get_etag(self.serialize())
  199. @classmethod
  200. def create_collection(cls, href, collection=None, props=None):
  201. """Create a collection.
  202. If the collection already exists and neither ``collection`` nor
  203. ``props`` are set, this method shouldn't do anything. Otherwise the
  204. existing collection must be replaced.
  205. ``collection`` is a list of vobject components.
  206. ``props`` are metadata values for the collection.
  207. ``props["tag"]`` is the type of collection (VCALENDAR or
  208. VADDRESSBOOK). If the key ``tag`` is missing, it is guessed from the
  209. collection.
  210. """
  211. raise NotImplementedError
  212. def list(self):
  213. """List collection items."""
  214. raise NotImplementedError
  215. def get(self, href):
  216. """Fetch a single item."""
  217. raise NotImplementedError
  218. def get_multi(self, hrefs):
  219. """Fetch multiple items. Duplicate hrefs must be ignored.
  220. Functionally similar to ``get``, but might bring performance benefits
  221. on some storages when used cleverly.
  222. """
  223. for href in set(hrefs):
  224. yield self.get(href)
  225. def pre_filtered_list(self, filters):
  226. """List collection items with optional pre filtering.
  227. This could largely improve performance of reports depending on
  228. the filters and this implementation.
  229. This returns all event by default
  230. """
  231. return [self.get(href) for href in self.list()]
  232. def has(self, href):
  233. """Check if an item exists by its href.
  234. Functionally similar to ``get``, but might bring performance benefits
  235. on some storages when used cleverly.
  236. """
  237. return self.get(href) is not None
  238. def upload(self, href, vobject_item):
  239. """Upload a new or replace an existing item."""
  240. raise NotImplementedError
  241. def delete(self, href=None):
  242. """Delete an item.
  243. When ``href`` is ``None``, delete the collection.
  244. """
  245. raise NotImplementedError
  246. def get_meta(self, key):
  247. """Get metadata value for collection."""
  248. raise NotImplementedError
  249. def set_meta(self, props):
  250. """Set metadata values for collection."""
  251. raise NotImplementedError
  252. @property
  253. def last_modified(self):
  254. """Get the HTTP-datetime of when the collection was modified."""
  255. raise NotImplementedError
  256. def serialize(self):
  257. """Get the unicode string representing the whole collection."""
  258. raise NotImplementedError
  259. @classmethod
  260. @contextmanager
  261. def acquire_lock(cls, mode, user=None):
  262. """Set a context manager to lock the whole storage.
  263. ``mode`` must either be "r" for shared access or "w" for exclusive
  264. access.
  265. ``user`` is the name of the logged in user or empty.
  266. """
  267. raise NotImplementedError
  268. class Collection(BaseCollection):
  269. """Collection stored in several files per calendar."""
  270. def __init__(self, path, principal=False, folder=None):
  271. if not folder:
  272. folder = self._get_collection_root_folder()
  273. # Path should already be sanitized
  274. self.path = sanitize_path(path).strip("/")
  275. self.encoding = self.configuration.get("encoding", "stock")
  276. self._filesystem_path = path_to_filesystem(folder, self.path)
  277. self._props_path = os.path.join(
  278. self._filesystem_path, ".Radicale.props")
  279. split_path = self.path.split("/")
  280. self.owner = split_path[0] if len(split_path) > 1 else None
  281. self.is_principal = principal
  282. @classmethod
  283. def _get_collection_root_folder(cls):
  284. filesystem_folder = os.path.expanduser(
  285. cls.configuration.get("storage", "filesystem_folder"))
  286. return os.path.join(filesystem_folder, "collection-root")
  287. @contextmanager
  288. def _atomic_write(self, path, mode="w", newline=None):
  289. directory = os.path.dirname(path)
  290. tmp = NamedTemporaryFile(
  291. mode=mode, dir=directory, encoding=self.encoding,
  292. delete=False, prefix=".Radicale.tmp-", newline=newline)
  293. try:
  294. yield tmp
  295. if self.configuration.getboolean("storage", "fsync"):
  296. if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
  297. fcntl.fcntl(tmp.fileno(), fcntl.F_FULLFSYNC)
  298. else:
  299. os.fsync(tmp.fileno())
  300. tmp.close()
  301. os.replace(tmp.name, path)
  302. except:
  303. tmp.close()
  304. os.remove(tmp.name)
  305. raise
  306. self._sync_directory(directory)
  307. @staticmethod
  308. def _find_available_file_name(exists_fn):
  309. # Prevent infinite loop
  310. for _ in range(10000):
  311. file_name = hex(getrandbits(32))[2:]
  312. if not exists_fn(file_name):
  313. return file_name
  314. raise FileExistsError(errno.EEXIST, "No usable file name found")
  315. @classmethod
  316. def _sync_directory(cls, path):
  317. """Sync directory to disk.
  318. This only works on POSIX and does nothing on other systems.
  319. """
  320. if not cls.configuration.getboolean("storage", "fsync"):
  321. return
  322. if os.name == "posix":
  323. fd = os.open(path, 0)
  324. try:
  325. if hasattr(fcntl, "F_FULLFSYNC"):
  326. fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
  327. else:
  328. os.fsync(fd)
  329. finally:
  330. os.close(fd)
  331. @classmethod
  332. def _makedirs_synced(cls, filesystem_path):
  333. """Recursively create a directory and its parents in a sync'ed way.
  334. This method acts silently when the folder already exists.
  335. """
  336. if os.path.isdir(filesystem_path):
  337. return
  338. parent_filesystem_path = os.path.dirname(filesystem_path)
  339. # Prevent infinite loop
  340. if filesystem_path != parent_filesystem_path:
  341. # Create parent dirs recursively
  342. cls._makedirs_synced(parent_filesystem_path)
  343. # Possible race!
  344. os.makedirs(filesystem_path, exist_ok=True)
  345. cls._sync_directory(parent_filesystem_path)
  346. @classmethod
  347. def discover(cls, path, depth="0"):
  348. if path is None:
  349. # Wrong URL
  350. return
  351. # Path should already be sanitized
  352. sane_path = sanitize_path(path).strip("/")
  353. attributes = sane_path.split("/")
  354. if not attributes[0]:
  355. attributes.pop()
  356. folder = cls._get_collection_root_folder()
  357. # Create the root collection
  358. cls._makedirs_synced(folder)
  359. try:
  360. filesystem_path = path_to_filesystem(folder, sane_path)
  361. except ValueError:
  362. # Path is unsafe
  363. return
  364. # Check if the path exists and if it leads to a collection or an item
  365. if not os.path.isdir(filesystem_path):
  366. if attributes and os.path.isfile(filesystem_path):
  367. href = attributes.pop()
  368. else:
  369. return
  370. else:
  371. href = None
  372. path = "/".join(attributes)
  373. principal = len(attributes) == 1
  374. collection = cls(path, principal)
  375. if href:
  376. yield collection.get(href)
  377. return
  378. yield collection
  379. if depth == "0":
  380. return
  381. for item in collection.list():
  382. yield collection.get(item)
  383. for href in os.listdir(filesystem_path):
  384. if not is_safe_filesystem_path_component(href):
  385. if not href.startswith(".Radicale"):
  386. cls.logger.debug("Skipping collection: %s", href)
  387. continue
  388. child_filesystem_path = path_to_filesystem(filesystem_path, href)
  389. if os.path.isdir(child_filesystem_path):
  390. child_path = posixpath.join(path, href)
  391. child_principal = len(attributes) == 0
  392. yield cls(child_path, child_principal)
  393. @classmethod
  394. def create_collection(cls, href, collection=None, props=None):
  395. folder = cls._get_collection_root_folder()
  396. # Path should already be sanitized
  397. sane_path = sanitize_path(href).strip("/")
  398. attributes = sane_path.split("/")
  399. if not attributes[0]:
  400. attributes.pop()
  401. principal = len(attributes) == 1
  402. filesystem_path = path_to_filesystem(folder, sane_path)
  403. if not props:
  404. props = {}
  405. if not props.get("tag") and collection:
  406. props["tag"] = collection[0].name
  407. if not props:
  408. cls._makedirs_synced(filesystem_path)
  409. return cls(sane_path, principal=principal)
  410. parent_dir = os.path.dirname(filesystem_path)
  411. cls._makedirs_synced(parent_dir)
  412. # Create a temporary directory with an unsafe name
  413. with TemporaryDirectory(
  414. prefix=".Radicale.tmp-", dir=parent_dir) as tmp_dir:
  415. # The temporary directory itself can't be renamed
  416. tmp_filesystem_path = os.path.join(tmp_dir, "collection")
  417. os.makedirs(tmp_filesystem_path)
  418. self = cls("/", principal=principal, folder=tmp_filesystem_path)
  419. self.set_meta(props)
  420. if collection:
  421. if props.get("tag") == "VCALENDAR":
  422. collection, = collection
  423. items = []
  424. for content in ("vevent", "vtodo", "vjournal"):
  425. items.extend(
  426. getattr(collection, "%s_list" % content, []))
  427. items_by_uid = groupby(sorted(items, key=get_uid), get_uid)
  428. vobject_items = {}
  429. for uid, items in items_by_uid:
  430. new_collection = vobject.iCalendar()
  431. for item in items:
  432. new_collection.add(item)
  433. href = self._find_available_file_name(vobject_items.get)
  434. vobject_items[href] = new_collection
  435. self.upload_all_nonatomic(vobject_items)
  436. elif props.get("tag") == "VCARD":
  437. vobject_items = {}
  438. for card in collection:
  439. href = self._find_available_file_name(vobject_items.get)
  440. vobject_items[href] = card
  441. self.upload_all_nonatomic(vobject_items)
  442. # This operation is not atomic on the filesystem level but it's
  443. # very unlikely that one rename operations succeeds while the
  444. # other fails or that only one gets written to disk.
  445. if os.path.exists(filesystem_path):
  446. os.rename(filesystem_path, os.path.join(tmp_dir, "delete"))
  447. os.rename(tmp_filesystem_path, filesystem_path)
  448. cls._sync_directory(parent_dir)
  449. return cls(sane_path, principal=principal)
  450. def upload_all_nonatomic(self, vobject_items):
  451. """Upload a new set of items.
  452. This takes a mapping of href and vobject items and
  453. uploads them nonatomic and without existence checks.
  454. """
  455. fs = []
  456. for href, item in vobject_items.items():
  457. path = path_to_filesystem(self._filesystem_path, href)
  458. fs.append(open(path, "w", encoding=self.encoding, newline=""))
  459. fs[-1].write(item.serialize())
  460. fsync_fn = lambda fd: None
  461. if self.configuration.getboolean("storage", "fsync"):
  462. if os.name == "posix" and hasattr(fcntl, "F_FULLFSYNC"):
  463. fsync_fn = lambda fd: fcntl.fcntl(fd, fcntl.F_FULLFSYNC)
  464. else:
  465. fsync_fn = os.fsync
  466. # sync everything at once because it's slightly faster.
  467. for f in fs:
  468. fsync_fn(f.fileno())
  469. f.close()
  470. self._sync_directory(self._filesystem_path)
  471. @classmethod
  472. def move(cls, item, to_collection, to_href):
  473. os.replace(
  474. path_to_filesystem(item.collection._filesystem_path, item.href),
  475. path_to_filesystem(to_collection._filesystem_path, to_href))
  476. cls._sync_directory(to_collection._filesystem_path)
  477. if item.collection._filesystem_path != to_collection._filesystem_path:
  478. cls._sync_directory(item.collection._filesystem_path)
  479. def list(self):
  480. try:
  481. hrefs = os.listdir(self._filesystem_path)
  482. except IOError:
  483. return
  484. for href in hrefs:
  485. if not is_safe_filesystem_path_component(href):
  486. if not href.startswith(".Radicale"):
  487. self.logger.debug("Skipping component: %s", href)
  488. continue
  489. path = os.path.join(self._filesystem_path, href)
  490. if os.path.isfile(path):
  491. yield href
  492. def get(self, href):
  493. if not href:
  494. return None
  495. href = href.strip("{}").replace("/", "_")
  496. if not is_safe_filesystem_path_component(href):
  497. self.logger.debug(
  498. "Can't translate name safely to filesystem: %s", href)
  499. return None
  500. path = path_to_filesystem(self._filesystem_path, href)
  501. if not os.path.isfile(path):
  502. return None
  503. with open(path, encoding=self.encoding, newline="") as fd:
  504. text = fd.read()
  505. last_modified = time.strftime(
  506. "%a, %d %b %Y %H:%M:%S GMT",
  507. time.gmtime(os.path.getmtime(path)))
  508. try:
  509. item = Item(self, vobject.readOne(text), href, last_modified)
  510. except:
  511. self.logger.exception("Object broken (skip 'get'): %s", path)
  512. return None;
  513. return item;
  514. def has(self, href):
  515. return self.get(href) is not None
  516. def upload(self, href, vobject_item):
  517. if not is_safe_filesystem_path_component(href):
  518. raise UnsafePathError(href)
  519. path = path_to_filesystem(self._filesystem_path, href)
  520. item = Item(self, vobject_item, href)
  521. with self._atomic_write(path, newline="") as fd:
  522. fd.write(item.serialize())
  523. return item
  524. def delete(self, href=None):
  525. if href is None:
  526. # Delete the collection
  527. if os.path.isdir(self._filesystem_path):
  528. parent_dir = os.path.dirname(self._filesystem_path)
  529. try:
  530. os.rmdir(self._filesystem_path)
  531. except OSError:
  532. with TemporaryDirectory(
  533. prefix=".Radicale.tmp-", dir=parent_dir) as tmp:
  534. os.rename(self._filesystem_path, os.path.join(
  535. tmp, os.path.basename(self._filesystem_path)))
  536. self._sync_directory(parent_dir)
  537. else:
  538. self._sync_directory(parent_dir)
  539. else:
  540. # Delete an item
  541. if not is_safe_filesystem_path_component(href):
  542. raise UnsafePathError(href)
  543. path = path_to_filesystem(self._filesystem_path, href)
  544. if not os.path.isfile(path):
  545. raise ComponentNotFoundError(href)
  546. os.remove(path)
  547. self._sync_directory(os.path.dirname(path))
  548. def get_meta(self, key):
  549. if os.path.exists(self._props_path):
  550. with open(self._props_path, encoding=self.encoding) as prop:
  551. return json.load(prop).get(key)
  552. def set_meta(self, props):
  553. if os.path.exists(self._props_path):
  554. with open(self._props_path, encoding=self.encoding) as prop:
  555. old_props = json.load(prop)
  556. old_props.update(props)
  557. props = old_props
  558. props = {key: value for key, value in props.items() if value}
  559. with self._atomic_write(self._props_path, "w+") as prop:
  560. json.dump(props, prop)
  561. @property
  562. def last_modified(self):
  563. last = max([os.path.getmtime(self._filesystem_path)] + [
  564. os.path.getmtime(os.path.join(self._filesystem_path, filename))
  565. for filename in os.listdir(self._filesystem_path)] or [0])
  566. return time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime(last))
  567. def serialize(self):
  568. if not os.path.exists(self._filesystem_path):
  569. return None
  570. items = []
  571. time_begin = datetime.datetime.now()
  572. for href in os.listdir(self._filesystem_path):
  573. if not is_safe_filesystem_path_component(href):
  574. self.logger.debug("Skipping component: %s", href)
  575. continue
  576. path = os.path.join(self._filesystem_path, href)
  577. if os.path.isfile(path):
  578. self.logger.debug("Read object: %s", path)
  579. with open(path, encoding=self.encoding, newline="") as fd:
  580. try:
  581. # check whether vobject liks the item
  582. item = vobject.readOne(fd.read())
  583. except:
  584. self.logger.exception("Object broken (skip 'list'): %s", path)
  585. continue
  586. if self.get_meta("tag") == "VADDRESSBOOK":
  587. try:
  588. # check whether vobject liks the VCARD item
  589. item.serialize()
  590. except:
  591. self.logger.exception("Object broken (skip 'read'): %s", path)
  592. self.logger.error("Broken VCARD content: %s", item)
  593. continue
  594. items.append(item)
  595. time_end = datetime.datetime.now()
  596. self.logger.info("Collection read %d items in %s sec from %s", len(items),(time_end - time_begin).total_seconds(), self._filesystem_path)
  597. if self.get_meta("tag") == "VCALENDAR":
  598. collection = vobject.iCalendar()
  599. for item in items:
  600. for content in ("vevent", "vtodo", "vjournal"):
  601. if content in item.contents:
  602. for item_part in getattr(item, "%s_list" % content):
  603. collection.add(item_part)
  604. break
  605. return collection.serialize()
  606. elif self.get_meta("tag") == "VADDRESSBOOK":
  607. return "".join([item.serialize() for item in items])
  608. return ""
  609. _lock = threading.Lock()
  610. _waiters = []
  611. _lock_file = None
  612. _lock_file_locked = False
  613. _readers = 0
  614. _writer = False
  615. @classmethod
  616. @contextmanager
  617. def acquire_lock(cls, mode, user=None):
  618. def condition():
  619. if mode == "r":
  620. return not cls._writer
  621. else:
  622. return not cls._writer and cls._readers == 0
  623. folder = os.path.expanduser(cls.configuration.get(
  624. "storage", "filesystem_folder"))
  625. # Use a primitive lock which only works within one process as a
  626. # precondition for inter-process file-based locking
  627. with cls._lock:
  628. if cls._waiters or not condition():
  629. # Use FIFO for access requests
  630. waiter = threading.Condition(lock=cls._lock)
  631. cls._waiters.append(waiter)
  632. while True:
  633. waiter.wait()
  634. if condition():
  635. break
  636. cls._waiters.pop(0)
  637. if mode == "r":
  638. cls._readers += 1
  639. # Notify additional potential readers
  640. if cls._waiters:
  641. cls._waiters[0].notify()
  642. else:
  643. cls._writer = True
  644. if not cls._lock_file:
  645. cls._makedirs_synced(folder)
  646. lock_path = os.path.join(folder, ".Radicale.lock")
  647. cls._lock_file = open(lock_path, "w+")
  648. # Set access rights to a necessary minimum to prevent locking
  649. # by arbitrary users
  650. try:
  651. os.chmod(lock_path, stat.S_IWUSR | stat.S_IRUSR)
  652. except OSError:
  653. cls.logger.debug("Failed to set permissions on lock file")
  654. if not cls._lock_file_locked:
  655. if os.name == "nt":
  656. handle = msvcrt.get_osfhandle(cls._lock_file.fileno())
  657. flags = LOCKFILE_EXCLUSIVE_LOCK if mode == "w" else 0
  658. overlapped = Overlapped()
  659. if not lock_file_ex(handle, flags, 0, 1, 0, overlapped):
  660. cls.logger.debug("Locking not supported")
  661. elif os.name == "posix":
  662. _cmd = fcntl.LOCK_EX if mode == "w" else fcntl.LOCK_SH
  663. try:
  664. fcntl.flock(cls._lock_file.fileno(), _cmd)
  665. except OSError:
  666. cls.logger.debug("Locking not supported")
  667. cls._lock_file_locked = True
  668. try:
  669. yield
  670. # execute hook
  671. hook = cls.configuration.get("storage", "hook")
  672. if mode == "w" and hook:
  673. cls.logger.debug("Running hook")
  674. subprocess.check_call(
  675. hook % {"user": shlex.quote(user or "Anonymous")},
  676. shell=True, cwd=folder)
  677. finally:
  678. with cls._lock:
  679. if mode == "r":
  680. cls._readers -= 1
  681. else:
  682. cls._writer = False
  683. if cls._readers == 0:
  684. if os.name == "nt":
  685. handle = msvcrt.get_osfhandle(cls._lock_file.fileno())
  686. overlapped = Overlapped()
  687. if not unlock_file_ex(handle, 0, 1, 0, overlapped):
  688. cls.logger.debug("Unlocking not supported")
  689. elif os.name == "posix":
  690. try:
  691. fcntl.flock(cls._lock_file.fileno(), fcntl.LOCK_UN)
  692. except OSError:
  693. cls.logger.debug("Unlocking not supported")
  694. cls._lock_file_locked = False
  695. if cls._waiters:
  696. cls._waiters[0].notify()
  697. if (cls.configuration.getboolean("storage", "close_lock_file")
  698. and cls._readers == 0 and not cls._waiters):
  699. cls._lock_file.close()
  700. cls._lock_file = None