Coverage for drivers/linstorcowutil.py : 24%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python3
2#
3# Copyright (C) 2020 Vates SAS - ronan.abhamon@vates.fr
4#
5# This program is free software: you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation, either version 3 of the License, or
8# (at your option) any later version.
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program. If not, see <https://www.gnu.org/licenses/>.
17from sm_typing import Any, Callable, Dict, IO, List, Optional, override
19from linstorjournaler import LinstorJournaler
20from linstorvolumemanager import LinstorVolumeManager, LinstorVolumeOpeners
22from concurrent.futures import ThreadPoolExecutor
23import base64
24import contextlib
25import errno
26import json
27import socket
28import threading
29import time
31from cowutil import CowImageInfo, CowUtil, getCowUtil
32import util
33import xs_errors
35from vditype import VdiType
37MANAGER_PLUGIN = 'linstor-manager'
39MULTI_MAX_WORKERS = 4
41def call_remote_method(session, host_ref, method, args):
42 try:
43 response = session.xenapi.host.call_plugin(
44 host_ref, MANAGER_PLUGIN, method, args
45 )
46 except Exception as e:
47 util.SMlog('call-plugin on {} ({} with {}) exception: {}'.format(
48 host_ref, method, args, e
49 ))
50 raise util.SMException(str(e))
52 util.SMlog('call-plugin on {} ({} with {}) returned: {}'.format(
53 host_ref, method, args, response
54 ))
56 return response
59class LinstorCallException(util.SMException):
60 def __init__(self, cmd_err):
61 self.cmd_err = cmd_err
63 @override
64 def __str__(self) -> str:
65 return str(self.cmd_err)
68class ErofsLinstorCallException(LinstorCallException):
69 pass
72class NoDataLinstorCallException(LinstorCallException):
73 pass
76class NoPathLinstorCallException(LinstorCallException):
77 pass
80def log_successful_call(target_host, device_path, vdi_uuid, remote_method, response):
81 util.SMlog('Successful access on {} for device {} ({}): `{}` => {}'.format(
82 target_host, device_path, vdi_uuid, remote_method, str(response)
83 ), priority=util.LOG_DEBUG)
85def log_failed_call(target_host, next_target, device_path, vdi_uuid, remote_method, e):
86 util.SMlog('Failed to call method on {} for device {} ({}): {}. Trying accessing on {}... (cause: {})'.format(
87 target_host, device_path, vdi_uuid, remote_method, next_target, e
88 ), priority=util.LOG_DEBUG)
90def linstorhostcall(local_method, remote_method=None):
91 if not remote_method:
92 remote_method = local_method
94 def decorated(response_parser):
95 def wrapper(*args, **kwargs):
96 self = args[0]
97 vdi_uuid = args[1]
99 device_path = self._linstor.build_device_path(
100 self._linstor.get_volume_name(vdi_uuid)
101 )
103 if not self._session:
104 return self._call_local_method(local_method, device_path, *args[2:], **kwargs)
106 remote_args = {
107 'devicePath': device_path,
108 'groupName': self._linstor.group_name,
109 'vdiType': self._vdi_type
110 }
111 remote_args.update(**kwargs)
112 remote_args = {str(key): str(value) for key, value in remote_args.items()}
114 this_host_ref = util.get_this_host_ref(self._session)
115 def call_method(host_label, host_ref):
116 if host_ref == this_host_ref:
117 return self._call_local_method(local_method, device_path, *args[2:], **kwargs)
118 response = call_remote_method(self._session, host_ref, remote_method, remote_args)
119 log_successful_call(host_label, device_path, vdi_uuid, remote_method, response)
120 return response_parser(self, vdi_uuid, response)
122 # 1. Try on attached host.
123 try:
124 host_ref_attached = next(iter(util.get_hosts_attached_on(self._session, [vdi_uuid])), None)
125 if host_ref_attached:
126 return call_method('attached host', host_ref_attached)
127 except Exception as e:
128 log_failed_call('attached host', 'master', device_path, vdi_uuid, remote_method, e)
130 # 2. Try on master host.
131 try:
132 return call_method('master', util.get_master_ref(self._session))
133 except Exception as e:
134 log_failed_call('master', 'primary', device_path, vdi_uuid, remote_method, e)
136 # 3. Try on a primary.
137 hosts = self._get_hosts(remote_method, device_path)
139 nodes, primary_hostname = self._linstor.find_up_to_date_diskful_nodes(vdi_uuid)
140 if primary_hostname:
141 try:
142 return call_method('primary', self._find_host_ref_from_hostname(hosts, primary_hostname))
143 except Exception as remote_e:
144 self._raise_openers_exception(device_path, remote_e)
146 log_failed_call('primary', 'another node', device_path, vdi_uuid, remote_method, 'no primary')
148 # 4. Try on any host with local data.
149 try:
150 return call_method('another node', next(filter(None,
151 (self._find_host_ref_from_hostname(hosts, hostname) for hostname in nodes)
152 ), None))
153 except Exception as remote_e:
154 self._raise_openers_exception(device_path, remote_e)
156 return wrapper
157 return decorated
160def linstormodifier():
161 def decorated(func):
162 def wrapper(*args, **kwargs):
163 self = args[0]
165 ret = func(*args, **kwargs)
166 self._linstor.invalidate_resource_cache()
167 return ret
168 return wrapper
169 return decorated
172class LinstorCowUtil:
173 class Chain(object):
174 def __init__(self, files: List[IO], leaf_path: str):
175 self._files = files
176 self._leaf_path = leaf_path
178 @property
179 def leaf_path(self) -> str:
180 return self._leaf_path
182 def close(self) -> None:
183 for file in self._files:
184 with contextlib.suppress(Exception):
185 file.close()
187 def __init__(self, session, linstor, vdi_type: str):
188 self._session = session
189 self._linstor = linstor
190 self._cowutil = getCowUtil(vdi_type)
191 self._vdi_type = vdi_type
193 @property
194 def cowutil(self) -> CowUtil:
195 return self._cowutil
197 def create_chain_paths(
198 self,
199 vdi_uuid: str,
200 readonly=False,
201 cb_openers: Optional[Callable[[str, LinstorVolumeOpeners], Any]] = None
202 ) -> Chain:
203 # OPTIMIZE: Add a limit_to_first_allocated_block param to limit cowutil calls.
204 # Useful for the snapshot code algorithm.
206 files: List[IO] = []
208 leaf_path = self._linstor.get_device_path(vdi_uuid)
209 path = leaf_path
210 try:
211 while True:
212 if not util.pathexists(path):
213 raise xs_errors.XenError(
214 'VDIUnavailable', opterr='Could not find: {}'.format(path)
215 )
217 # Diskless path can be created on the fly, ensure we can open it.
218 def check_volume_usable():
219 while True:
220 try:
221 files.append(open(path, 'r' if readonly else 'r+'))
222 except (IOError, OSError) as e:
223 if e.errno == errno.ENODATA:
224 time.sleep(2)
225 continue
226 if e.errno in (errno.EROFS, errno.EMEDIUMTYPE):
227 openers = self._linstor.get_volume_openers(vdi_uuid)
228 util.SMlog(f'Volume not attachable because used. Openers: {openers}')
229 if cb_openers:
230 cb_openers(vdi_uuid, openers)
231 raise
232 break
233 util.retry(check_volume_usable, 15, 2)
235 vdi_uuid = self.get_info(vdi_uuid).parentUuid
236 if not vdi_uuid:
237 break
238 path = self._linstor.get_device_path(vdi_uuid)
239 readonly = True # Non-leaf is always readonly.
240 except Exception as e:
241 self.Chain(files, leaf_path).close()
242 raise e
244 return self.Chain(files, leaf_path)
246 # --------------------------------------------------------------------------
247 # Getters: read locally and try on another host in case of failure.
248 # --------------------------------------------------------------------------
250 def check(self, vdi_uuid, ignore_missing_footer=False, fast=False):
251 kwargs = {
252 'ignoreMissingFooter': ignore_missing_footer,
253 'fast': fast
254 }
255 return self._check(vdi_uuid, **kwargs)
257 @linstorhostcall('check')
258 def _check(self, vdi_uuid, response):
259 return CowUtil.CheckResult(int(response))
261 def get_info(self, vdi_uuid, include_parent=True):
262 kwargs = {
263 'includeParent': include_parent,
264 'resolveParent': False
265 }
267 try:
268 return self._get_info(vdi_uuid, self._extract_uuid, **kwargs)
269 except Exception as e:
270 # Backward compatibility with non-QCOW2 versions.
271 if str(e).startswith("['UNKNOWN_XENAPI_PLUGIN_FUNCTION', 'getInfo']"):
272 return self._get_vhd_info(vdi_uuid, self._extract_uuid, **kwargs)
273 raise
275 @linstorhostcall('getInfo')
276 def _get_info(self, vdi_uuid, response):
277 return self._get_info_impl(vdi_uuid, response)
279 # Backward compatibility with non-QCOW2 versions.
280 @linstorhostcall('getVHDInfo')
281 def _get_vhd_info(self, vdi_uuid, response):
282 return self._get_info_impl(vdi_uuid, response)
284 def _get_info_impl(self, vdi_uuid, response):
285 obj = json.loads(response)
287 image_info = CowImageInfo(vdi_uuid)
288 image_info.sizeVirt = obj['sizeVirt']
289 image_info.sizePhys = obj['sizePhys']
290 if 'parentPath' in obj:
291 image_info.parentPath = obj['parentPath']
292 image_info.parentUuid = obj['parentUuid']
293 image_info.hidden = obj['hidden']
294 image_info.path = obj['path']
296 return image_info
298 @linstorhostcall('hasParent')
299 def has_parent(self, vdi_uuid, response):
300 return util.strtobool(response)
302 def get_parent(self, vdi_uuid):
303 return self._get_parent(vdi_uuid, self._extract_uuid)
305 @linstorhostcall('getParent')
306 def _get_parent(self, vdi_uuid, response):
307 # response will be an empty string for a remote call, it would be `None` for a local
308 # call so we transform it back to `None` here.
309 return response or None
311 @linstorhostcall('getSizeVirt')
312 def get_size_virt(self, vdi_uuid, response):
313 return int(response)
315 @linstorhostcall('getMaxResizeSize')
316 def get_max_resize_size(self, vdi_uuid, response):
317 return int(response)
319 @linstorhostcall('getSizePhys')
320 def get_size_phys(self, vdi_uuid, response):
321 return int(response)
323 @linstorhostcall('getAllocatedSize')
324 def get_allocated_size(self, vdi_uuid, response):
325 return int(response)
327 @linstorhostcall('getDepth')
328 def get_depth(self, vdi_uuid, response):
329 return int(response)
331 @linstorhostcall('getKeyHash')
332 def get_key_hash(self, vdi_uuid, response):
333 return response or None
335 @linstorhostcall('getBlockBitmap')
336 def get_block_bitmap(self, vdi_uuid, response):
337 return base64.b64decode(response)
339 @linstorhostcall('_get_drbd_size', 'getDrbdSize')
340 def get_drbd_size(self, vdi_uuid, response):
341 return int(response)
343 def _get_drbd_size(self, path):
344 (ret, stdout, stderr) = util.doexec(['blockdev', '--getsize64', path])
345 if ret == 0:
346 return int(stdout.strip())
347 raise util.SMException('Failed to get DRBD size: {}'.format(stderr))
349 # --------------------------------------------------------------------------
350 # Setters: only used locally.
351 # --------------------------------------------------------------------------
353 @linstormodifier()
354 def create(self, path, size, static, msize=0):
355 return self._call_local_method_or_fail(self._cowutil.create, path, size, static, msize)
357 @linstormodifier()
358 def set_size_phys(self, path, size, debug=True):
359 return self._call_local_method_or_fail(self._cowutil.setSizePhys, path, size, debug)
361 @linstormodifier()
362 def set_parent(self, path, parentPath, parentRaw=False):
363 return self._call_local_method_or_fail(self._cowutil.setParent, path, parentPath, parentRaw)
365 @linstormodifier()
366 def set_hidden(self, path, hidden=True):
367 return self._call_local_method_or_fail(self._cowutil.setHidden, path, hidden)
369 @linstormodifier()
370 def set_key(self, path, key_hash):
371 return self._call_local_method_or_fail(self._cowutil.setKey, path, key_hash)
373 @linstormodifier()
374 def kill_data(self, path):
375 return self._call_local_method_or_fail(self._cowutil.killData, path)
377 @linstormodifier()
378 def snapshot(self, path, parent, parentRaw, msize=0, checkEmpty=True):
379 return self._call_local_method_or_fail(self._cowutil.snapshot, path, parent, parentRaw, msize, checkEmpty)
381 def inflate(self, journaler, vdi_uuid, vdi_path, new_size, old_size):
382 # Only inflate if the LINSTOR volume capacity is not enough.
383 new_size = LinstorVolumeManager.round_up_volume_size(new_size)
384 if new_size <= old_size:
385 return
387 util.SMlog(
388 'Inflate {} (size={}, previous={})'
389 .format(vdi_path, new_size, old_size)
390 )
392 journaler.create(
393 LinstorJournaler.INFLATE, vdi_uuid, old_size
394 )
395 self._linstor.resize_volume(vdi_uuid, new_size)
397 result_size = self.get_drbd_size(vdi_uuid)
398 if result_size < new_size:
399 util.SMlog(
400 'WARNING: Cannot inflate volume to {}B, result size: {}B'
401 .format(new_size, result_size)
402 )
404 self._zeroize(vdi_path, result_size - self._cowutil.getFooterSize())
405 self.set_size_phys(vdi_path, result_size, False)
406 journaler.remove(LinstorJournaler.INFLATE, vdi_uuid)
408 def deflate(self, vdi_path, new_size, old_size, zeroize=False):
409 if zeroize:
410 assert old_size > self._cowutil.getFooterSize()
411 self._zeroize(vdi_path, old_size - self._cowutil.getFooterSize())
413 new_size = LinstorVolumeManager.round_up_volume_size(new_size)
414 if new_size >= old_size:
415 return
417 util.SMlog(
418 'Deflate {} (new size={}, previous={})'
419 .format(vdi_path, new_size, old_size)
420 )
422 self.set_size_phys(vdi_path, new_size)
423 # TODO: Change the LINSTOR volume size using linstor.resize_volume.
425 # --------------------------------------------------------------------------
426 # Remote setters: write locally and try on another host in case of failure.
427 # --------------------------------------------------------------------------
429 @linstormodifier()
430 def set_size_virt(self, path, size, jFile):
431 kwargs = {
432 'size': size,
433 'jFile': jFile
434 }
435 return self._call_method(self._cowutil.setSizeVirt, 'setSizeVirt', path, use_parent=False, **kwargs)
437 @linstormodifier()
438 def set_size_virt_fast(self, path, size):
439 kwargs = {
440 'size': size
441 }
442 return self._call_method(self._cowutil.setSizeVirtFast, 'setSizeVirtFast', path, use_parent=False, **kwargs)
444 @linstormodifier()
445 def force_parent(self, path, parentPath, parentRaw=False):
446 kwargs = {
447 'parentPath': str(parentPath),
448 'parentRaw': parentRaw
449 }
450 return self._call_method(self._cowutil.setParent, 'setParent', path, use_parent=False, **kwargs)
452 @linstormodifier()
453 def force_coalesce(self, path):
454 return int(self._call_method(self._cowutil.coalesce, 'coalesce', path, use_parent=True))
456 @linstormodifier()
457 def force_repair(self, path):
458 return self._call_method(self._cowutil.repair, 'repair', path, use_parent=False)
460 @linstormodifier()
461 def force_deflate(self, path, newSize, oldSize, zeroize):
462 kwargs = {
463 'newSize': newSize,
464 'oldSize': oldSize,
465 'zeroize': zeroize
466 }
467 return self._call_method('_force_deflate', 'deflate', path, use_parent=False, **kwargs)
469 def _force_deflate(self, path, newSize, oldSize, zeroize):
470 self.deflate(path, newSize, oldSize, zeroize)
472 # --------------------------------------------------------------------------
473 # Helpers.
474 # --------------------------------------------------------------------------
476 def compute_volume_size(self, virtual_size: int) -> int:
477 volume_size = virtual_size
478 if VdiType.isCowImage(self._vdi_type):
479 # All LINSTOR VDIs have the metadata area preallocated for
480 # the maximum possible virtual size (for fast online VDI.resize).
481 meta_overhead = self._cowutil.calcOverheadEmpty(
482 max(virtual_size, self._cowutil.getDefaultPreallocationSizeVirt())
483 )
484 bitmap_overhead = self._cowutil.calcOverheadBitmap(virtual_size)
485 volume_size += meta_overhead + bitmap_overhead
486 elif self._vdi_type != VdiType.RAW:
487 raise Exception('Invalid image type: {}'.format(self._vdi_type))
489 return LinstorVolumeManager.round_up_volume_size(volume_size)
491 def _extract_uuid(self, device_path):
492 # TODO: Remove new line in the vhdutil module. Not here.
493 return self._linstor.get_volume_uuid_from_device_path(
494 device_path.rstrip('\n')
495 )
497 def _get_hosts(self, remote_method, device_path):
498 try:
499 return self._session.xenapi.host.get_all_records()
500 except Exception as e:
501 raise xs_errors.XenError(
502 'VDIUnavailable',
503 opterr='Unable to get host list to run cowutil command `{}` (path={}): {}'
504 .format(remote_method, device_path, e)
505 )
507 # --------------------------------------------------------------------------
509 @staticmethod
510 def _find_host_ref_from_hostname(hosts, hostname):
511 return next((ref for ref, rec in hosts.items() if rec['hostname'] == hostname), None)
513 def _raise_openers_exception(self, device_path, e):
514 if isinstance(e, util.CommandException):
515 e_str = 'cmd: `{}`, code: `{}`, reason: `{}`'.format(e.cmd, e.code, e.reason)
516 else:
517 e_str = str(e)
519 try:
520 volume_uuid = self._linstor.get_volume_uuid_from_device_path(
521 device_path
522 )
523 e_wrapper = Exception(
524 e_str + ' (openers: {})'.format(
525 self._linstor.get_volume_openers(volume_uuid)
526 )
527 )
528 except Exception as illformed_e:
529 e_wrapper = Exception(
530 e_str + ' (unable to get openers: {})'.format(illformed_e)
531 )
532 util.SMlog('raise opener exception: {}'.format(e_wrapper))
533 raise e_wrapper # pylint: disable = E0702
535 def _sanitize_local_method(self, local_method):
536 if isinstance(local_method, str):
537 return getattr(self if local_method.startswith('_') else self._cowutil, local_method)
538 return local_method
540 def _call_local_method(self, local_method, device_path, *args, **kwargs):
541 local_method = self._sanitize_local_method(local_method)
543 try:
544 def local_call():
545 try:
546 return local_method(device_path, *args, **kwargs)
547 except util.CommandException as e:
548 if e.code in (errno.EROFS, errno.EMEDIUMTYPE):
549 raise ErofsLinstorCallException(e)
550 if e.code == errno.ENODATA:
551 raise NoDataLinstorCallException(e)
552 if e.code == errno.ENOENT:
553 raise NoPathLinstorCallException(e)
554 raise e # Continue to retry calls.
555 # Retry only locally if it's not a LinstorCallException.
556 # ENODATA is retried on blktap2 directly, to handle transient DRBD errors.
557 return util.retry(local_call, 5, 2, exceptions=[util.CommandException])
558 except util.CommandException as e:
559 util.SMlog('failed to execute locally CowUtil (sys {})'.format(e.code))
560 raise e
562 def _call_local_method_or_fail(self, local_method, device_path, *args, **kwargs):
563 try:
564 return self._call_local_method(local_method, device_path, *args, **kwargs)
565 except ErofsLinstorCallException as e:
566 # Volume is locked on a host, find openers.
567 self._raise_openers_exception(device_path, e.cmd_err)
569 def _call_method(self, local_method, remote_method, device_path, use_parent, *args, **kwargs):
570 # Note: `use_parent` exists to know if the COW image parent is used by the local/remote method.
571 # Normally in case of failure, if the parent is unused we try to execute the method on
572 # another host using the DRBD opener list. In the other case, if the parent is required,
573 # we must check where this last one is open instead of the child.
575 local_method = self._sanitize_local_method(local_method)
577 # A. Try to write locally...
578 try:
579 return self._call_local_method(local_method, device_path, *args, **kwargs)
580 except Exception:
581 pass
583 util.SMlog('unable to execute `{}` locally, retry using a writable host...'.format(remote_method))
585 # B. Execute the command on another host.
586 # B.1. Get host list.
587 hosts = self._get_hosts(remote_method, device_path)
589 # B.2. Prepare remote args.
590 remote_args = {
591 'devicePath': device_path,
592 'groupName': self._linstor.group_name,
593 'vdiType': self._vdi_type
594 }
595 remote_args.update(**kwargs)
596 remote_args = {str(key): str(value) for key, value in remote_args.items()}
598 volume_uuid = self._linstor.get_volume_uuid_from_device_path(
599 device_path
600 )
601 parent_volume_uuid = None
602 if use_parent:
603 parent_volume_uuid = self.get_parent(volume_uuid)
605 openers_uuid = parent_volume_uuid if use_parent else volume_uuid
607 # B.3. Call!
608 def remote_call():
609 try:
610 openers = self._linstor.get_volume_openers(openers_uuid)
611 except Exception as e:
612 raise xs_errors.XenError(
613 'VDIUnavailable',
614 opterr='Unable to get DRBD openers to run CowUtil command `{}` (path={}): {}'
615 .format(remote_method, device_path, e)
616 )
618 no_host_found = True
619 for hostname, host_openers in openers.items():
620 if not host_openers:
621 continue
623 host_ref = self._find_host_ref_from_hostname(hosts, hostname)
624 if not host_ref:
625 continue
627 no_host_found = False
628 try:
629 return call_remote_method(self._session, host_ref, remote_method, remote_args)
630 except Exception:
631 pass
633 if no_host_found:
634 try:
635 return local_method(device_path, *args, **kwargs)
636 except Exception as e:
637 self._raise_openers_exception(device_path, e)
639 raise xs_errors.XenError(
640 'VDIUnavailable',
641 opterr='No valid host found to run CowUtil command `{}` (path=`{}`, openers=`{}`)'
642 .format(remote_method, device_path, openers)
643 )
644 return util.retry(remote_call, 5, 2)
646 def _zeroize(self, path, size):
647 if not util.zeroOut(path, size, self._cowutil.getFooterSize()):
648 raise xs_errors.XenError(
649 'EIO',
650 opterr='Failed to zero out COW image footer {}'.format(path)
651 )
653class MultiLinstorCowUtil:
654 class ExecutorData(threading.local):
655 def __init__(self):
656 self.clear()
658 def clear(self):
659 self.session = None
660 self.linstor = None
661 self.vdi_type_to_cowutil = {}
663 def __init__(self, uri, group_name) -> None:
664 self._uri = uri
665 self._group_name = group_name
666 self._loads: List[util.ApiSession] = []
667 self._executor_data = self.ExecutorData()
669 def __del__(self):
670 self._cleanup()
672 def run(self, func, user_data_list):
673 def wrapper(func, user_data):
674 if not self._executor_data.session:
675 self._init_executor_thread()
676 return func(user_data, self)
678 with ThreadPoolExecutor(max_workers=MULTI_MAX_WORKERS, thread_name_prefix="CowUtil") as executor:
679 return executor.map(lambda user_data: wrapper(func, user_data), user_data_list)
681 def get_local_cowutil(self, vdi_type):
682 instance = self._executor_data.vdi_type_to_cowutil.get(vdi_type)
683 if not instance:
684 instance = LinstorCowUtil(
685 self._executor_data.session,
686 self._executor_data.linstor,
687 vdi_type
688 )
689 self._executor_data.vdi_type_to_cowutil[vdi_type] = instance
690 return instance
692 def _init_executor_thread(self):
693 api_session = util.ApiSession("SM-linstor-multi-cow-util")
694 try:
695 linstor = LinstorVolumeManager(
696 self._uri,
697 self._group_name,
698 repair=False,
699 logger=util.SMlog
700 )
701 self._executor_data.linstor = linstor
702 self._executor_data.session = api_session.session
703 except:
704 self._executor_data.clear()
705 raise
706 self._loads.append(api_session)
708 def _cleanup(self):
709 for load in self._loads:
710 try:
711 load.logout()
712 except Exception as e:
713 util.SMlog(f"Failed to clean load executor: {e}")
714 self._loads.clear()