1# Copyright (C) Citrix Systems Inc. 

2# 

3# This program is free software; you can redistribute it and/or modify 

4# it under the terms of the GNU Lesser General Public License as published 

5# by the Free Software Foundation; version 2.1 only. 

6# 

7# This program is distributed in the hope that it will be useful, 

8# but WITHOUT ANY WARRANTY; without even the implied warranty of 

9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

10# GNU Lesser General Public License for more details. 

11# 

12# You should have received a copy of the GNU Lesser General Public License 

13# along with this program; if not, write to the Free Software Foundation, Inc., 

14# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 

15# 

16# Miscellaneous utility functions 

17# 

18 

19from sm_typing import Any, List, Optional, override 

20 

21import contextlib 

22import atexit 

23import os 

24import re 

25import sys 

26import subprocess 

27import shutil 

28import tempfile 

29import signal 

30import time 

31import types 

32import datetime 

33import errno 

34import functools 

35import socket 

36import threading 

37import xml.dom.minidom 

38import scsiutil 

39import stat 

40import xs_errors 

41import XenAPI # pylint: disable=import-error 

42import xmlrpc.client 

43import http.client 

44import base64 

45import syslog 

46import resource 

47import traceback 

48import glob 

49import copy 

50import contextlib 

51from sm_typing import override 

52 

53from functools import reduce 

54from sm_typing import List, Optional 

55 

56NO_LOGGING_STAMPFILE = '/etc/xensource/no_sm_log' 

57 

58IORETRY_MAX = 20 # retries 

59IORETRY_PERIOD = 1.0 # seconds 

60 

61LOGGING = not (os.path.exists(NO_LOGGING_STAMPFILE)) 

62_SM_SYSLOG_FACILITY = syslog.LOG_LOCAL2 

63LOG_EMERG = syslog.LOG_EMERG 

64LOG_ALERT = syslog.LOG_ALERT 

65LOG_CRIT = syslog.LOG_CRIT 

66LOG_ERR = syslog.LOG_ERR 

67LOG_WARNING = syslog.LOG_WARNING 

68LOG_NOTICE = syslog.LOG_NOTICE 

69LOG_INFO = syslog.LOG_INFO 

70LOG_DEBUG = syslog.LOG_DEBUG 

71 

72ISCSI_REFDIR = '/var/run/sr-ref' 

73 

74CMD_DD = "/bin/dd" 

75CMD_KICKPIPE = '/opt/xensource/libexec/kickpipe' 

76 

77FIST_PAUSE_PERIOD = 30 # seconds 

78 

79class _TimeoutContextManager(contextlib.AbstractContextManager): 

80 def __init__(self, delay: int) -> None: 

81 self.delay = delay 

82 self.old_handler: Any = None 

83 

84 @override 

85 def __enter__(self) -> "_TimeoutContextManager": 

86 def handle_timeout(_signum: int, _frame: Optional[types.FrameType]) -> None: 

87 raise TimeoutError(f"Timed out after {self.delay} seconds") 

88 self.old_handler = signal.signal(signal.SIGALRM, handle_timeout) 

89 signal.alarm(self.delay) 

90 return self 

91 

92 @override 

93 def __exit__( 

94 self, 

95 exc_type: Optional[Any], 

96 exc_val: Optional[BaseException], 

97 exc_tb: Optional[types.TracebackType] 

98 ): 

99 signal.alarm(0) 

100 if self.old_handler is not None: 

101 signal.signal(signal.SIGALRM, self.old_handler) 

102 self.old_handler = None 

103 return False 

104 

105 def __call__(self, func): 

106 def wrapper(*args, **kwargs): 

107 with self: 

108 return func(*args, **kwargs) 

109 return wrapper 

110 

111def timeout(*args, **kwargs): 

112 if len(args) >= 2 and callable(args[1]): 

113 delay, func = args[0], args[1] 

114 remaining_args = args[2:] 

115 with _TimeoutContextManager(delay): 

116 return func(*remaining_args, **kwargs) 

117 

118 if len(args) == 1 and callable(args[0]): 

119 raise TypeError("timeout() missing 1 required argument: 'delay'") 

120 

121 if args: 

122 delay = args[0] 

123 elif "delay" in kwargs: 

124 delay = kwargs["delay"] 

125 else: 

126 raise TypeError("timeout() missing 1 required argument: 'delay'") 

127 return _TimeoutContextManager(delay) 

128 

129class SMException(Exception): 

130 """Base class for all SM exceptions for easier catching & wrapping in 

131 XenError""" 

132 

133 

134class CommandException(SMException): 

135 def error_message(self, code): 

136 if code > 0: 

137 return os.strerror(code) 

138 elif code < 0: 

139 return "Signalled %s" % (abs(code)) 

140 return "Success" 

141 

142 def __init__(self, code, cmd="", reason='exec failed'): 

143 self.code = code 

144 self.cmd = cmd 

145 self.reason = reason 

146 Exception.__init__(self, self.error_message(code)) 

147 

148 

149class SRBusyException(SMException): 

150 """The SR could not be locked""" 

151 pass 

152 

153 

154def logException(tag): 

155 info = sys.exc_info() 

156 if info[0] == SystemExit: 156 ↛ 158line 156 didn't jump to line 158, because the condition on line 156 was never true

157 # this should not be happening when catching "Exception", but it is 

158 sys.exit(0) 

159 tb = reduce(lambda a, b: "%s%s" % (a, b), traceback.format_tb(info[2])) 

160 str = "***** %s: EXCEPTION %s, %s\n%s" % (tag, info[0], info[1], tb) 

161 SMlog(str) 

162 

163 

164def roundup(divisor, value): 

165 """Retruns the rounded up value so it is divisible by divisor.""" 

166 

167 if value == 0: 167 ↛ 168line 167 didn't jump to line 168, because the condition on line 167 was never true

168 value = 1 

169 if value % divisor != 0: 

170 return ((int(value) // divisor) + 1) * divisor 

171 return value 

172 

173 

174def to_plain_string(obj): 

175 if obj is None: 

176 return None 

177 if isinstance(obj, dict) and len(obj) == 0: 

178 SMlog(f"util.to_plain_string() corrected empty dict to empty str") 

179 return "" 

180 return str(obj) 

181 

182 

183def shellquote(arg): 

184 return '"%s"' % arg.replace('"', '\\"') 

185 

186 

187def make_WWN(name): 

188 hex_prefix = name.find("0x") 

189 if (hex_prefix >= 0): 189 ↛ 192line 189 didn't jump to line 192, because the condition on line 189 was never false

190 name = name[name.find("0x") + 2:len(name)] 

191 # inject dashes for each nibble 

192 if (len(name) == 16): # sanity check 192 ↛ 196line 192 didn't jump to line 196, because the condition on line 192 was never false

193 name = name[0:2] + "-" + name[2:4] + "-" + name[4:6] + "-" + \ 

194 name[6:8] + "-" + name[8:10] + "-" + name[10:12] + "-" + \ 

195 name[12:14] + "-" + name[14:16] 

196 return name 

197 

198 

199def synchronized(func): 

200 lock = threading.RLock() 

201 

202 @functools.wraps(func) 

203 def wrapper(*args, **kwargs): 

204 with lock: 

205 return func(*args, **kwargs) 

206 

207 return wrapper 

208 

209 

210@synchronized 

211def _writeToSyslog(ident, facility, priority, message): 

212 syslog.openlog(ident, 0, facility) 

213 syslog.syslog(priority, message) 

214 syslog.closelog() 

215 

216 

217def _logToSyslog(ident, facility, priority, message): 

218 pid = os.getpid() 

219 thread_name = threading.current_thread().name 

220 _writeToSyslog(ident, facility, priority, f"[{pid}][{thread_name}] {message}") 

221 

222 

223def SMlog(message, ident="SM", priority=LOG_INFO): 

224 if LOGGING: 224 ↛ exitline 224 didn't return from function 'SMlog', because the condition on line 224 was never false

225 for message_line in str(message).split('\n'): 

226 _logToSyslog(ident, _SM_SYSLOG_FACILITY, priority, message_line) 

227 

228 

229class LoggerCounter: 

230 def __init__(self, max_repeats): 

231 self.previous_message = None 

232 self.max_repeats = max_repeats 

233 self.repeat_counter = 0 

234 

235 def log(self, message): 

236 self.repeat_counter += 1 

237 if self.previous_message != message or self.repeat_counter == self.max_repeats: 

238 SMlog(message) 

239 self.previous_message = message 

240 self.repeat_counter = 0 

241 

242def _getDateString(): 

243 d = datetime.datetime.now() 

244 t = d.timetuple() 

245 return "%s-%s-%s:%s:%s:%s" % \ 

246 (t[0], t[1], t[2], t[3], t[4], t[5]) 

247 

248 

249def doexec(args, inputtext=None, new_env=None, text=True): 

250 """Execute a subprocess, then return its return code, stdout and stderr""" 

251 env = None 

252 if new_env: 

253 env = dict(os.environ) 

254 env.update(new_env) 

255 proc = subprocess.Popen(args, stdin=subprocess.PIPE, 

256 stdout=subprocess.PIPE, 

257 stderr=subprocess.PIPE, 

258 close_fds=True, env=env, 

259 universal_newlines=text) 

260 

261 if not text and inputtext is not None: 261 ↛ 262line 261 didn't jump to line 262, because the condition on line 261 was never true

262 inputtext = inputtext.encode() 

263 

264 (stdout, stderr) = proc.communicate(inputtext) 

265 

266 rc = proc.returncode 

267 return rc, stdout, stderr 

268 

269 

270def is_string(value): 

271 return isinstance(value, str) 

272 

273 

274# These are partially tested functions that replicate the behaviour of 

275# the original pread,pread2 and pread3 functions. Potentially these can 

276# replace the original ones at some later date. 

277# 

278# cmdlist is a list of either single strings or pairs of strings. For 

279# each pair, the first component is passed to exec while the second is 

280# written to the logs. 

281def pread(cmdlist, close_stdin=False, scramble=None, expect_rc=0, 

282 quiet=False, new_env=None, text=True): 

283 cmdlist_for_exec = [] 

284 cmdlist_for_log = [] 

285 for item in cmdlist: 

286 if is_string(item): 286 ↛ 296line 286 didn't jump to line 296, because the condition on line 286 was never false

287 cmdlist_for_exec.append(item) 

288 if scramble: 288 ↛ 289line 288 didn't jump to line 289, because the condition on line 288 was never true

289 if item.find(scramble) != -1: 

290 cmdlist_for_log.append("<filtered out>") 

291 else: 

292 cmdlist_for_log.append(item) 

293 else: 

294 cmdlist_for_log.append(item) 

295 else: 

296 cmdlist_for_exec.append(item[0]) 

297 cmdlist_for_log.append(item[1]) 

298 

299 if not quiet: 299 ↛ 301line 299 didn't jump to line 301, because the condition on line 299 was never false

300 SMlog(cmdlist_for_log) 

301 (rc, stdout, stderr) = doexec(cmdlist_for_exec, new_env=new_env, text=text) 

302 if rc != expect_rc: 

303 SMlog("FAILED in util.pread: (rc %d) stdout: '%s', stderr: '%s'" % \ 

304 (rc, stdout, stderr)) 

305 if quiet: 305 ↛ 306line 305 didn't jump to line 306, because the condition on line 305 was never true

306 SMlog("Command was: %s" % cmdlist_for_log) 

307 if '' == stderr: 307 ↛ 308line 307 didn't jump to line 308, because the condition on line 307 was never true

308 stderr = stdout 

309 raise CommandException(rc, str(cmdlist), stderr.strip()) 

310 if not quiet: 310 ↛ 312line 310 didn't jump to line 312, because the condition on line 310 was never false

311 SMlog(" pread SUCCESS") 

312 return stdout 

313 

314 

315# POSIX guaranteed atomic within the same file system. 

316# Supply directory to ensure tempfile is created 

317# in the same directory. 

318def atomicFileWrite(targetFile, directory, text): 

319 

320 file = None 

321 try: 

322 # Create file only current pid can write/read to 

323 # our responsibility to clean it up. 

324 _, tempPath = tempfile.mkstemp(dir=directory) 

325 file = open(tempPath, 'w') 

326 file.write(text) 

327 

328 # Ensure flushed to disk. 

329 file.flush() 

330 os.fsync(file.fileno()) 

331 file.close() 

332 

333 os.rename(tempPath, targetFile) 

334 except OSError: 

335 SMlog("FAILED to atomic write to %s" % (targetFile)) 

336 

337 finally: 

338 if (file is not None) and (not file.closed): 

339 file.close() 

340 

341 if os.path.isfile(tempPath): 

342 os.remove(tempPath) 

343 

344 

345#Read STDOUT from cmdlist and discard STDERR output 

346def pread2(cmdlist, quiet=False, text=True): 

347 return pread(cmdlist, quiet=quiet, text=text) 

348 

349 

350#Read STDOUT from cmdlist, feeding 'text' to STDIN 

351def pread3(cmdlist, text): 

352 SMlog(cmdlist) 

353 (rc, stdout, stderr) = doexec(cmdlist, text) 

354 if rc: 

355 SMlog("FAILED in util.pread3: (errno %d) stdout: '%s', stderr: '%s'" % \ 

356 (rc, stdout, stderr)) 

357 if '' == stderr: 

358 stderr = stdout 

359 raise CommandException(rc, str(cmdlist), stderr.strip()) 

360 SMlog(" pread3 SUCCESS") 

361 return stdout 

362 

363 

364def listdir(path, quiet=False): 

365 cmd = ["ls", path, "-1", "--color=never"] 

366 try: 

367 text = pread2(cmd, quiet=quiet)[:-1] 

368 if len(text) == 0: 

369 return [] 

370 return text.split('\n') 

371 except CommandException as inst: 

372 if inst.code == errno.ENOENT: 

373 raise CommandException(errno.EIO, inst.cmd, inst.reason) 

374 else: 

375 raise CommandException(inst.code, inst.cmd, inst.reason) 

376 

377 

378def gen_uuid(): 

379 cmd = ["uuidgen", "-r"] 

380 return pread(cmd)[:-1] 

381 

382 

383def match_uuid(s): 

384 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}") 

385 return regex.search(s, 0) 

386 

387 

388def findall_uuid(s): 

389 regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") 

390 return regex.findall(s, 0) 

391 

392 

393def exactmatch_uuid(s): 

394 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}$") 

395 return regex.search(s, 0) 

396 

397 

398def start_log_entry(srpath, path, args): 

399 logstring = str(datetime.datetime.now()) 

400 logstring += " log: " 

401 logstring += srpath 

402 logstring += " " + path 

403 for element in args: 

404 logstring += " " + element 

405 try: 

406 file = open(srpath + "/filelog.txt", "a") 

407 file.write(logstring) 

408 file.write("\n") 

409 file.close() 

410 except: 

411 pass 

412 

413 # failed to write log ... 

414 

415def end_log_entry(srpath, path, args): 

416 # for teminating, use "error" or "done" 

417 logstring = str(datetime.datetime.now()) 

418 logstring += " end: " 

419 logstring += srpath 

420 logstring += " " + path 

421 for element in args: 

422 logstring += " " + element 

423 try: 

424 file = open(srpath + "/filelog.txt", "a") 

425 file.write(logstring) 

426 file.write("\n") 

427 file.close() 

428 except: 

429 pass 

430 

431 # failed to write log ... 

432 # for now print 

433 # print "%s" % logstring 

434 

435def ioretry(f, errlist=[errno.EIO], maxretry=IORETRY_MAX, period=IORETRY_PERIOD, **ignored): 

436 retries = 0 

437 while True: 

438 try: 

439 return f() 

440 except OSError as ose: 

441 err = int(ose.errno) 

442 if not err in errlist: 

443 raise CommandException(err, str(f), "OSError") 

444 except CommandException as ce: 

445 if not int(ce.code) in errlist: 

446 raise 

447 

448 retries += 1 

449 if retries >= maxretry: 

450 break 

451 

452 time.sleep(period) 

453 

454 raise CommandException(errno.ETIMEDOUT, str(f), "Timeout") 

455 

456 

457def ioretry_stat(path, maxretry=IORETRY_MAX): 

458 # this ioretry is similar to the previous method, but 

459 # stat does not raise an error -- so check its return 

460 retries = 0 

461 while retries < maxretry: 

462 stat = os.statvfs(path) 

463 if stat.f_blocks != -1: 

464 return stat 

465 time.sleep(1) 

466 retries += 1 

467 raise CommandException(errno.EIO, "os.statvfs") 

468 

469 

470def sr_get_capability(sr_uuid, session=None): 

471 result = [] 

472 api_session = None 

473 try: 

474 if session is None: 

475 api_session = ApiSession("SM-sr-get-capability") 

476 session = api_session.session 

477 sr_ref = session.xenapi.SR.get_by_uuid(sr_uuid) 

478 sm_type = session.xenapi.SR.get_record(sr_ref)['type'] 

479 sm_rec = session.xenapi.SM.get_all_records_where( 

480 "field \"type\" = \"%s\"" % sm_type) 

481 

482 # SM expects at least one entry of any SR type 

483 if len(sm_rec) > 0: 

484 result = list(sm_rec.values())[0]['capabilities'] 

485 finally: 

486 if api_session: 

487 session.logout() 

488 return result 

489 

490def sr_get_driver_info(driver_info): 

491 results = {} 

492 # first add in the vanilla stuff 

493 for key in ['name', 'description', 'vendor', 'copyright', \ 

494 'driver_version', 'required_api_version']: 

495 results[key] = driver_info[key] 

496 # add the capabilities (xmlrpc array) 

497 # enforcing activate/deactivate for blktap2 

498 caps = driver_info['capabilities'] 

499 if "ATOMIC_PAUSE" in caps: 499 ↛ 500line 499 didn't jump to line 500, because the condition on line 499 was never true

500 for cap in ("VDI_ACTIVATE", "VDI_DEACTIVATE"): 

501 if not cap in caps: 

502 caps.append(cap) 

503 elif "VDI_ACTIVATE" in caps or "VDI_DEACTIVATE" in caps: 503 ↛ 504line 503 didn't jump to line 504, because the condition on line 503 was never true

504 SMlog("Warning: vdi_[de]activate present for %s" % driver_info["name"]) 

505 

506 results['capabilities'] = caps 

507 # add in the configuration options 

508 options = [] 

509 for option in driver_info['configuration']: 

510 options.append({'key': option[0], 'description': option[1]}) 

511 results['configuration'] = options 

512 return xmlrpc.client.dumps((results, ), "", True) 

513 

514 

515def return_nil(): 

516 return xmlrpc.client.dumps((None, ), "", True, allow_none=True) 

517 

518 

519def SRtoXML(SRlist): 

520 dom = xml.dom.minidom.Document() 

521 driver = dom.createElement("SRlist") 

522 dom.appendChild(driver) 

523 

524 for key in SRlist.keys(): 

525 dict = SRlist[key] 

526 entry = dom.createElement("SR") 

527 driver.appendChild(entry) 

528 

529 e = dom.createElement("UUID") 

530 entry.appendChild(e) 

531 textnode = dom.createTextNode(key) 

532 e.appendChild(textnode) 

533 

534 if 'size' in dict: 

535 e = dom.createElement("Size") 

536 entry.appendChild(e) 

537 textnode = dom.createTextNode(str(dict['size'])) 

538 e.appendChild(textnode) 

539 

540 if 'storagepool' in dict: 

541 e = dom.createElement("StoragePool") 

542 entry.appendChild(e) 

543 textnode = dom.createTextNode(str(dict['storagepool'])) 

544 e.appendChild(textnode) 

545 

546 if 'aggregate' in dict: 

547 e = dom.createElement("Aggregate") 

548 entry.appendChild(e) 

549 textnode = dom.createTextNode(str(dict['aggregate'])) 

550 e.appendChild(textnode) 

551 

552 return dom.toprettyxml() 

553 

554 

555def pathexists(path): 

556 try: 

557 os.lstat(path) 

558 return True 

559 except OSError as inst: 

560 if inst.errno == errno.EIO: 560 ↛ 561line 560 didn't jump to line 561, because the condition on line 560 was never true

561 time.sleep(1) 

562 try: 

563 os.lstat(path) 

564 return True 

565 except: 

566 pass 

567 raise CommandException(errno.EIO, "os.lstat(%s)" % path, "failed") 

568 return False 

569 

570 

571def force_unlink(path): 

572 try: 

573 os.unlink(path) 

574 except OSError as e: 

575 if e.errno != errno.ENOENT: 575 ↛ 576line 575 didn't jump to line 576, because the condition on line 575 was never true

576 raise 

577 

578 

579def create_secret(session, secret): 

580 ref = session.xenapi.secret.create({'value': secret}) 

581 return session.xenapi.secret.get_uuid(ref) 

582 

583 

584def get_secret(session, uuid): 

585 try: 

586 ref = session.xenapi.secret.get_by_uuid(uuid) 

587 return session.xenapi.secret.get_value(ref) 

588 except: 

589 raise xs_errors.XenError('InvalidSecret', opterr='Unable to look up secret [%s]' % uuid) 

590 

591 

592def get_real_path(path): 

593 "Follow symlinks to the actual file" 

594 absPath = path 

595 directory = '' 

596 while os.path.islink(absPath): 

597 directory = os.path.dirname(absPath) 

598 absPath = os.readlink(absPath) 

599 absPath = os.path.join(directory, absPath) 

600 return absPath 

601 

602 

603def wait_for_path(path, timeout): 

604 for i in range(0, timeout): 604 ↛ 608line 604 didn't jump to line 608, because the loop on line 604 didn't complete

605 if len(glob.glob(path)): 605 ↛ 607line 605 didn't jump to line 607, because the condition on line 605 was never false

606 return True 

607 time.sleep(1) 

608 return False 

609 

610 

611def wait_for_nopath(path, timeout): 

612 for i in range(0, timeout): 

613 if not os.path.exists(path): 

614 return True 

615 time.sleep(1) 

616 return False 

617 

618 

619def wait_for_path_multi(path, timeout): 

620 for i in range(0, timeout): 

621 paths = glob.glob(path) 

622 SMlog("_wait_for_paths_multi: paths = %s" % paths) 

623 if len(paths): 

624 SMlog("_wait_for_paths_multi: return first path: %s" % paths[0]) 

625 return paths[0] 

626 time.sleep(1) 

627 return "" 

628 

629 

630def isdir(path): 

631 try: 

632 st = os.stat(path) 

633 return stat.S_ISDIR(st.st_mode) 

634 except OSError as inst: 

635 if inst.errno == errno.EIO: 635 ↛ 636line 635 didn't jump to line 636, because the condition on line 635 was never true

636 raise CommandException(errno.EIO, "os.stat(%s)" % path, "failed") 

637 return False 

638 

639 

640def get_single_entry(path): 

641 f = open(path, 'r') 

642 line = f.readline() 

643 f.close() 

644 return line.rstrip() 

645 

646 

647def get_fs_size(path): 

648 st = ioretry_stat(path) 

649 return st.f_blocks * st.f_frsize 

650 

651 

652def get_fs_utilisation(path): 

653 st = ioretry_stat(path) 

654 return (st.f_blocks - st.f_bfree) * \ 

655 st.f_frsize 

656 

657 

658def ismount(path): 

659 """Test whether a path is a mount point""" 

660 try: 

661 s1 = os.stat(path) 

662 s2 = os.stat(os.path.join(path, '..')) 

663 except OSError as inst: 

664 raise CommandException(inst.errno, "os.stat") 

665 dev1 = s1.st_dev 

666 dev2 = s2.st_dev 

667 if dev1 != dev2: 

668 return True # path/.. on a different device as path 

669 ino1 = s1.st_ino 

670 ino2 = s2.st_ino 

671 if ino1 == ino2: 

672 return True # path/.. is the same i-node as path 

673 return False 

674 

675 

676def makedirs(name, mode=0o777): 

677 head, tail = os.path.split(name) 

678 if not tail: 678 ↛ 679line 678 didn't jump to line 679, because the condition on line 678 was never true

679 head, tail = os.path.split(head) 

680 if head and tail and not pathexists(head): 

681 makedirs(head, mode) 

682 if tail == os.curdir: 682 ↛ 683line 682 didn't jump to line 683, because the condition on line 682 was never true

683 return 

684 try: 

685 os.mkdir(name, mode) 

686 except OSError as exc: 

687 if exc.errno == errno.EEXIST and os.path.isdir(name): 687 ↛ 688line 687 didn't jump to line 688, because the condition on line 687 was never true

688 if mode: 

689 os.chmod(name, mode) 

690 pass 

691 else: 

692 raise 

693 

694 

695def zeroOut(path, fromByte, bytes): 

696 """write 'bytes' zeros to 'path' starting from fromByte (inclusive)""" 

697 blockSize = 4096 

698 

699 fromBlock = fromByte // blockSize 

700 if fromByte % blockSize: 

701 fromBlock += 1 

702 bytesBefore = fromBlock * blockSize - fromByte 

703 if bytesBefore > bytes: 

704 bytesBefore = bytes 

705 bytes -= bytesBefore 

706 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1", 

707 "seek=%s" % fromByte, "count=%s" % bytesBefore] 

708 try: 

709 pread2(cmd) 

710 except CommandException: 

711 return False 

712 

713 blocks = bytes // blockSize 

714 bytes -= blocks * blockSize 

715 fromByte = (fromBlock + blocks) * blockSize 

716 if blocks: 

717 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=%s" % blockSize, 

718 "seek=%s" % fromBlock, "count=%s" % blocks] 

719 try: 

720 pread2(cmd) 

721 except CommandException: 

722 return False 

723 

724 if bytes: 

725 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1", 

726 "seek=%s" % fromByte, "count=%s" % bytes] 

727 try: 

728 pread2(cmd) 

729 except CommandException: 

730 return False 

731 

732 return True 

733 

734 

735def wipefs(blockdev): 

736 "Wipe filesystem signatures from `blockdev`" 

737 pread2(["/usr/sbin/wipefs", "-a", blockdev]) 

738 

739 

740def match_rootdev(s): 

741 regex = re.compile("^PRIMARY_DISK") 

742 return regex.search(s, 0) 

743 

744 

745def getrootdev(): 

746 filename = '/etc/xensource-inventory' 

747 try: 

748 f = open(filename, 'r') 

749 except: 

750 raise xs_errors.XenError('EIO', \ 

751 opterr="Unable to open inventory file [%s]" % filename) 

752 rootdev = '' 

753 for line in filter(match_rootdev, f.readlines()): 

754 rootdev = line.split("'")[1] 

755 if not rootdev: 755 ↛ 756line 755 didn't jump to line 756, because the condition on line 755 was never true

756 raise xs_errors.XenError('NoRootDev') 

757 return rootdev 

758 

759 

760def getrootdevID(): 

761 rootdev = getrootdev() 

762 try: 

763 rootdevID = scsiutil.getSCSIid(rootdev) 

764 except: 

765 SMlog("util.getrootdevID: Unable to verify serial or SCSIid of device: %s" \ 

766 % rootdev) 

767 return '' 

768 

769 if not len(rootdevID): 

770 SMlog("util.getrootdevID: Unable to identify scsi device [%s] via scsiID" \ 

771 % rootdev) 

772 

773 return rootdevID 

774 

775 

776class ApiSession(contextlib.AbstractContextManager): 

777 session = None 

778 

779 def __init__(self, originator="SM"): 

780 self.originator = originator 

781 # First acquire a valid session 

782 self.session = self._login() 

783 atexit.register(self._atexit) 

784 

785 def _login(self): 

786 if self.session: 

787 return self.session 

788 session = XenAPI.xapi_local() 

789 try: 

790 session.xenapi.login_with_password('root', '', '', self.originator) 

791 except Exception as exc: 

792 raise xs_errors.XenError( 

793 "APISession", 

794 opterr=f"[{self.originator}] Unable to open local XAPI session" 

795 ) from exc 

796 SMlog("ApiSession [{}] login".format(self.originator), priority=LOG_DEBUG) 

797 return session 

798 

799 def _logout(self, log): 

800 """Closes an API session""" 

801 if self.session is None: 801 ↛ 802line 801 didn't jump to line 802, because the condition on line 801 was never true

802 SMlog("ApiSession [{}] session is None {}".format(self.originator, log), priority=LOG_DEBUG) 

803 return 

804 self.session.xenapi.session.logout() 

805 SMlog("ApiSession [{}] {}".format(self.originator, log), priority=LOG_DEBUG) 

806 self.session = None 

807 

808 def logout(self, log="logout"): 

809 atexit.unregister(self._atexit) 

810 self._logout(log=log) 

811 

812 def __del__(self): 

813 if self.session: 813 ↛ 814line 813 didn't jump to line 814, because the condition on line 813 was never true

814 self.logout(log="logout del") 

815 

816 def _atexit(self): 

817 self._logout(log="logout atexit") 

818 

819 @override 

820 def __enter__(self): 

821 self.session = self._login() 

822 return self.session 

823 

824 @override 

825 def __exit__(self, _type, _value, _traceback): 

826 self.logout(log=f"logout exception[{_type}] {_value}" if _type else "logout context") 

827 

828 

829def xapi_safe_call(function): 

830 """ 

831 Decorator to catch classic XAPI connection problems and retry forever. 

832 The method should be only called for XAPI calls. 

833 eg: blktap2.py#VDI:_remove_tag() 

834 """ 

835 def wrapper(*args, **kwargs): 

836 call_str = f"{function.__name__}(args={args}, kwargs={kwargs})" 

837 while True: 

838 try: 

839 return function(*args, **kwargs) 

840 except xmlrpc.client.ProtocolError as e: 

841 # If there's a connection error, keep trying forever. 

842 if e.errcode == http.HTTPStatus.INTERNAL_SERVER_ERROR.value: 

843 continue 

844 raise 

845 except Exception as e: 

846 SMlog(f"Failed XAPI call `{call_str}`: `{e}`") 

847 raise 

848 return wrapper 

849 

850 

851def get_this_host(): 

852 uuid = None 

853 f = open("/etc/xensource-inventory", 'r') 

854 for line in f.readlines(): 

855 if line.startswith("INSTALLATION_UUID"): 

856 uuid = line.split("'")[1] 

857 f.close() 

858 return uuid 

859 

860 

861def get_master_ref(session): 

862 pools = session.xenapi.pool.get_all() 

863 return session.xenapi.pool.get_master(pools[0]) 

864 

865 

866def is_master(session): 

867 return get_this_host_ref(session) == get_master_ref(session) 

868 

869 

870def get_localhost_ref(session): 

871 filename = '/etc/xensource-inventory' 

872 try: 

873 f = open(filename, 'r') 

874 except: 

875 raise xs_errors.XenError('EIO', \ 

876 opterr="Unable to open inventory file [%s]" % filename) 

877 domid = '' 

878 for line in filter(match_domain_id, f.readlines()): 

879 domid = line.split("'")[1] 

880 if not domid: 

881 raise xs_errors.XenError('APILocalhost') 

882 

883 vms = session.xenapi.VM.get_all_records_where('field "uuid" = "%s"' % domid) 

884 for vm in vms: 

885 record = vms[vm] 

886 if record["uuid"] == domid: 

887 hostid = record["resident_on"] 

888 return hostid 

889 raise xs_errors.XenError('APILocalhost') 

890 

891 

892def match_domain_id(s): 

893 regex = re.compile("^CONTROL_DOMAIN_UUID") 

894 return regex.search(s, 0) 

895 

896 

897def get_hosts_attached_on(session, vdi_uuids): 

898 host_refs = {} 

899 for vdi_uuid in vdi_uuids: 

900 try: 

901 vdi_ref = session.xenapi.VDI.get_by_uuid(vdi_uuid) 

902 except XenAPI.Failure: 

903 SMlog("VDI %s not in db, ignoring" % vdi_uuid) 

904 continue 

905 sm_config = session.xenapi.VDI.get_sm_config(vdi_ref) 

906 for key in [x for x in sm_config.keys() if x.startswith('host_')]: 

907 host_refs[key[len('host_'):]] = True 

908 return host_refs.keys() 

909 

910def get_hosts_attached_on_with_vdi_uuid(session, vdi_uuids): 

911 """ 

912 Return a dict of {vdi_uuid: host OpaqueRef} 

913 """ 

914 host_refs = {} 

915 for vdi_uuid in vdi_uuids: 

916 try: 

917 vdi_ref = session.xenapi.VDI.get_by_uuid(vdi_uuid) 

918 except XenAPI.Failure: 

919 SMlog("VDI %s not in db, ignoring" % vdi_uuid) 

920 continue 

921 sm_config = session.xenapi.VDI.get_sm_config(vdi_ref) 

922 for key in [x for x in sm_config.keys() if x.startswith('host_')]: 

923 host_refs[vdi_uuid] = key[len('host_'):] 

924 return host_refs 

925 

926def get_this_host_address(session): 

927 host_uuid = get_this_host() 

928 host_ref = session.xenapi.host.get_by_uuid(host_uuid) 

929 return session.xenapi.host.get_record(host_ref)['address'] 

930 

931def get_host_addresses(session): 

932 addresses = [] 

933 hosts = session.xenapi.host.get_all_records() 

934 for record in hosts.values(): 

935 addresses.append(record['address']) 

936 return addresses 

937 

938def get_this_host_ref(session): 

939 host_uuid = get_this_host() 

940 host_ref = session.xenapi.host.get_by_uuid(host_uuid) 

941 return host_ref 

942 

943 

944def get_slaves_attached_on(session, vdi_uuids): 

945 "assume this host is the SR master" 

946 host_refs = get_hosts_attached_on(session, vdi_uuids) 

947 master_ref = get_this_host_ref(session) 

948 return [x for x in host_refs if x != master_ref] 

949 

950def get_enabled_hosts(session): 

951 """ 

952 Returns a list of host refs that are enabled in the pool. 

953 """ 

954 return list(session.xenapi.host.get_all_records_where('field "enabled" = "true"').keys()) 

955 

956def get_online_hosts(session): 

957 online_hosts = [] 

958 hosts = session.xenapi.host.get_all_records() 

959 for host_ref, host_rec in hosts.items(): 

960 metricsRef = host_rec["metrics"] 

961 metrics = session.xenapi.host_metrics.get_record(metricsRef) 

962 if metrics["live"]: 

963 online_hosts.append(host_ref) 

964 return online_hosts 

965 

966 

967def get_all_slaves(session): 

968 "assume this host is the SR master" 

969 host_refs = get_online_hosts(session) 

970 master_ref = get_this_host_ref(session) 

971 return [x for x in host_refs if x != master_ref] 

972 

973 

974def is_attached_rw(sm_config): 

975 for key, val in sm_config.items(): 

976 if key.startswith("host_") and val == "RW": 

977 return True 

978 return False 

979 

980 

981def attached_as(sm_config): 

982 for key, val in sm_config.items(): 

983 if key.startswith("host_") and (val == "RW" or val == "RO"): 983 ↛ 984line 983 didn't jump to line 984, because the condition on line 983 was never true

984 return val 

985 

986 

987def find_my_pbd_record(session, host_ref, sr_ref): 

988 try: 

989 pbds = session.xenapi.PBD.get_all_records() 

990 for pbd_ref in pbds.keys(): 

991 if pbds[pbd_ref]['host'] == host_ref and pbds[pbd_ref]['SR'] == sr_ref: 

992 return [pbd_ref, pbds[pbd_ref]] 

993 return None 

994 except Exception as e: 

995 SMlog("Caught exception while looking up PBD for host %s SR %s: %s" % (str(host_ref), str(sr_ref), str(e))) 

996 return None 

997 

998 

999def find_my_pbd(session, host_ref, sr_ref): 

1000 ret = find_my_pbd_record(session, host_ref, sr_ref) 

1001 if ret is not None: 

1002 return ret[0] 

1003 else: 

1004 return None 

1005 

1006 

1007def test_hostPBD_devs(session, sr_uuid, devs): 

1008 host = get_localhost_ref(session) 

1009 sr = session.xenapi.SR.get_by_uuid(sr_uuid) 

1010 try: 

1011 pbds = session.xenapi.PBD.get_all_records() 

1012 except: 

1013 raise xs_errors.XenError('APIPBDQuery') 

1014 for dev in devs.split(','): 

1015 for pbd in pbds: 

1016 record = pbds[pbd] 

1017 # it's ok if it's *our* PBD 

1018 if record["SR"] == sr: 

1019 break 

1020 if record["host"] == host: 

1021 devconfig = record["device_config"] 

1022 if 'device' in devconfig: 

1023 for device in devconfig['device'].split(','): 

1024 if os.path.realpath(device) == os.path.realpath(dev): 

1025 return True 

1026 return False 

1027 

1028 

1029def test_hostPBD_lun(session, targetIQN, LUNid): 

1030 host = get_localhost_ref(session) 

1031 try: 

1032 pbds = session.xenapi.PBD.get_all_records() 

1033 except: 

1034 raise xs_errors.XenError('APIPBDQuery') 

1035 for pbd in pbds: 

1036 record = pbds[pbd] 

1037 if record["host"] == host: 

1038 devconfig = record["device_config"] 

1039 if 'targetIQN' in devconfig and 'LUNid' in devconfig: 

1040 if devconfig['targetIQN'] == targetIQN and \ 

1041 devconfig['LUNid'] == LUNid: 

1042 return True 

1043 return False 

1044 

1045 

1046def test_SCSIid(session, sr_uuid, SCSIid): 

1047 if sr_uuid is not None: 

1048 sr = session.xenapi.SR.get_by_uuid(sr_uuid) 

1049 try: 

1050 pbds = session.xenapi.PBD.get_all_records() 

1051 except: 

1052 raise xs_errors.XenError('APIPBDQuery') 

1053 for pbd in pbds: 

1054 record = pbds[pbd] 

1055 # it's ok if it's *our* PBD 

1056 # During FC SR creation, devscan.py passes sr_uuid as None 

1057 if sr_uuid is not None: 

1058 if record["SR"] == sr: 

1059 break 

1060 devconfig = record["device_config"] 

1061 sm_config = session.xenapi.SR.get_sm_config(record["SR"]) 

1062 if 'SCSIid' in devconfig and devconfig['SCSIid'] == SCSIid: 

1063 return True 

1064 elif 'SCSIid' in sm_config and sm_config['SCSIid'] == SCSIid: 

1065 return True 

1066 elif 'scsi-' + SCSIid in sm_config: 

1067 return True 

1068 return False 

1069 

1070 

1071class TimeoutException(SMException): 

1072 pass 

1073 

1074 

1075def _incr_iscsiSR_refcount(targetIQN, uuid): 

1076 if not os.path.exists(ISCSI_REFDIR): 

1077 os.mkdir(ISCSI_REFDIR) 

1078 filename = os.path.join(ISCSI_REFDIR, targetIQN) 

1079 try: 

1080 f = open(filename, 'a+') 

1081 except: 

1082 raise xs_errors.XenError('LVMRefCount', \ 

1083 opterr='file %s' % filename) 

1084 

1085 f.seek(0) 

1086 found = False 

1087 refcount = 0 

1088 for line in filter(match_uuid, f.readlines()): 

1089 refcount += 1 

1090 if line.find(uuid) != -1: 

1091 found = True 

1092 if not found: 

1093 f.write("%s\n" % uuid) 

1094 refcount += 1 

1095 f.close() 

1096 return refcount 

1097 

1098 

1099def _decr_iscsiSR_refcount(targetIQN, uuid): 

1100 filename = os.path.join(ISCSI_REFDIR, targetIQN) 

1101 if not os.path.exists(filename): 

1102 return 0 

1103 try: 

1104 f = open(filename, 'a+') 

1105 except: 

1106 raise xs_errors.XenError('LVMRefCount', \ 

1107 opterr='file %s' % filename) 

1108 

1109 f.seek(0) 

1110 output = [] 

1111 refcount = 0 

1112 for line in filter(match_uuid, f.readlines()): 

1113 if line.find(uuid) == -1: 

1114 output.append(line.rstrip()) 

1115 refcount += 1 

1116 if not refcount: 

1117 os.unlink(filename) 

1118 return refcount 

1119 

1120 # Re-open file and truncate 

1121 f.close() 

1122 f = open(filename, 'w') 

1123 for i in range(0, refcount): 

1124 f.write("%s\n" % output[i]) 

1125 f.close() 

1126 return refcount 

1127 

1128 

1129# The agent enforces 1 PBD per SR per host, so we 

1130# check for active SR entries not attached to this host 

1131def test_activePoolPBDs(session, host, uuid): 

1132 try: 

1133 pbds = session.xenapi.PBD.get_all_records() 

1134 except: 

1135 raise xs_errors.XenError('APIPBDQuery') 

1136 for pbd in pbds: 

1137 record = pbds[pbd] 

1138 if record["host"] != host and record["SR"] == uuid \ 

1139 and record["currently_attached"]: 

1140 return True 

1141 return False 

1142 

1143 

1144def remove_mpathcount_field(session, host_ref, sr_ref, SCSIid): 

1145 try: 

1146 pbdref = find_my_pbd(session, host_ref, sr_ref) 

1147 if pbdref is not None: 

1148 key = "mpath-" + SCSIid 

1149 session.xenapi.PBD.remove_from_other_config(pbdref, key) 

1150 except: 

1151 pass 

1152 

1153 

1154def kickpipe_mpathcount(): 

1155 """ 

1156 Issue a kick to the mpathcount service. This will ensure that mpathcount runs 

1157 shortly to update the multipath config records, if it was not already activated 

1158 by a UDEV event. 

1159 """ 

1160 cmd = [CMD_KICKPIPE, "mpathcount"] 

1161 (rc, stdout, stderr) = doexec(cmd) 

1162 return (rc == 0) 

1163 

1164 

1165def _testHost(hostname, port, errstring): 

1166 SMlog("_testHost: Testing host/port: %s,%d" % (hostname, port)) 

1167 try: 

1168 sockinfo = socket.getaddrinfo(hostname, int(port))[0] 

1169 except: 

1170 logException('Exception occured getting IP for %s' % hostname) 

1171 raise xs_errors.XenError('DNSError') 

1172 

1173 timeout = 5 

1174 

1175 sock = socket.socket(sockinfo[0], socket.SOCK_STREAM) 

1176 # Only allow the connect to block for up to timeout seconds 

1177 sock.settimeout(timeout) 

1178 try: 

1179 sock.connect(sockinfo[4]) 

1180 # Fix for MS storage server bug 

1181 sock.send(b'\n') 

1182 sock.close() 

1183 except socket.error as reason: 

1184 SMlog("_testHost: Connect failed after %d seconds (%s) - %s" \ 

1185 % (timeout, hostname, reason)) 

1186 raise xs_errors.XenError(errstring) 

1187 

1188 

1189def match_scsiID(s, id): 

1190 regex = re.compile(id) 

1191 return regex.search(s, 0) 

1192 

1193 

1194def _isSCSIid(s): 

1195 regex = re.compile("^scsi-") 

1196 return regex.search(s, 0) 

1197 

1198 

1199def is_usb_device(device): 

1200 cmd = ["udevadm", "info", "-q", "path", "-n", device] 

1201 result = pread2(cmd).split('/') 

1202 return len(result) >= 5 and result[4].startswith('usb') 

1203 

1204 

1205def test_scsiserial(session, device): 

1206 device = os.path.realpath(device) 

1207 if not scsiutil._isSCSIdev(device): 

1208 SMlog("util.test_scsiserial: Not a serial device: %s" % device) 

1209 return False 

1210 serial = "" 

1211 try: 

1212 serial += scsiutil.getserial(device) 

1213 except: 

1214 # Error allowed, SCSIid is the important one 

1215 pass 

1216 

1217 try: 

1218 scsiID = scsiutil.getSCSIid(device) 

1219 except: 

1220 SMlog("util.test_scsiserial: Unable to verify serial or SCSIid of device: %s" \ 

1221 % device) 

1222 return False 

1223 if not len(scsiID): 

1224 SMlog("util.test_scsiserial: Unable to identify scsi device [%s] via scsiID" \ 

1225 % device) 

1226 return False 

1227 

1228 # USB devices can have identical SCSI IDs - prefer matching with serial number 

1229 try: 

1230 usb_device_with_serial = serial and is_usb_device(device) 

1231 except: 

1232 usb_device_with_serial = False 

1233 SMlog("Unable to check if device is USB:") 

1234 SMlog(traceback.format_exc()) 

1235 

1236 try: 

1237 SRs = session.xenapi.SR.get_all_records() 

1238 except: 

1239 raise xs_errors.XenError('APIFailure') 

1240 for SR in SRs: 

1241 record = SRs[SR] 

1242 conf = record["sm_config"] 

1243 if 'devserial' in conf: 

1244 for dev in conf['devserial'].split(','): 

1245 if not usb_device_with_serial and _isSCSIid(dev): 

1246 if match_scsiID(dev, scsiID): 

1247 return True 

1248 elif len(serial) and dev == serial: 

1249 return True 

1250 return False 

1251 

1252 

1253def default(self, field, thunk): 

1254 try: 

1255 return getattr(self, field) 

1256 except: 

1257 return thunk() 

1258 

1259 

1260def list_VDI_records_in_sr(sr): 

1261 """Helper function which returns a list of all VDI records for this SR 

1262 stored in the XenAPI server, useful for implementing SR.scan""" 

1263 sr_ref = sr.session.xenapi.SR.get_by_uuid(sr.uuid) 

1264 vdis = sr.session.xenapi.VDI.get_all_records_where("field \"SR\" = \"%s\"" % sr_ref) 

1265 return vdis 

1266 

1267 

1268# Given a partition (e.g. sda1), get a disk name: 

1269def diskFromPartition(partition): 

1270 # check whether this is a device mapper device (e.g. /dev/dm-0) 

1271 m = re.match('(/dev/)?(dm-[0-9]+)(p[0-9]+)?$', partition) 

1272 if m is not None: 1272 ↛ 1273line 1272 didn't jump to line 1273, because the condition on line 1272 was never true

1273 return m.group(2) 

1274 

1275 numlen = 0 # number of digit characters 

1276 m = re.match(r"\D+(\d+)", partition) 

1277 if m is not None: 1277 ↛ 1278line 1277 didn't jump to line 1278, because the condition on line 1277 was never true

1278 numlen = len(m.group(1)) 

1279 

1280 # is it a cciss? 

1281 if True in [partition.startswith(x) for x in ['cciss', 'ida', 'rd']]: 1281 ↛ 1282line 1281 didn't jump to line 1282, because the condition on line 1281 was never true

1282 numlen += 1 # need to get rid of trailing 'p' 

1283 

1284 # is it a mapper path? 

1285 if partition.startswith("mapper"): 1285 ↛ 1286line 1285 didn't jump to line 1286, because the condition on line 1285 was never true

1286 if re.search("p[0-9]*$", partition): 

1287 numlen = len(re.match(r"\d+", partition[::-1]).group(0)) + 1 

1288 SMlog("Found mapper part, len %d" % numlen) 

1289 else: 

1290 numlen = 0 

1291 

1292 # is it /dev/disk/by-id/XYZ-part<k>? 

1293 if partition.startswith("disk/by-id"): 1293 ↛ 1294line 1293 didn't jump to line 1294, because the condition on line 1293 was never true

1294 return partition[:partition.rfind("-part")] 

1295 

1296 return partition[:len(partition) - numlen] 

1297 

1298 

1299def dom0_disks(): 

1300 """Disks carrying dom0, e.g. ['/dev/sda']""" 

1301 disks = [] 

1302 with open("/etc/mtab", 'r') as f: 

1303 for line in f: 

1304 (dev, mountpoint, fstype, opts, freq, passno) = line.split(' ') 

1305 if mountpoint == '/': 

1306 disk = diskFromPartition(dev) 

1307 if not (disk in disks): 

1308 disks.append(disk) 

1309 SMlog("Dom0 disks: %s" % disks) 

1310 return disks 

1311 

1312 

1313def set_scheduler_sysfs_node(node, scheds): 

1314 """ 

1315 Set the scheduler for a sysfs node (e.g. '/sys/block/sda') 

1316 according to prioritized list schedulers 

1317 Try to set the first item, then fall back to the next on failure 

1318 """ 

1319 

1320 path = os.path.join(node, "queue", "scheduler") 

1321 if not os.path.exists(path): 1321 ↛ 1325line 1321 didn't jump to line 1325, because the condition on line 1321 was never false

1322 SMlog("no path %s" % path) 

1323 return 

1324 

1325 stored_error = None 

1326 for sched in scheds: 

1327 try: 

1328 with open(path, 'w') as file: 

1329 file.write("%s\n" % sched) 

1330 SMlog("Set scheduler to [%s] on [%s]" % (sched, node)) 

1331 return 

1332 except (OSError, IOError) as err: 

1333 stored_error = err 

1334 

1335 SMlog("Error setting schedulers to [%s] on [%s], %s" % (scheds, node, str(stored_error))) 

1336 

1337 

1338def set_scheduler(dev, schedulers=None): 

1339 if schedulers is None: 1339 ↛ 1342line 1339 didn't jump to line 1342, because the condition on line 1339 was never false

1340 schedulers = ["none", "noop"] 

1341 

1342 devices = [] 

1343 if not scsiutil.match_dm(dev): 1343 ↛ 1347line 1343 didn't jump to line 1347, because the condition on line 1343 was never false

1344 # Remove partition numbers 

1345 devices.append(diskFromPartition(dev).replace('/', '!')) 

1346 else: 

1347 rawdev = diskFromPartition(dev) 

1348 devices = [os.path.realpath(x)[5:] for x in scsiutil._genReverseSCSIidmap(rawdev.split('/')[-1])] 

1349 

1350 for d in devices: 

1351 set_scheduler_sysfs_node("/sys/block/%s" % d, schedulers) 

1352 

1353 

1354# This function queries XAPI for the existing VDI records for this SR 

1355def _getVDIs(srobj): 

1356 VDIs = [] 

1357 try: 

1358 sr_ref = getattr(srobj, 'sr_ref') 

1359 except AttributeError: 

1360 return VDIs 

1361 

1362 refs = srobj.session.xenapi.SR.get_VDIs(sr_ref) 

1363 for vdi in refs: 

1364 ref = srobj.session.xenapi.VDI.get_record(vdi) 

1365 ref['vdi_ref'] = vdi 

1366 VDIs.append(ref) 

1367 return VDIs 

1368 

1369 

1370def get_sr_uuid_from_vdi_ref(session, vdi_ref: str) -> str: 

1371 sr_ref = session.xenapi.VDI.get_SR(vdi_ref) 

1372 return session.xenapi.SR.get_uuid(sr_ref) 

1373 

1374 

1375def get_sr_uuid_from_vdi_uuid(session, vdi_uuid: str) -> str: 

1376 return get_sr_uuid_from_vdi_ref(session, session.xenapi.VDI.get_by_uuid(vdi_uuid)) 

1377 

1378 

1379def _getVDI(srobj, vdi_uuid): 

1380 vdi = srobj.session.xenapi.VDI.get_by_uuid(vdi_uuid) 

1381 ref = srobj.session.xenapi.VDI.get_record(vdi) 

1382 ref['vdi_ref'] = vdi 

1383 return ref 

1384 

1385 

1386def _convertDNS(name): 

1387 addr = socket.getaddrinfo(name, None)[0][4][0] 

1388 return addr 

1389 

1390 

1391def _containsVDIinuse(srobj): 

1392 VDIs = _getVDIs(srobj) 

1393 for vdi in VDIs: 

1394 if not vdi['managed']: 

1395 continue 

1396 sm_config = vdi['sm_config'] 

1397 if 'SRRef' in sm_config: 

1398 try: 

1399 PBDs = srobj.session.xenapi.SR.get_PBDs(sm_config['SRRef']) 

1400 for pbd in PBDs: 

1401 record = PBDs[pbd] 

1402 if record["host"] == srobj.host_ref and \ 

1403 record["currently_attached"]: 

1404 return True 

1405 except: 

1406 pass 

1407 return False 

1408 

1409 

1410def isVDICommand(cmd): 

1411 if cmd is None or cmd in ["vdi_attach", "vdi_detach", 

1412 "vdi_activate", "vdi_deactivate", 

1413 "vdi_epoch_begin", "vdi_epoch_end"]: 

1414 return True 

1415 else: 

1416 return False 

1417 

1418 

1419######################### 

1420# Daemon helper functions 

1421def p_id_fork(): 

1422 try: 

1423 p_id = os.fork() 

1424 except OSError as e: 

1425 print("Fork failed: %s (%d)" % (e.strerror, e.errno)) 

1426 sys.exit(-1) 

1427 

1428 if (p_id == 0): 

1429 os.setsid() 

1430 try: 

1431 p_id = os.fork() 

1432 except OSError as e: 

1433 print("Fork failed: %s (%d)" % (e.strerror, e.errno)) 

1434 sys.exit(-1) 

1435 if (p_id == 0): 

1436 os.chdir('/opt/xensource/sm') 

1437 os.umask(0) 

1438 else: 

1439 os._exit(0) 

1440 else: 

1441 os._exit(0) 

1442 

1443 

1444def daemon(): 

1445 p_id_fork() 

1446 # Query the max file descriptor parameter for this process 

1447 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] 

1448 

1449 # Close any fds that are open 

1450 for fd in range(0, maxfd): 

1451 try: 

1452 os.close(fd) 

1453 except: 

1454 pass 

1455 

1456 # Redirect STDIN to STDOUT and STDERR 

1457 os.open('/dev/null', os.O_RDWR) 

1458 os.dup2(0, 1) 

1459 os.dup2(0, 2) 

1460 

1461################################################################################ 

1462# 

1463# Fist points 

1464# 

1465 

1466# * The global variable 'fistpoint' define the list of all possible fistpoints; 

1467# 

1468# * To activate a fistpoint called 'name', you need to create the file '/tmp/fist_name' 

1469# on the SR master; 

1470# 

1471# * At the moment, activating a fist point can lead to two possible behaviors: 

1472# - if '/tmp/fist_LVHDRT_exit' exists, then the function called during the fistpoint is _exit; 

1473# - otherwise, the function called is _pause. 

1474 

1475def _pause(secs, name): 

1476 SMlog("Executing fist point %s: sleeping %d seconds ..." % (name, secs)) 

1477 time.sleep(secs) 

1478 SMlog("Executing fist point %s: done" % name) 

1479 

1480 

1481def _exit(name): 

1482 SMlog("Executing fist point %s: exiting the current process ..." % name) 

1483 raise xs_errors.XenError('FistPoint', opterr='%s' % name) 

1484 

1485 

1486class FistPoint: 

1487 def __init__(self, points): 

1488 #SMlog("Fist points loaded") 

1489 self.points = points 

1490 

1491 def is_legal(self, name): 

1492 return (name in self.points) 

1493 

1494 def is_active(self, name): 

1495 return os.path.exists("/tmp/fist_%s" % name) 

1496 

1497 def mark_sr(self, name, sruuid, started): 

1498 with ApiSession("SM-sr-fist-point") as session: 

1499 sr = session.xenapi.SR.get_by_uuid(sruuid) 

1500 if started: 

1501 session.xenapi.SR.add_to_other_config(sr, name, "active") 

1502 else: 

1503 session.xenapi.SR.remove_from_other_config(sr, name) 

1504 

1505 def activate(self, name, sruuid): 

1506 if name in self.points: 

1507 if self.is_active(name): 

1508 self.mark_sr(name, sruuid, True) 

1509 if self.is_active("LVHDRT_exit"): 1509 ↛ 1510line 1509 didn't jump to line 1510, because the condition on line 1509 was never true

1510 self.mark_sr(name, sruuid, False) 

1511 _exit(name) 

1512 else: 

1513 _pause(FIST_PAUSE_PERIOD, name) 

1514 self.mark_sr(name, sruuid, False) 

1515 else: 

1516 SMlog("Unknown fist point: %s" % name) 

1517 

1518 def activate_custom_fn(self, name, fn): 

1519 if name in self.points: 1519 ↛ 1525line 1519 didn't jump to line 1525, because the condition on line 1519 was never false

1520 if self.is_active(name): 1520 ↛ 1521line 1520 didn't jump to line 1521, because the condition on line 1520 was never true

1521 SMlog("Executing fist point %s: starting ..." % name) 

1522 fn() 

1523 SMlog("Executing fist point %s: done" % name) 

1524 else: 

1525 SMlog("Unknown fist point: %s" % name) 

1526 

1527 

1528def list_find(f, seq): 

1529 for item in seq: 

1530 if f(item): 

1531 return item 

1532 

1533GCPAUSE_FISTPOINT = "GCLoop_no_pause" 

1534 

1535fistpoint = FistPoint(["LVHDRT_finding_a_suitable_pair", 

1536 "LVHDRT_inflating_the_parent", 

1537 "LVHDRT_resizing_while_vdis_are_paused", 

1538 "LVHDRT_coalescing_VHD_data", 

1539 "LVHDRT_coalescing_before_inflate_grandparent", 

1540 "LVHDRT_relinking_grandchildren", 

1541 "LVHDRT_before_create_relink_journal", 

1542 "LVHDRT_xapiSM_serialization_tests", 

1543 "LVHDRT_clone_vdi_after_create_journal", 

1544 "LVHDRT_clone_vdi_after_shrink_parent", 

1545 "LVHDRT_clone_vdi_after_first_snap", 

1546 "LVHDRT_clone_vdi_after_second_snap", 

1547 "LVHDRT_clone_vdi_after_parent_hidden", 

1548 "LVHDRT_clone_vdi_after_parent_ro", 

1549 "LVHDRT_clone_vdi_before_remove_journal", 

1550 "LVHDRT_clone_vdi_after_lvcreate", 

1551 "LVHDRT_clone_vdi_before_undo_clone", 

1552 "LVHDRT_clone_vdi_after_undo_clone", 

1553 "LVHDRT_inflate_after_create_journal", 

1554 "LVHDRT_inflate_after_setSize", 

1555 "LVHDRT_inflate_after_zeroOut", 

1556 "LVHDRT_inflate_after_setSizePhys", 

1557 "LVHDRT_inflate_after_setSizePhys", 

1558 "LVHDRT_coaleaf_before_coalesce", 

1559 "LVHDRT_coaleaf_after_coalesce", 

1560 "LVHDRT_coaleaf_one_renamed", 

1561 "LVHDRT_coaleaf_both_renamed", 

1562 "LVHDRT_coaleaf_after_vdirec", 

1563 "LVHDRT_coaleaf_before_delete", 

1564 "LVHDRT_coaleaf_after_delete", 

1565 "LVHDRT_coaleaf_before_remove_j", 

1566 "LVHDRT_coaleaf_undo_after_rename", 

1567 "LVHDRT_coaleaf_undo_after_rename2", 

1568 "LVHDRT_coaleaf_undo_after_refcount", 

1569 "LVHDRT_coaleaf_undo_after_deflate", 

1570 "LVHDRT_coaleaf_undo_end", 

1571 "LVHDRT_coaleaf_stop_after_recovery", 

1572 "LVHDRT_coaleaf_finish_after_inflate", 

1573 "LVHDRT_coaleaf_finish_end", 

1574 "LVHDRT_coaleaf_delay_1", 

1575 "LVHDRT_coaleaf_delay_2", 

1576 "LVHDRT_coaleaf_delay_3", 

1577 "testsm_clone_allow_raw", 

1578 "xenrt_default_vdi_type_legacy", 

1579 "blktap_activate_inject_failure", 

1580 "blktap_activate_error_handling", 

1581 GCPAUSE_FISTPOINT, 

1582 "cleanup_coalesceVHD_inject_failure", 

1583 "cleanup_tracker_no_progress", 

1584 "FileSR_fail_hardlink", 

1585 "FileSR_fail_snap1", 

1586 "FileSR_fail_snap2", 

1587 "LVM_journaler_exists", 

1588 "LVM_journaler_none", 

1589 "LVM_journaler_badname", 

1590 "LVM_journaler_readfail", 

1591 "LVM_journaler_writefail"]) 

1592 

1593 

1594def set_dirty(session, sr): 

1595 try: 

1596 session.xenapi.SR.add_to_other_config(sr, "dirty", "") 

1597 SMlog("set_dirty %s succeeded" % (repr(sr))) 

1598 except: 

1599 SMlog("set_dirty %s failed (flag already set?)" % (repr(sr))) 

1600 

1601 

1602def doesFileHaveOpenHandles(fileName): 

1603 SMlog("Entering doesFileHaveOpenHandles with file: %s" % fileName) 

1604 (retVal, processAndPidTuples) = \ 

1605 findRunningProcessOrOpenFile(fileName, False) 

1606 

1607 if not retVal: 

1608 SMlog("Failed to determine if file %s has open handles." % \ 

1609 fileName) 

1610 # err on the side of caution 

1611 return True 

1612 else: 

1613 if len(processAndPidTuples) > 0: 

1614 return True 

1615 else: 

1616 return False 

1617 

1618 

1619# extract SR uuid from the passed in devmapper entry and return 

1620# /dev/mapper/VG_XenStorage--c3d82e92--cb25--c99b--b83a--482eebab4a93-MGT 

1621def extractSRFromDevMapper(path): 

1622 try: 

1623 path = os.path.basename(path) 

1624 path = path[len('VG_XenStorage-') + 1:] 

1625 path = path.replace('--', '/') 

1626 path = path[0:path.rfind('-')] 

1627 return path.replace('/', '-') 

1628 except: 

1629 return '' 

1630 

1631 

1632def pid_is_alive(pid): 

1633 """ 

1634 Try to kill PID with signal 0. 

1635 If we succeed, the PID is alive, so return True. 

1636 If we get an EPERM error, the PID is alive but we are not allowed to 

1637 signal it. Still return true. 

1638 Any other error (e.g. ESRCH), return False 

1639 """ 

1640 try: 

1641 os.kill(pid, 0) 

1642 return True 

1643 except OSError as e: 

1644 if e.errno == errno.EPERM: 

1645 return True 

1646 return False 

1647 

1648 

1649def get_process_cmdline(pid: int) -> List[str]: 

1650 try: 

1651 with open(os.path.join('/proc', str(pid), 'cmdline'), 'rb') as f: 

1652 line = f.read().split(b'\0') 

1653 return [arg.decode() for arg in line] 

1654 except OSError as e: 

1655 if e.errno != errno.ENOENT: 

1656 raise 

1657 return [] 

1658 

1659 

1660# Looks at /proc and figures either 

1661# If a process is still running (default), returns open file names 

1662# If any running process has open handles to the given file (process = False) 

1663# returns process names and pids 

1664def findRunningProcessOrOpenFile(name, process=True): 

1665 retVal = True 

1666 links = [] 

1667 processandpids = [] 

1668 sockets = set() 

1669 try: 

1670 SMlog("Entering findRunningProcessOrOpenFile with params: %s" % \ 

1671 [name, process]) 

1672 

1673 # Look at all pids 

1674 pids = [pid for pid in os.listdir('/proc') if pid.isdigit()] 

1675 for pid in sorted(pids): 

1676 try: 

1677 try: 

1678 f = None 

1679 f = open(os.path.join('/proc', pid, 'cmdline'), 'r') 

1680 prog = f.read()[:-1] 

1681 if prog: 1681 ↛ 1690line 1681 didn't jump to line 1690, because the condition on line 1681 was never false

1682 # Just want the process name 

1683 argv = prog.split('\x00') 

1684 prog = argv[0] 

1685 except IOError as e: 

1686 if e.errno in (errno.ENOENT, errno.ESRCH): 

1687 SMlog("ERROR %s reading %s, ignore" % (e.errno, pid)) 

1688 continue 

1689 finally: 

1690 if f is not None: 1690 ↛ 1675,   1690 ↛ 16932 missed branches: 1) line 1690 didn't jump to line 1675, because the continue on line 1688 wasn't executed, 2) line 1690 didn't jump to line 1693, because the condition on line 1690 was never false

1691 f.close() 1691 ↛ 1675line 1691 didn't jump to line 1675, because the continue on line 1688 wasn't executed

1692 

1693 try: 

1694 fd_dir = os.path.join('/proc', pid, 'fd') 

1695 files = os.listdir(fd_dir) 

1696 except OSError as e: 

1697 if e.errno in (errno.ENOENT, errno.ESRCH): 

1698 SMlog("ERROR %s reading fds for %s, ignore" % (e.errno, pid)) 

1699 # Ignore pid that are no longer valid 

1700 continue 

1701 else: 

1702 raise 

1703 

1704 for file in files: 

1705 try: 

1706 link = os.readlink(os.path.join(fd_dir, file)) 

1707 except OSError: 

1708 continue 

1709 

1710 if process: 1710 ↛ 1715line 1710 didn't jump to line 1715, because the condition on line 1710 was never false

1711 if name == prog: 1711 ↛ 1704line 1711 didn't jump to line 1704, because the condition on line 1711 was never false

1712 links.append(link) 

1713 else: 

1714 # need to return process name and pid tuples 

1715 if link == name: 

1716 processandpids.append((prog, pid)) 

1717 

1718 # Get the connected sockets 

1719 if name == prog: 

1720 sockets.update(get_connected_sockets(pid)) 

1721 

1722 # We will only have a non-empty processandpids if some fd entries were found. 

1723 # Before returning them, verify that all the PIDs in question are properly alive. 

1724 # There is no specific guarantee of when a PID's /proc directory will disappear 

1725 # when it exits, particularly relative to filedescriptor cleanup, so we want to 

1726 # make sure we're not reporting a false positive. 

1727 processandpids = [x for x in processandpids if pid_is_alive(int(x[1]))] 

1728 for pp in processandpids: 1728 ↛ 1729line 1728 didn't jump to line 1729, because the loop on line 1728 never started

1729 SMlog(f"File {name} has an open handle with process {pp[0]} with pid {pp[1]}") 

1730 

1731 except Exception as e: 

1732 SMlog("Exception checking running process or open file handles. " \ 

1733 "Error: %s" % str(e)) 

1734 retVal = False 

1735 

1736 if process: 1736 ↛ 1739line 1736 didn't jump to line 1739, because the condition on line 1736 was never false

1737 return retVal, links, sockets 

1738 else: 

1739 return retVal, processandpids 

1740 

1741 

1742def get_connected_sockets(pid): 

1743 sockets = set() 

1744 try: 

1745 # Lines in /proc/<pid>/net/unix are formatted as follows 

1746 # (see Linux source net/unix/af_unix.c, unix_seq_show() ) 

1747 # - Pointer address to socket (hex) 

1748 # - Refcount (HEX) 

1749 # - 0 

1750 # - State (HEX, 0 or __SO_ACCEPTCON) 

1751 # - Type (HEX - but only 0001 of interest) 

1752 # - Connection state (HEX - but only 03, SS_CONNECTED of interest) 

1753 # - Inode number 

1754 # - Path (optional) 

1755 open_sock_matcher = re.compile( 

1756 r'^[0-9a-f]+: [0-9A-Fa-f]+ [0-9A-Fa-f]+ [0-9A-Fa-f]+ 0001 03 \d+ (.*)$') 

1757 with open( 

1758 os.path.join('/proc', str(pid), 'net', 'unix'), 'r') as f: 

1759 lines = f.readlines() 

1760 for line in lines: 

1761 match = open_sock_matcher.match(line) 

1762 if match: 

1763 sockets.add(match[1]) 

1764 except OSError as e: 

1765 if e.errno in (errno.ENOENT, errno.ESRCH): 

1766 # Ignore pid that are no longer valid 

1767 SMlog("ERROR %s reading sockets for %s, ignore" % 

1768 (e.errno, pid)) 

1769 else: 

1770 raise 

1771 return sockets 

1772 

1773 

1774def retry(f, maxretry=20, period=3, exceptions=[Exception]): 

1775 retries = 0 

1776 while True: 

1777 try: 

1778 return f() 

1779 except Exception as e: 

1780 for exception in exceptions: 

1781 if isinstance(e, exception): 

1782 SMlog('Got exception: {}. Retry number: {}'.format( 

1783 str(e), retries 

1784 )) 

1785 break 

1786 else: 

1787 SMlog('Got bad exception: {}. Raising...'.format(e)) 

1788 raise e 

1789 

1790 retries += 1 

1791 if retries >= maxretry: 

1792 break 

1793 

1794 time.sleep(period) 

1795 

1796 return f() 

1797 

1798 

1799def getCslDevPath(svid): 

1800 basepath = "/dev/disk/by-csldev/" 

1801 if svid.startswith("NETAPP_"): 

1802 # special attention for NETAPP SVIDs 

1803 svid_parts = svid.split("__") 

1804 globstr = basepath + "NETAPP__LUN__" + "*" + svid_parts[2] + "*" + svid_parts[-1] + "*" 

1805 else: 

1806 globstr = basepath + svid + "*" 

1807 

1808 return globstr 

1809 

1810 

1811# Use device in /dev pointed to by cslg path which consists of svid 

1812def get_scsiid_from_svid(md_svid): 

1813 cslg_path = getCslDevPath(md_svid) 

1814 abs_path = glob.glob(cslg_path) 

1815 if abs_path: 

1816 real_path = os.path.realpath(abs_path[0]) 

1817 return scsiutil.getSCSIid(real_path) 

1818 else: 

1819 return None 

1820 

1821 

1822def get_isl_scsiids(session): 

1823 # Get cslg type SRs 

1824 SRs = session.xenapi.SR.get_all_records_where('field "type" = "cslg"') 

1825 

1826 # Iterate through the SR to get the scsi ids 

1827 scsi_id_ret = [] 

1828 for SR in SRs: 

1829 sr_rec = SRs[SR] 

1830 # Use the md_svid to get the scsi id 

1831 scsi_id = get_scsiid_from_svid(sr_rec['sm_config']['md_svid']) 

1832 if scsi_id: 

1833 scsi_id_ret.append(scsi_id) 

1834 

1835 # Get the vdis in the SR and do the same procedure 

1836 vdi_recs = session.xenapi.VDI.get_all_records_where('field "SR" = "%s"' % SR) 

1837 for vdi_rec in vdi_recs: 

1838 vdi_rec = vdi_recs[vdi_rec] 

1839 scsi_id = get_scsiid_from_svid(vdi_rec['sm_config']['SVID']) 

1840 if scsi_id: 

1841 scsi_id_ret.append(scsi_id) 

1842 

1843 return scsi_id_ret 

1844 

1845 

1846class extractXVA: 

1847 # streams files as a set of file and checksum, caller should remove 

1848 # the files, if not needed. The entire directory (Where the files 

1849 # and checksum) will only be deleted as part of class cleanup. 

1850 HDR_SIZE = 512 

1851 BLOCK_SIZE = 512 

1852 SIZE_LEN = 12 - 1 # To remove \0 from tail 

1853 SIZE_OFFSET = 124 

1854 ZERO_FILLED_REC = 2 

1855 NULL_IDEN = '\x00' 

1856 DIR_IDEN = '/' 

1857 CHECKSUM_IDEN = '.checksum' 

1858 OVA_FILE = 'ova.xml' 

1859 

1860 # Init gunzips the file using a subprocess, and reads stdout later 

1861 # as and when needed 

1862 def __init__(self, filename): 

1863 self.__extract_path = '' 

1864 self.__filename = filename 

1865 cmd = 'gunzip -cd %s' % filename 

1866 try: 

1867 self.spawn_p = subprocess.Popen( 

1868 cmd, shell=True, \ 

1869 stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ 

1870 stderr=subprocess.PIPE, close_fds=True) 

1871 except Exception as e: 

1872 SMlog("Error: %s. Uncompress failed for %s" % (str(e), filename)) 

1873 raise Exception(str(e)) 

1874 

1875 # Create dir to extract the files 

1876 self.__extract_path = tempfile.mkdtemp() 

1877 

1878 def __del__(self): 

1879 shutil.rmtree(self.__extract_path) 

1880 

1881 # Class supports Generator expression. 'for f_name, checksum in getTuple()' 

1882 # returns filename, checksum content. Returns filename, '' in case 

1883 # of checksum file missing. e.g. ova.xml 

1884 def getTuple(self): 

1885 zerod_record = 0 

1886 ret_f_name = '' 

1887 ret_base_f_name = '' 

1888 

1889 try: 

1890 # Read tar file as sets of file and checksum. 

1891 while True: 

1892 # Read the output of spawned process, or output of gunzip 

1893 f_hdr = self.spawn_p.stdout.read(self.HDR_SIZE) 

1894 

1895 # Break out in case of end of file 

1896 if f_hdr == '': 

1897 if zerod_record == extractXVA.ZERO_FILLED_REC: 

1898 break 

1899 else: 

1900 SMlog('Error. Expects %d zero records', \ 

1901 extractXVA.ZERO_FILLED_REC) 

1902 raise Exception('Unrecognized end of file') 

1903 

1904 # Watch out for zero records, two zero records 

1905 # denote end of file. 

1906 if f_hdr == extractXVA.NULL_IDEN * extractXVA.HDR_SIZE: 

1907 zerod_record += 1 

1908 continue 

1909 

1910 f_name = f_hdr[:f_hdr.index(extractXVA.NULL_IDEN)] 

1911 # File header may be for a folder, if so ignore the header 

1912 if not f_name.endswith(extractXVA.DIR_IDEN): 

1913 f_size_octal = f_hdr[extractXVA.SIZE_OFFSET: \ 

1914 extractXVA.SIZE_OFFSET + extractXVA.SIZE_LEN] 

1915 f_size = int(f_size_octal, 8) 

1916 if f_name.endswith(extractXVA.CHECKSUM_IDEN): 

1917 if f_name.rstrip(extractXVA.CHECKSUM_IDEN) == \ 

1918 ret_base_f_name: 

1919 checksum = self.spawn_p.stdout.read(f_size) 

1920 yield(ret_f_name, checksum) 

1921 else: 

1922 # Expects file followed by its checksum 

1923 SMlog('Error. Sequence mismatch starting with %s', \ 

1924 ret_f_name) 

1925 raise Exception( \ 

1926 'Files out of sequence starting with %s', \ 

1927 ret_f_name) 

1928 else: 

1929 # In case of ova.xml, read the contents into a file and 

1930 # return the file name to the caller. For other files, 

1931 # read the contents into a file, it will 

1932 # be used when a .checksum file is encountered. 

1933 ret_f_name = '%s/%s' % (self.__extract_path, f_name) 

1934 ret_base_f_name = f_name 

1935 

1936 # Check if the folder exists on the target location, 

1937 # else create it. 

1938 folder_path = ret_f_name[:ret_f_name.rfind('/')] 

1939 if not os.path.exists(folder_path): 

1940 os.mkdir(folder_path) 

1941 

1942 # Store the file to the tmp folder, strip the tail \0 

1943 f = open(ret_f_name, 'w') 

1944 f.write(self.spawn_p.stdout.read(f_size)) 

1945 f.close() 

1946 if f_name == extractXVA.OVA_FILE: 

1947 yield(ret_f_name, '') 

1948 

1949 # Skip zero'd portion of data block 

1950 round_off = f_size % extractXVA.BLOCK_SIZE 

1951 if round_off != 0: 

1952 zeros = self.spawn_p.stdout.read( 

1953 extractXVA.BLOCK_SIZE - round_off) 

1954 except Exception as e: 

1955 SMlog("Error: %s. File set extraction failed %s" % (str(e), \ 

1956 self.__filename)) 

1957 

1958 # Kill and Drain stdout of the gunzip process, 

1959 # else gunzip might block on stdout 

1960 os.kill(self.spawn_p.pid, signal.SIGTERM) 

1961 self.spawn_p.communicate() 

1962 raise Exception(str(e)) 

1963 

1964illegal_xml_chars = [(0x00, 0x08), (0x0B, 0x1F), (0x7F, 0x84), (0x86, 0x9F), 

1965 (0xD800, 0xDFFF), (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF), 

1966 (0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF), (0x3FFFE, 0x3FFFF), 

1967 (0x4FFFE, 0x4FFFF), (0x5FFFE, 0x5FFFF), (0x6FFFE, 0x6FFFF), 

1968 (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF), (0x9FFFE, 0x9FFFF), 

1969 (0xAFFFE, 0xAFFFF), (0xBFFFE, 0xBFFFF), (0xCFFFE, 0xCFFFF), 

1970 (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF), (0xFFFFE, 0xFFFFF), 

1971 (0x10FFFE, 0x10FFFF)] 

1972 

1973illegal_ranges = ["%s-%s" % (chr(low), chr(high)) 

1974 for (low, high) in illegal_xml_chars 

1975 if low < sys.maxunicode] 

1976 

1977illegal_xml_re = re.compile(u'[%s]' % u''.join(illegal_ranges)) 

1978 

1979 

1980def isLegalXMLString(s): 

1981 """Tells whether this is a valid XML string (i.e. it does not contain 

1982 illegal XML characters specified in 

1983 http://www.w3.org/TR/2004/REC-xml-20040204/#charsets). 

1984 """ 

1985 

1986 if len(s) > 0: 

1987 return re.search(illegal_xml_re, s) is None 

1988 else: 

1989 return True 

1990 

1991 

1992def unictrunc(string, max_bytes): 

1993 """ 

1994 Given a string, returns the largest number of elements for a prefix 

1995 substring of it, such that the UTF-8 encoding of this substring takes no 

1996 more than the given number of bytes. 

1997 

1998 The string may be given as a unicode string or a UTF-8 encoded byte 

1999 string, and the number returned will be in characters or bytes 

2000 accordingly. Note that in the latter case, the substring will still be a 

2001 valid UTF-8 encoded string (which is to say, it won't have been truncated 

2002 part way through a multibyte sequence for a unicode character). 

2003 

2004 string: the string to truncate 

2005 max_bytes: the maximum number of bytes the truncated string can be 

2006 """ 

2007 if isinstance(string, str): 

2008 return_chars = True 

2009 else: 

2010 return_chars = False 

2011 string = string.decode('UTF-8') 

2012 

2013 cur_chars = 0 

2014 cur_bytes = 0 

2015 for char in string: 

2016 charsize = len(char.encode('UTF-8')) 

2017 if cur_bytes + charsize > max_bytes: 

2018 break 

2019 else: 

2020 cur_chars += 1 

2021 cur_bytes += charsize 

2022 return cur_chars if return_chars else cur_bytes 

2023 

2024 

2025def hideValuesInPropMap(propmap, propnames): 

2026 """ 

2027 Worker function: input simple map of prop name/value pairs, and 

2028 a list of specific propnames whose values we want to hide. 

2029 Loop through the "hide" list, and if any are found, hide the 

2030 value and return the altered map. 

2031 If none found, return the original map 

2032 """ 

2033 matches = [] 

2034 for propname in propnames: 

2035 if propname in propmap: 2035 ↛ 2036line 2035 didn't jump to line 2036, because the condition on line 2035 was never true

2036 matches.append(propname) 

2037 

2038 if matches: 2038 ↛ 2039line 2038 didn't jump to line 2039, because the condition on line 2038 was never true

2039 deepCopyRec = copy.deepcopy(propmap) 

2040 for match in matches: 

2041 deepCopyRec[match] = '******' 

2042 return deepCopyRec 

2043 

2044 return propmap 

2045# define the list of propnames whose value we want to hide 

2046 

2047PASSWD_PROP_KEYS = ['password', 'cifspassword', 'chappassword', 'incoming_chappassword'] 

2048DEFAULT_SEGMENT_LEN = 950 

2049 

2050 

2051def hidePasswdInConfig(config): 

2052 """ 

2053 Function to hide passwd values in a simple prop map, 

2054 for example "device_config" 

2055 """ 

2056 return hideValuesInPropMap(config, PASSWD_PROP_KEYS) 

2057 

2058 

2059def hidePasswdInParams(params, configProp): 

2060 """ 

2061 Function to hide password values in a specified property which 

2062 is a simple map of prop name/values, and is itself an prop entry 

2063 in a larger property map. 

2064 For example, param maps containing "device_config", or 

2065 "sm_config", etc 

2066 """ 

2067 params[configProp] = hideValuesInPropMap(params[configProp], PASSWD_PROP_KEYS) 

2068 return params 

2069 

2070 

2071def hideMemberValuesInXmlParams(xmlParams, propnames=PASSWD_PROP_KEYS): 

2072 """ 

2073 Function to hide password values in XML params, specifically 

2074 for the XML format of incoming params to SR modules. 

2075 Uses text parsing: loop through the list of specific propnames 

2076 whose values we want to hide, and: 

2077 - Assemble a full "prefix" containing each property name, e.g., 

2078 "<member><name>password</name><value>" 

2079 - Test the XML if it contains that string, save the index. 

2080 - If found, get the index of the ending tag 

2081 - Truncate the return string starting with the password value. 

2082 - Append the substitute "*******" value string. 

2083 - Restore the rest of the original string starting with the end tag. 

2084 """ 

2085 findStrPrefixHead = "<member><name>" 

2086 findStrPrefixTail = "</name><value>" 

2087 findStrSuffix = "</value>" 

2088 strlen = len(xmlParams) 

2089 

2090 for propname in propnames: 

2091 findStrPrefix = findStrPrefixHead + propname + findStrPrefixTail 

2092 idx = xmlParams.find(findStrPrefix) 

2093 if idx != -1: # if found any of them 

2094 idx += len(findStrPrefix) 

2095 idx2 = xmlParams.find(findStrSuffix, idx) 

2096 if idx2 != -1: 

2097 retStr = xmlParams[0:idx] 

2098 retStr += "******" 

2099 retStr += xmlParams[idx2:strlen] 

2100 return retStr 

2101 else: 

2102 return xmlParams 

2103 return xmlParams 

2104 

2105 

2106def splitXmlText(xmlData, segmentLen=DEFAULT_SEGMENT_LEN, showContd=False): 

2107 """ 

2108 Split xml string data into substrings small enough for the 

2109 syslog line length limit. Split at tag end markers ( ">" ). 

2110 Usage: 

2111 strList = [] 

2112 strList = splitXmlText( longXmlText, maxLineLen ) # maxLineLen is optional 

2113 """ 

2114 remainingData = str(xmlData) 

2115 

2116 # "Un-pretty-print" 

2117 remainingData = remainingData.replace('\n', '') 

2118 remainingData = remainingData.replace('\t', '') 

2119 

2120 remainingChars = len(remainingData) 

2121 returnData = '' 

2122 

2123 thisLineNum = 0 

2124 while remainingChars > segmentLen: 

2125 thisLineNum = thisLineNum + 1 

2126 index = segmentLen 

2127 tmpStr = remainingData[:segmentLen] 

2128 tmpIndex = tmpStr.rfind('>') 

2129 if tmpIndex != -1: 

2130 index = tmpIndex + 1 

2131 

2132 tmpStr = tmpStr[:index] 

2133 remainingData = remainingData[index:] 

2134 remainingChars = len(remainingData) 

2135 

2136 if showContd: 

2137 if thisLineNum != 1: 

2138 tmpStr = '(Cont\'d): ' + tmpStr 

2139 tmpStr = tmpStr + ' (Cont\'d):' 

2140 

2141 returnData += tmpStr + '\n' 

2142 

2143 if showContd and thisLineNum > 0: 

2144 remainingData = '(Cont\'d): ' + remainingData 

2145 returnData += remainingData 

2146 

2147 return returnData 

2148 

2149 

2150def inject_failure(): 

2151 raise Exception('injected failure') 

2152 

2153 

2154def open_atomic(path, mode=None): 

2155 """Atomically creates a file if, and only if it does not already exist. 

2156 Leaves the file open and returns the file object. 

2157 

2158 path: the path to atomically open 

2159 mode: "r" (read), "w" (write), or "rw" (read/write) 

2160 returns: an open file object""" 

2161 

2162 assert path 

2163 

2164 flags = os.O_CREAT | os.O_EXCL 

2165 modes = {'r': os.O_RDONLY, 'w': os.O_WRONLY, 'rw': os.O_RDWR} 

2166 if mode: 

2167 if mode not in modes: 

2168 raise Exception('invalid access mode ' + mode) 

2169 flags |= modes[mode] 

2170 fd = os.open(path, flags) 

2171 try: 

2172 if mode: 

2173 return os.fdopen(fd, mode) 

2174 else: 

2175 return os.fdopen(fd) 

2176 except: 

2177 os.close(fd) 

2178 raise 

2179 

2180 

2181def isInvalidVDI(exception): 

2182 return exception.details[0] == "HANDLE_INVALID" or \ 

2183 exception.details[0] == "UUID_INVALID" 

2184 

2185 

2186def get_pool_restrictions(session): 

2187 """Returns pool restrictions as a map, @session must be already 

2188 established.""" 

2189 return list(session.xenapi.pool.get_all_records().values())[0]['restrictions'] 

2190 

2191 

2192def read_caching_is_restricted(session): 

2193 """Tells whether read caching is restricted.""" 

2194 if session is None: 2194 ↛ 2195line 2194 didn't jump to line 2195, because the condition on line 2194 was never true

2195 return True 

2196 restrictions = get_pool_restrictions(session) 

2197 if 'restrict_read_caching' in restrictions and \ 2197 ↛ 2199line 2197 didn't jump to line 2199, because the condition on line 2197 was never true

2198 restrictions['restrict_read_caching'] == "true": 

2199 return True 

2200 return False 

2201 

2202 

2203def sessions_less_than_targets(other_config, device_config): 

2204 if 'multihomelist' in device_config and 'iscsi_sessions' in other_config: 

2205 sessions = int(other_config['iscsi_sessions']) 

2206 targets = len(device_config['multihomelist'].split(',')) 

2207 SMlog("Targets %d and iscsi_sessions %d" % (targets, sessions)) 

2208 return (sessions < targets) 

2209 else: 

2210 return False 

2211 

2212 

2213def enable_and_start_service(name, start): 

2214 attempt = 0 

2215 while True: 

2216 attempt += 1 

2217 fn = 'enable' if start else 'disable' 

2218 args = ('systemctl', fn, '--now', name) 

2219 (ret, out, err) = doexec(args) 

2220 if ret == 0: 

2221 return 

2222 elif attempt >= 3: 

2223 raise Exception( 

2224 'Failed to {} {}: {} {}'.format(fn, name, out, err) 

2225 ) 

2226 time.sleep(1) 

2227 

2228 

2229def stop_service(name): 

2230 args = ('systemctl', 'stop', name) 

2231 (ret, out, err) = doexec(args) 

2232 if ret == 0: 

2233 return 

2234 raise Exception('Failed to stop {}: {} {}'.format(name, out, err)) 

2235 

2236 

2237def restart_service(name): 

2238 attempt = 0 

2239 while True: 

2240 attempt += 1 

2241 SMlog('Restarting service {} {}...'.format(name, attempt)) 

2242 args = ('systemctl', 'restart', name) 

2243 (ret, out, err) = doexec(args) 

2244 if ret == 0: 

2245 return 

2246 elif attempt >= 3: 

2247 SMlog('Restart service FAILED {} {}'.format(name, attempt)) 

2248 raise Exception( 

2249 'Failed to restart {}: {} {}'.format(name, out, err) 

2250 ) 

2251 time.sleep(1) 

2252 

2253 

2254def check_pid_exists(pid): 

2255 try: 

2256 os.kill(pid, 0) 

2257 except OSError: 

2258 return False 

2259 else: 

2260 return True 

2261 

2262 

2263def get_openers_pid(path: str) -> Optional[List[int]]: 

2264 cmd = ["lsof", "-t", path] 

2265 

2266 try: 

2267 list = [] 

2268 ret = pread2(cmd) 

2269 for line in ret.splitlines(): 

2270 list.append(int(line)) 

2271 return list 

2272 except CommandException as e: 

2273 if e.code == 1: # `lsof` return 1 if there is no openers 

2274 return None 

2275 else: 

2276 raise e 

2277 

2278 

2279def make_profile(name, function): 

2280 """ 

2281 Helper to execute cProfile using unique log file. 

2282 """ 

2283 

2284 import cProfile 

2285 import itertools 

2286 import os.path 

2287 import time 

2288 

2289 assert name 

2290 assert function 

2291 

2292 FOLDER = '/tmp/sm-perfs/' 

2293 makedirs(FOLDER) 

2294 

2295 filename = time.strftime('{}_%Y%m%d_%H%M%S.prof'.format(name)) 

2296 

2297 def gen_path(path): 

2298 yield path 

2299 root, ext = os.path.splitext(path) 

2300 for i in itertools.count(start=1, step=1): 

2301 yield root + '.{}.'.format(i) + ext 

2302 

2303 for profile_path in gen_path(FOLDER + filename): 

2304 try: 

2305 file = open_atomic(profile_path, 'w') 

2306 file.close() 

2307 break 

2308 except OSError as e: 

2309 if e.errno == errno.EEXIST: 

2310 pass 

2311 else: 

2312 raise 

2313 

2314 try: 

2315 SMlog('* Start profiling of {} ({}) *'.format(name, filename)) 

2316 cProfile.runctx('function()', None, locals(), profile_path) 

2317 finally: 

2318 SMlog('* End profiling of {} ({}) *'.format(name, filename)) 

2319 

2320 

2321def strtobool(str: str) -> bool: 

2322 # Note: `distutils` package is deprecated and slated for removal in Python 3.12. 

2323 # There is not alternative for strtobool. 

2324 # See: https://peps.python.org/pep-0632/#migration-advice 

2325 # So this is a custom implementation with differences: 

2326 # - A boolean is returned instead of integer 

2327 # - Empty string and None are supported (False is returned in this case) 

2328 if not str: 2328 ↛ 2330line 2328 didn't jump to line 2330, because the condition on line 2328 was never false

2329 return False 

2330 str = str.lower() 

2331 if str in ('y', 'yes', 't', 'true', 'on', '1'): 

2332 return True 

2333 if str in ('n', 'no', 'f', 'false', 'off', '0'): 

2334 return False 

2335 raise ValueError("invalid truth value '{}'".format(str)) 

2336 

2337 

2338def find_executable(name): 

2339 return shutil.which(name) 

2340 

2341 

2342def conditional_decorator(decorator, condition): 

2343 def wrapper(func): 

2344 if not condition: 2344 ↛ 2346line 2344 didn't jump to line 2346, because the condition on line 2344 was never false

2345 return func 

2346 return decorator(func) 

2347 return wrapper 

2348 

2349 

2350def get_srs_uuid_from_type(session, type): 

2351 srs = session.xenapi.SR.get_all_records_where(f"field \"type\" = \"{type}\"") 

2352 return {sr["uuid"]: sr for sr in srs.values()} 

2353 

2354 

2355def get_linstor_srs_uuid(session): 

2356 import LinstorSR # pylint: disable=C0415 

2357 return get_srs_uuid_from_type(session, LinstorSR.LinstorSR.DRIVER_TYPE) 

2358 

2359 

2360def find_pbd_ref_from_dconf_value(session, srs, key, value, value_modifier = None): 

2361 for sr in srs.values(): 

2362 for pbd_ref in sr["PBDs"]: 

2363 device_config = session.xenapi.PBD.get_device_config(pbd_ref) 

2364 cur_value = device_config.get(key) 

2365 if value_modifier: 

2366 cur_value = value_modifier(cur_value) 

2367 if cur_value and cur_value == value: 

2368 return pbd_ref 

2369 return None