Coverage for hdl_registers/parser/parser.py: 97%

1# --------------------------------------------------------------------------------------------------

4# This file is part of the hdl-registers project, an HDL register generator fast enough to run

5# in real time.

6# https://hdl-registers.com

7# https://github.com/hdl-registers/hdl-registers

8# --------------------------------------------------------------------------------------------------

10# Standard libraries

11import copy

12import json

13from pathlib import Path

14from typing import TYPE_CHECKING, Any, Optional

16# Third party libraries

17import tomli_w

18import yaml

19from tsfpga import DEFAULT_FILE_ENCODING

21# First party libraries

22from hdl_registers.about import WEBSITE_URL

23from hdl_registers.constant.bit_vector_constant import UnsignedVector

24from hdl_registers.register_list import RegisterList

25from hdl_registers.register_modes import REGISTER_MODES

27if TYPE_CHECKING:

28 # First party libraries

29 from hdl_registers.register import Register

30 from hdl_registers.register_mode import RegisterMode

33class RegisterParser:

34 """

35 Parse register data in the form of a dictionary into a :class:`.RegisterList` object.

36 See :ref:`toml_format` for further documentation.

38 A note on sanity check strategy:

39 The parser performs only the basic sanity checks related to the data file format.

40 For example, missing properties, unknown properties, etc.

41 A lot of other sanity checks are performed in the :class:`.Register`, :class:`.RegisterArray`,

42 :class:`.RegisterField`, etc, classes themselves.

44 For example, the default value of a bit field should be a string with the value "0" or "1".

45 This is checked in the constructor of the :class:`.Bit` class, not here in the parser.

46 Similar for a lot of other things.

48 This is because these objects can be created from the Python API also, without involving

49 the parser.

50 Hence these sanity checks have to be present there.

51 Having them also in the parser would enable better error messages, but would be redundant

52 and would slow down the parser.

53 Since the parser is run in real time, the performance is critical, and we can not afford

54 to slow it down.

55 """

57 # Attributes of the constant.

58 recognized_constant_items = {"type", "value", "description", "data_type"}

59 # Note that "type" being present is implied. We would not be parsing a constant unless we

60 # know it to be a "constant" type.

61 # So we save some CPU cycles by not checking for it.

62 required_constant_items = ["value"]

64 # Attributes of the register.

65 # Anything apart from these are names of fields.

66 default_register_items = {

67 "type",

68 "mode",

69 "description",

70 }

71 # While a 'mode' is required for a register, it may NOT be specified/changed in the data file

72 # for a default register.

73 # Hence this property is handled separately.

74 # And hence, registers have no required items.

76 # Attributes of the register array.

77 # Anything apart from these are names of registers.

78 default_register_array_items = {"type", "array_length", "description"}

79 # Note that "type" being present is implied.

80 # We would not be parsing a register array unless we know it to be a "register_array" type.

81 # So we save some CPU cycles by not checking for it.

82 required_register_array_items = ["array_length"]

84 # Attributes of the "bit" register field.

85 recognized_bit_items = {"type", "description", "default_value"}

86 # Note that "type" being present is implied.

87 # We would not be parsing a bit unless we know it to be a "bit" type.

88 # So we save some CPU cycles by not checking for it.

89 required_bit_items: list[str] = []

91 # Attributes of the "bit_vector" register field.

92 recognized_bit_vector_items = {"type", "description", "width", "default_value"}

93 # Note that "type" being present is implied.

94 # We would not be parsing a bit_vector unless we know it to be a "bit_vector" type.

95 # So we save some CPU cycles by not checking for it.

96 required_bit_vector_items = ["width"]

98 # Attributes of the "enumeration" register field.

99 recognized_enumeration_items = {"type", "description", "default_value", "element"}

100 # Note that "type" being present is implied.

101 # We would not be parsing a enumeration unless we know it to be a "enumeration" type.

102 # So we save some CPU cycles by not checking for it.

103 required_enumeration_items = ["element"]

104

105 # Attributes of the "integer" register field.

106 recognized_integer_items = {"type", "description", "min_value", "max_value", "default_value"}

107 # Note that "type" being present is implied.

108 # We would not be parsing a integer unless we know it to be a "integer" type.

109 # So we save some CPU cycles by not checking for it.

110 required_integer_items = ["max_value"]

111

112 def __init__(

113 self,

114 name: str,

115 source_definition_file: Path,

116 default_registers: Optional[list["Register"]] = None,

117 ):

118 """

119 Arguments:

120 name: The name of the register list.

121 source_definition_file: The source file that defined this register list.

122 Will be displayed in generated source code and documentation

123 for traceability.

124 default_registers: List of default registers.

125 Note that this list with :class:`.Register` objects will be deep copied, so you can

126 use the same list many times without worrying about mutability.

127 """

128 self._register_list = RegisterList(name=name, source_definition_file=source_definition_file)

129 self._source_definition_file = source_definition_file

130

131 self._default_register_names = []

132 if default_registers:

133 # Perform deep copy of the mutable register objects.

134 # Ignore a mypy error that seems buggy.

135 # We are assigning list[Register] to list[Register | RegisterArray]

136 # which should be absolutely fine, but mypy warns.

137 self._register_list.register_objects = copy.deepcopy(

138 default_registers # type: ignore[arg-type]

139 )

140 for register in default_registers:

141 self._default_register_names.append(register.name)

142

143 def parse(self, register_data: dict[str, Any]) -> RegisterList:

144 """

145 Parse the register data.

146

147 Arguments:

148 register_data: Register data as a dictionary.

149 Preferably read by the :func:`.from_toml`, :func:`.from_json` or

150 :func:`.from_yaml` functions.

151

152 Return:

153 The resulting register list.

154 """

155 for old_top_level_key_name in ["constant", "register", "register_array"]:

156 if old_top_level_key_name in register_data:

157 source_file = self._source_definition_file

158 output_file = (

159 source_file.parent.resolve()

160 / f"{source_file.stem}_version_6_format{source_file.suffix}"

161 )

162

163 print(

164 f"""

165ERROR: Parsing register data that appears to be in the old pre-6.0.0 format.

166ERROR: For more information, see: {WEBSITE_URL}/rst/basic_feature/basic_feature_register_modes.html

167ERROR: Your data will be automatically converted to the new format and saved to: {output_file}

168ERROR: Please inspect that file and update your data file to the new format.

169"""

170 )

171 _save_to_new_format(old_data=register_data, output_file=output_file)

172 raise ValueError("Found register data in old format. See message above.")

173

174 parser_methods = {

175 "constant": self._parse_constant,

176 "register": self._parse_plain_register,

177 "register_array": self._parse_register_array,

178 }

179

180 for top_level_name, top_level_items in register_data.items():

181 if not isinstance(top_level_items, dict):

182 message = (

183 f"Error while parsing {self._source_definition_file}: "

184 f'Got unknown top-level property "{top_level_name}".'

185 )

186 raise ValueError(message)

187

188 top_level_type = top_level_items.get("type", "register")

189

190 if top_level_type not in parser_methods:

191 valid_types_str = ", ".join(f'"{parser_key}"' for parser_key in parser_methods)

192 message = (

193 f'Error while parsing "{top_level_name}" in {self._source_definition_file}: '

194 f'Got unknown type "{top_level_type}". Expected one of {valid_types_str}.'

195 )

196 raise ValueError(message)

197

198 parser_methods[top_level_type](name=top_level_name, items=top_level_items)

199

200 return self._register_list

201

202 def _parse_constant(self, name: str, items: dict[str, Any]) -> None:

203 for item_name in self.required_constant_items:

204 if item_name not in items:

205 message = (

206 f'Error while parsing constant "{name}" in {self._source_definition_file}: '

207 f'Missing required property "{item_name}".'

208 )

209 raise ValueError(message)

210

211 for item_name in items.keys():

212 if item_name not in self.recognized_constant_items:

213 message = (

214 f'Error while parsing constant "{name}" in {self._source_definition_file}: '

215 f'Got unknown property "{item_name}".'

216 )

217 raise ValueError(message)

218

219 value = items["value"]

220 description = items.get("description", "")

221 data_type_str = items.get("data_type")

222

223 if data_type_str is not None:

224 if not isinstance(value, str):

225 raise ValueError(

226 f'Error while parsing constant "{name}" in '

227 f"{self._source_definition_file}: "

228 'May not set "data_type" for non-string constant.'

229 )

230

231 if data_type_str == "unsigned":

232 value = UnsignedVector(value)

233 else:

234 raise ValueError(

235 f'Error while parsing constant "{name}" in '

236 f"{self._source_definition_file}: "

237 f'Invalid data type "{data_type_str}".'

238 )

239

240 self._register_list.add_constant(name=name, value=value, description=description)

241

242 def _parse_plain_register(self, name: str, items: dict[str, Any]) -> None:

243 description = items.get("description", "")

244

245 if name in self._default_register_names:

246 # Default registers can be "updated" in the sense that the user can set a custom

247 # 'description' and add whatever fields they want in the current register list.

248 # They may not, however, change the 'mode' which is part of the default definition.

249 if "mode" in items:

250 message = (

251 f'Error while parsing register "{name}" in {self._source_definition_file}: '

252 'A "mode" may not be specified for a default register.'

253 )

254 raise ValueError(message)

255

256 register = self._register_list.get_register(register_name=name)

257 register.description = description

258

259 else:

260 # If it is a new register however, the 'mode' has to be specified.

261 if "mode" not in items:

262 message = (

263 f'Error while parsing register "{name}" in {self._source_definition_file}: '

264 f'Missing required property "mode".'

265 )

266 raise ValueError(message)

267

268 mode = self._get_mode(mode_name=items["mode"], register_name=name)

269

270 register = self._register_list.append_register(

271 name=name, mode=mode, description=description

272 )

273

274 self._parse_register_fields(register=register, register_items=items, register_array_note="")

275

276 def _get_mode(self, mode_name: str, register_name: str) -> "RegisterMode":

277 if mode_name in REGISTER_MODES:

278 return REGISTER_MODES[mode_name]

279

280 valid_modes_str = ", ".join(f'"{mode_key}"' for mode_key in REGISTER_MODES)

281 message = (

282 f'Error while parsing register "{register_name}" in {self._source_definition_file}: '

283 f'Got unknown mode "{mode_name}". Expected one of {valid_modes_str}.'

284 )

285 raise ValueError(message)

286

287 def _parse_register_fields(

288 self,

289 register_items: dict[str, Any],

290 register: "Register",

291 register_array_note: str,

292 ) -> None:

293 # Add any fields that are specified.

294 for item_name, item_value in register_items.items():

295 # Skip default items so we only get the fields.

296 if item_name in self.default_register_items:

297 continue

298

299 if not isinstance(item_value, dict):

300 message = (

301 f'Error while parsing register "{register.name}"{register_array_note} '

302 f"in {self._source_definition_file}: "

303 f'Got unknown property "{item_name}".'

304 )

305 raise ValueError(message)

306

307 if "type" not in item_value:

308 message = (

309 f'Error while parsing field "{item_name}" in register '

310 f'"{register.name}"{register_array_note} in {self._source_definition_file}: '

311 'Missing required property "type".'

312 )

313 raise ValueError(message)

314

315 field_type = item_value["type"]

316

317 parser_methods = {

318 "bit": self._parse_bit,

319 "bit_vector": self._parse_bit_vector,

320 "enumeration": self._parse_enumeration,

321 "integer": self._parse_integer,

322 }

323

324 if field_type not in parser_methods:

325 valid_types_str = ", ".join(f'"{parser_key}"' for parser_key in parser_methods)

326 message = (

327 f'Error while parsing field "{item_name}" in register '

328 f'"{register.name}"{register_array_note} in {self._source_definition_file}: '

329 f'Unknown field type "{field_type}". Expected one of {valid_types_str}.'

330 )

331 raise ValueError(message)

332

333 parser_methods[field_type](

334 register=register, field_name=item_name, field_items=item_value

335 )

336

337 def _parse_register_array(self, name: str, items: dict[str, Any]) -> None:

338 for required_property in self.required_register_array_items:

339 if required_property not in items:

340 message = (

341 f'Error while parsing register array "{name}" in '

342 f"{self._source_definition_file}: "

343 f'Missing required property "{required_property}".'

344 )

345 raise ValueError(message)

346

347 register_array_length = items["array_length"]

348 register_array_description = items.get("description", "")

349 register_array = self._register_list.append_register_array(

350 name=name, length=register_array_length, description=register_array_description

351 )

352

353 # Add all registers that are specified.

354 found_at_least_one_register = False

355 for item_name, item_value in items.items():

356 # Skip default items so we only get the registers.

357 if item_name in self.default_register_array_items:

358 continue

359

360 found_at_least_one_register = True

361

362 if not isinstance(item_value, dict):

363 message = (

364 f'Error while parsing register array "{name}" in '

365 f"{self._source_definition_file}: "

366 f'Got unknown property "{item_name}".'

367 )

368 raise ValueError(message)

369

370 item_type = item_value.get("type", "register")

371 if item_type != "register":

372 message = (

373 f'Error while parsing register "{item_name}" within array "{name}" in '

374 f"{self._source_definition_file}: "

375 f'Got unknown type "{item_type}". Expected "register".'

376 )

377 raise ValueError(message)

378

379 # A 'mode' is semi-required for plain registers, but always required for

380 # array registers.

381 if "mode" not in item_value:

382 raise ValueError(

383 f'Error while parsing register "{item_name}" within array "{name}" in '

384 f"{self._source_definition_file}: "

385 f'Missing required property "mode".'

386 )

387 register_mode = self._get_mode(mode_name=item_value["mode"], register_name=item_name)

388

389 register_description = item_value.get("description", "")

390

391 register = register_array.append_register(

392 name=item_name, mode=register_mode, description=register_description

393 )

394

395 self._parse_register_fields(

396 register_items=item_value,

397 register=register,

398 register_array_note=f' within array "{name}"',

399 )

400

401 if not found_at_least_one_register:

402 message = (

403 f'Error while parsing register array "{name}" in {self._source_definition_file}: '

404 "Array must contain at least one register."

405 )

406 raise ValueError(message)

407

408 def _check_field_items(

409 self,

410 register_name: str,

411 field_name: str,

412 field_items: dict[str, Any],

413 recognized_items: set[str],

414 required_items: list[str],

415 ) -> None:

416 """

417 Will raise exception if anything is wrong.

418 """

419 for item_name in required_items:

420 if item_name not in field_items:

421 message = (

422 f'Error while parsing field "{field_name}" in register "{register_name}" in '

423 f"{self._source_definition_file}: "

424 f'Missing required property "{item_name}".'

425 )

426 raise ValueError(message)

427

428 for item_name in field_items.keys():

429 if item_name not in recognized_items:

430 message = (

431 f'Error while parsing field "{field_name}" in register '

432 f'"{register_name}" in {self._source_definition_file}: '

433 f'Unknown property "{item_name}".'

434 )

435 raise ValueError(message)

436

437 def _parse_bit(

438 self, register: "Register", field_name: str, field_items: dict[str, Any]

439 ) -> None:

440 self._check_field_items(

441 register_name=register.name,

442 field_name=field_name,

443 field_items=field_items,

444 recognized_items=self.recognized_bit_items,

445 required_items=self.required_bit_items,

446 )

447

448 description = field_items.get("description", "")

449 default_value = field_items.get("default_value", "0")

450

451 register.append_bit(name=field_name, description=description, default_value=default_value)

452

453 def _parse_bit_vector(

454 self, register: "Register", field_name: str, field_items: dict[str, Any]

455 ) -> None:

456 self._check_field_items(

457 register_name=register.name,

458 field_name=field_name,

459 field_items=field_items,

460 recognized_items=self.recognized_bit_vector_items,

461 required_items=self.required_bit_vector_items,

462 )

463

464 width = field_items["width"]

465

466 description = field_items.get("description", "")

467 default_value = field_items.get("default_value", "0" * width)

468

469 register.append_bit_vector(

470 name=field_name, description=description, width=width, default_value=default_value

471 )

472

473 def _parse_enumeration(

474 self, register: "Register", field_name: str, field_items: dict[str, Any]

475 ) -> None:

476 self._check_field_items(

477 register_name=register.name,

478 field_name=field_name,

479 field_items=field_items,

480 recognized_items=self.recognized_enumeration_items,

481 # Check that we have at least one element.

482 # This is checked also in the Enumeration class, which is needed if the user

483 # is working directly with the Python API.

484 # That is where we usually sanity check, to avoid duplication.

485 # However, this particular check is needed here also since the logic for default

486 # value below does not work if there are no elements.

487 required_items=self.required_enumeration_items,

488 )

489

490 description = field_items.get("description", "")

491 # We assert above that the enumeration has at least one element.

492 # Meaning that the result of this get can not be None, as mypy thinks.

493 elements: dict[str, str] = field_items.get("element") # type: ignore[assignment]

494

495 # The default "default value" is the first declared enumeration element.

496 # Note that this works because dictionaries in Python are guaranteed ordered since

497 # Python 3.7.

498 default_value = field_items.get("default_value", list(elements)[0])

499

500 register.append_enumeration(

501 name=field_name,

502 description=description,

503 elements=elements,

504 default_value=default_value,

505 )

506

507 def _parse_integer(

508 self, register: "Register", field_name: str, field_items: dict[str, Any]

509 ) -> None:

510 self._check_field_items(

511 register_name=register.name,

512 field_name=field_name,

513 field_items=field_items,

514 recognized_items=self.recognized_integer_items,

515 required_items=self.required_integer_items,

516 )

517

518 max_value = field_items["max_value"]

519

520 description = field_items.get("description", "")

521 min_value = field_items.get("min_value", 0)

522 default_value = field_items.get("default_value", min_value)

523

524 register.append_integer(

525 name=field_name,

526 description=description,

527 min_value=min_value,

528 max_value=max_value,

529 default_value=default_value,

530 )

531

532

533def _convert_to_new_format( # pylint: disable=too-many-locals

534 old_data: dict[str, Any]

535) -> dict[str, Any]:

536 """

537 Convert pre-6.0.0 format to the new format.

538 """

539

540 def _get_register_dict(register_items: dict[str, Any]) -> dict[str, Any]:

541 register_dict = dict()

542

543 for register_item_name, register_item_value in register_items.items():

544 if register_item_name in RegisterParser.default_register_items:

545 register_dict[register_item_name] = register_item_value

546

547 elif register_item_name in ["bit", "bit_vector", "enumeration", "integer"]:

548 for field_name, field_items in register_item_value.items():

549 field_dict = dict(type=register_item_name)

550

551 for field_item_name, field_item_value in field_items.items():

552 field_dict[field_item_name] = field_item_value

553

554 register_dict[field_name] = field_dict

555

556 else:

557 raise ValueError(

558 f"Unknown item {register_item_name}. "

559 "Looks like an error in the user data file."

560 )

561

562 return register_dict

563

564 result = dict()

565

566 def _add_item(name: str, items: dict[str, Any]) -> None:

567 if name in result:

568 raise ValueError(f"Duplicate item {name}")

569

570 result[name] = items

571

572 if "register" in old_data:

573 for register_name, register_items in old_data["register"].items():

574 register_dict = _get_register_dict(register_items=register_items)

575 _add_item(name=register_name, items=register_dict)

576

577 if "register_array" in old_data:

578 for register_array_name, register_array_items in old_data["register_array"].items():

579 register_array_dict: dict[str, Any] = dict(type="register_array")

580

581 for register_array_item_name, register_array_item_value in register_array_items.items():

582 if register_array_item_name in RegisterParser.default_register_array_items:

583 register_array_dict[register_array_item_name] = register_array_item_value

584

585 elif register_array_item_name == "register":

586 for register_name, register_items in register_array_item_value.items():

587 register_array_dict[register_name] = _get_register_dict(

588 register_items=register_items

589 )

590

591 else:

592 raise ValueError(

593 f"Unknown item {register_array_item_name}. "

594 "Looks like an error in the user data file."

595 )

596

597 _add_item(name=register_array_name, items=register_array_dict)

598

599 if "constant" in old_data:

600 for constant_name, constant_items in old_data["constant"].items():

601 constant_dict = dict(type="constant")

602

603 for constant_item_name, constant_item_value in constant_items.items():

604 constant_dict[constant_item_name] = constant_item_value

605

606 _add_item(name=constant_name, items=constant_dict)

607

608 return result

609

610

611def _save_to_new_format(old_data: dict[str, Any], output_file: Path) -> None:

612 """

613 Save the old data to the new format.

614 """

615 new_data = _convert_to_new_format(old_data=old_data)

616

617 if output_file.suffix == ".toml":

618 with open(output_file, "wb") as file_handle:

619 tomli_w.dump(new_data, file_handle, multiline_strings=True)

620

621 return

622

623 if output_file.suffix == ".json":

624 with open(output_file, "w", encoding=DEFAULT_FILE_ENCODING) as file_handle:

625 json.dump(new_data, file_handle, indent=4)

626

627 return

628

629 if output_file.suffix == ".yaml":

630 with open(output_file, "w", encoding=DEFAULT_FILE_ENCODING) as file_handle:

631 yaml.dump(new_data, file_handle)

632

633 return

634

635 raise ValueError(f"Unknown file format {output_file}")