Coverage for hdl_registers/parser/parser.py: 97%

1# --------------------------------------------------------------------------------------------------

4# This file is part of the hdl-registers project, an HDL register generator fast enough to run

5# in real time.

6# https://hdl-registers.com

7# https://github.com/hdl-registers/hdl-registers

8# --------------------------------------------------------------------------------------------------

10# ruff: noqa: A005

12from __future__ import annotations

14import copy

15import json

16from typing import TYPE_CHECKING, Any, ClassVar

18import tomli_w

19import yaml

20from tsfpga import DEFAULT_FILE_ENCODING

22from hdl_registers.about import WEBSITE_URL

23from hdl_registers.constant.bit_vector_constant import UnsignedVector

24from hdl_registers.register_list import RegisterList

25from hdl_registers.register_modes import REGISTER_MODES

27if TYPE_CHECKING:

28 from pathlib import Path

30 from hdl_registers.register import Register

31 from hdl_registers.register_mode import RegisterMode

34class RegisterParser:

35 """

36 Parse register data in the form of a dictionary into a :class:`.RegisterList` object.

37 See :ref:`toml_format` for further documentation.

39 A note on sanity check strategy:

40 The parser performs only the basic sanity checks related to the data file format.

41 For example, missing properties, unknown properties, etc.

42 A lot of other sanity checks are performed in the :class:`.Register`, :class:`.RegisterArray`,

43 :class:`.RegisterField`, etc, classes themselves.

45 For example, the default value of a bit field should be a string with the value "0" or "1".

46 This is checked in the constructor of the :class:`.Bit` class, not here in the parser.

47 Similar for a lot of other things.

49 This is because these objects can be created from the Python API also, without involving

50 the parser.

51 Hence these sanity checks have to be present there.

52 Having them also in the parser would enable better error messages, but would be redundant

53 and would slow down the parser.

54 Since the parser is run in real time, the performance is critical, and we can not afford

55 to slow it down.

56 """

58 # Attributes of the constant.

59 recognized_constant_items: ClassVar = {"type", "value", "description", "data_type"}

60 # Note that "type" being present is implied. We would not be parsing a constant unless we

61 # know it to be a "constant" type.

62 # So we save some CPU cycles by not checking for it.

63 required_constant_items: ClassVar = ["value"]

65 # Attributes of the register.

66 # Anything apart from these are names of fields.

67 default_register_items: ClassVar = {

68 "type",

69 "mode",

70 "description",

71 }

72 # While a 'mode' is required for a register, it may NOT be specified/changed in the data file

73 # for a default register.

74 # Hence this property is handled separately.

75 # And hence, registers have no required items.

77 # Attributes of the register array.

78 # Anything apart from these are names of registers.

79 default_register_array_items: ClassVar = {"type", "array_length", "description"}

80 # Note that "type" being present is implied.

81 # We would not be parsing a register array unless we know it to be a "register_array" type.

82 # So we save some CPU cycles by not checking for it.

83 required_register_array_items: ClassVar = ["array_length"]

85 # Attributes of the "bit" register field.

86 recognized_bit_items: ClassVar = {"type", "description", "default_value"}

87 # Note that "type" being present is implied.

88 # We would not be parsing a bit unless we know it to be a "bit" type.

89 # So we save some CPU cycles by not checking for it.

90 required_bit_items: ClassVar[list[str]] = []

92 # Attributes of the "bit_vector" register field.

93 recognized_bit_vector_items: ClassVar = {"type", "description", "width", "default_value"}

94 # Note that "type" being present is implied.

95 # We would not be parsing a bit_vector unless we know it to be a "bit_vector" type.

96 # So we save some CPU cycles by not checking for it.

97 required_bit_vector_items: ClassVar = ["width"]

99 # Attributes of the "enumeration" register field.

100 recognized_enumeration_items: ClassVar = {"type", "description", "default_value", "element"}

101 # Note that "type" being present is implied.

102 # We would not be parsing a enumeration unless we know it to be a "enumeration" type.

103 # So we save some CPU cycles by not checking for it.

104 required_enumeration_items: ClassVar = ["element"]

105

106 # Attributes of the "integer" register field.

107 recognized_integer_items: ClassVar = {

108 "type",

109 "description",

110 "min_value",

111 "max_value",

112 "default_value",

113 }

114 # Note that "type" being present is implied.

115 # We would not be parsing a integer unless we know it to be a "integer" type.

116 # So we save some CPU cycles by not checking for it.

117 required_integer_items: ClassVar = ["max_value"]

118

119 def __init__(

120 self,

121 name: str,

122 source_definition_file: Path,

123 default_registers: list[Register] | None = None,

124 ) -> None:

125 """

126 Arguments:

127 name: The name of the register list.

128 source_definition_file: The source file that defined this register list.

129 Will be displayed in generated source code and documentation

130 for traceability.

131 default_registers: List of default registers.

132 Note that this list with :class:`.Register` objects will be deep copied, so you can

133 use the same list many times without worrying about mutability.

134 """

135 self._register_list = RegisterList(name=name, source_definition_file=source_definition_file)

136 self._source_definition_file = source_definition_file

137

138 self._default_register_names = []

139 if default_registers:

140 # Perform deep copy of the mutable register objects.

141 self._register_list.register_objects = copy.deepcopy(default_registers)

142 for register in default_registers:

143 self._default_register_names.append(register.name)

144

145 def parse(self, register_data: dict[str, Any]) -> RegisterList:

146 """

147 Parse the register data.

148

149 Arguments:

150 register_data: Register data as a dictionary.

151 Preferably read by the :func:`.from_toml`, :func:`.from_json` or

152 :func:`.from_yaml` functions.

153

154 Return:

155 The resulting register list.

156 """

157 for old_top_level_key_name in ["constant", "register", "register_array"]:

158 if old_top_level_key_name in register_data:

159 source_file = self._source_definition_file

160 output_file = (

161 source_file.parent.resolve()

162 / f"{source_file.stem}_version_6_format{source_file.suffix}"

163 )

164

165 print(

166 f"""

167ERROR: Parsing register data that appears to be in the old pre-6.0.0 format.

168ERROR: For more information, see: {WEBSITE_URL}/rst/about/new_data_file_format.html

169ERROR: Your data will be automatically converted to the new format and saved to: {output_file}

170ERROR: Please inspect that file and update your data file to the new format.

171"""

172 )

173 _save_to_new_format(old_data=register_data, output_file=output_file)

174 raise ValueError("Found register data in old format. See message above.")

175

176 parser_methods = {

177 "constant": self._parse_constant,

178 "register": self._parse_plain_register,

179 "register_array": self._parse_register_array,

180 }

181

182 for top_level_name, top_level_items in register_data.items():

183 if not isinstance(top_level_items, dict):

184 message = (

185 f"Error while parsing {self._source_definition_file}: "

186 f'Got unknown top-level property "{top_level_name}".'

187 )

188 # Seems to the linter like a type error, but it is actually the user specifying

189 # a property/value that they shouldn't.

190 # Corresponds better to a 'ValueError' than a 'TypeError'.

191 raise ValueError(message) # noqa: TRY004

192

193 top_level_type = top_level_items.get("type", "register")

194

195 if top_level_type not in parser_methods:

196 valid_types_str = ", ".join(f'"{parser_key}"' for parser_key in parser_methods)

197 message = (

198 f'Error while parsing "{top_level_name}" in {self._source_definition_file}: '

199 f'Got unknown type "{top_level_type}". Expected one of {valid_types_str}.'

200 )

201 raise ValueError(message)

202

203 parser_methods[top_level_type](name=top_level_name, items=top_level_items)

204

205 return self._register_list

206

207 def _parse_constant(self, name: str, items: dict[str, Any]) -> None:

208 for item_name in self.required_constant_items:

209 if item_name not in items:

210 message = (

211 f'Error while parsing constant "{name}" in {self._source_definition_file}: '

212 f'Missing required property "{item_name}".'

213 )

214 raise ValueError(message)

215

216 for item_name in items:

217 if item_name not in self.recognized_constant_items:

218 message = (

219 f'Error while parsing constant "{name}" in {self._source_definition_file}: '

220 f'Got unknown property "{item_name}".'

221 )

222 raise ValueError(message)

223

224 value = items["value"]

225 description = items.get("description", "")

226 data_type_str = items.get("data_type")

227

228 if data_type_str is not None:

229 if not isinstance(value, str):

230 raise ValueError(

231 f'Error while parsing constant "{name}" in '

232 f"{self._source_definition_file}: "

233 'May not set "data_type" for non-string constant.'

234 )

235

236 if data_type_str == "unsigned":

237 value = UnsignedVector(value)

238 else:

239 raise ValueError(

240 f'Error while parsing constant "{name}" in '

241 f"{self._source_definition_file}: "

242 f'Invalid data type "{data_type_str}".'

243 )

244

245 self._register_list.add_constant(name=name, value=value, description=description)

246

247 def _parse_plain_register(self, name: str, items: dict[str, Any]) -> None:

248 description = items.get("description", "")

249

250 if name in self._default_register_names:

251 # Default registers can be "updated" in the sense that the user can set a custom

252 # 'description' and add whatever fields they want in the current register list.

253 # They may not, however, change the 'mode' which is part of the default definition.

254 if "mode" in items:

255 message = (

256 f'Error while parsing register "{name}" in {self._source_definition_file}: '

257 'A "mode" may not be specified for a default register.'

258 )

259 raise ValueError(message)

260

261 register = self._register_list.get_register(register_name=name)

262 register.description = description

263

264 else:

265 # If it is a new register however, the 'mode' has to be specified.

266 if "mode" not in items:

267 message = (

268 f'Error while parsing register "{name}" in {self._source_definition_file}: '

269 f'Missing required property "mode".'

270 )

271 raise ValueError(message)

272

273 mode = self._get_mode(mode_name=items["mode"], register_name=name)

274

275 register = self._register_list.append_register(

276 name=name, mode=mode, description=description

277 )

278

279 self._parse_register_fields(register=register, register_items=items, register_array_note="")

280

281 def _get_mode(self, mode_name: str, register_name: str) -> RegisterMode:

282 if mode_name in REGISTER_MODES:

283 return REGISTER_MODES[mode_name]

284

285 valid_modes_str = ", ".join(f'"{mode_key}"' for mode_key in REGISTER_MODES)

286 message = (

287 f'Error while parsing register "{register_name}" in {self._source_definition_file}: '

288 f'Got unknown mode "{mode_name}". Expected one of {valid_modes_str}.'

289 )

290 raise ValueError(message)

291

292 def _parse_register_fields(

293 self,

294 register_items: dict[str, Any],

295 register: Register,

296 register_array_note: str,

297 ) -> None:

298 # Add any fields that are specified.

299 for item_name, item_value in register_items.items():

300 # Skip default items so we only get the fields.

301 if item_name in self.default_register_items:

302 continue

303

304 if not isinstance(item_value, dict):

305 message = (

306 f'Error while parsing register "{register.name}"{register_array_note} '

307 f"in {self._source_definition_file}: "

308 f'Got unknown property "{item_name}".'

309 )

310 # Seems to the linter like a type error, but it is actually the user specifying

311 # a property/value that they shouldn't.

312 # Corresponds better to a 'ValueError' than a 'TypeError'.

313 raise ValueError(message) # noqa: TRY004

314

315 if "type" not in item_value:

316 message = (

317 f'Error while parsing field "{item_name}" in register '

318 f'"{register.name}"{register_array_note} in {self._source_definition_file}: '

319 'Missing required property "type".'

320 )

321 raise ValueError(message)

322

323 field_type = item_value["type"]

324

325 parser_methods = {

326 "bit": self._parse_bit,

327 "bit_vector": self._parse_bit_vector,

328 "enumeration": self._parse_enumeration,

329 "integer": self._parse_integer,

330 }

331

332 if field_type not in parser_methods:

333 valid_types_str = ", ".join(f'"{parser_key}"' for parser_key in parser_methods)

334 message = (

335 f'Error while parsing field "{item_name}" in register '

336 f'"{register.name}"{register_array_note} in {self._source_definition_file}: '

337 f'Unknown field type "{field_type}". Expected one of {valid_types_str}.'

338 )

339 raise ValueError(message)

340

341 parser_methods[field_type](

342 register=register, field_name=item_name, field_items=item_value

343 )

344

345 def _parse_register_array(self, name: str, items: dict[str, Any]) -> None:

346 for required_property in self.required_register_array_items:

347 if required_property not in items:

348 message = (

349 f'Error while parsing register array "{name}" in '

350 f"{self._source_definition_file}: "

351 f'Missing required property "{required_property}".'

352 )

353 raise ValueError(message)

354

355 register_array_length = items["array_length"]

356 register_array_description = items.get("description", "")

357 register_array = self._register_list.append_register_array(

358 name=name, length=register_array_length, description=register_array_description

359 )

360

361 # Add all registers that are specified.

362 found_at_least_one_register = False

363 for item_name, item_value in items.items():

364 # Skip default items so we only get the registers.

365 if item_name in self.default_register_array_items:

366 continue

367

368 found_at_least_one_register = True

369

370 if not isinstance(item_value, dict):

371 message = (

372 f'Error while parsing register array "{name}" in '

373 f"{self._source_definition_file}: "

374 f'Got unknown property "{item_name}".'

375 )

376 # Seems to the linter like a type error, but it is actually the user specifying

377 # a property/value that they shouldn't.

378 # Corresponds better to a 'ValueError' than a 'TypeError'.

379 raise ValueError(message) # noqa: TRY004

380

381 item_type = item_value.get("type", "register")

382 if item_type != "register":

383 message = (

384 f'Error while parsing register "{item_name}" within array "{name}" in '

385 f"{self._source_definition_file}: "

386 f'Got unknown type "{item_type}". Expected "register".'

387 )

388 raise ValueError(message)

389

390 # A 'mode' is semi-required for plain registers, but always required for

391 # array registers.

392 if "mode" not in item_value:

393 raise ValueError(

394 f'Error while parsing register "{item_name}" within array "{name}" in '

395 f"{self._source_definition_file}: "

396 f'Missing required property "mode".'

397 )

398 register_mode = self._get_mode(mode_name=item_value["mode"], register_name=item_name)

399

400 register_description = item_value.get("description", "")

401

402 register = register_array.append_register(

403 name=item_name, mode=register_mode, description=register_description

404 )

405

406 self._parse_register_fields(

407 register_items=item_value,

408 register=register,

409 register_array_note=f' within array "{name}"',

410 )

411

412 if not found_at_least_one_register:

413 message = (

414 f'Error while parsing register array "{name}" in {self._source_definition_file}: '

415 "Array must contain at least one register."

416 )

417 raise ValueError(message)

418

419 def _check_field_items(

420 self,

421 register_name: str,

422 field_name: str,

423 field_items: dict[str, Any],

424 recognized_items: set[str],

425 required_items: list[str],

426 ) -> None:

427 """

428 Will raise exception if anything is wrong.

429 """

430 for item_name in required_items:

431 if item_name not in field_items:

432 message = (

433 f'Error while parsing field "{field_name}" in register "{register_name}" in '

434 f"{self._source_definition_file}: "

435 f'Missing required property "{item_name}".'

436 )

437 raise ValueError(message)

438

439 for item_name in field_items:

440 if item_name not in recognized_items:

441 message = (

442 f'Error while parsing field "{field_name}" in register '

443 f'"{register_name}" in {self._source_definition_file}: '

444 f'Unknown property "{item_name}".'

445 )

446 raise ValueError(message)

447

448 def _parse_bit(self, register: Register, field_name: str, field_items: dict[str, Any]) -> None:

449 self._check_field_items(

450 register_name=register.name,

451 field_name=field_name,

452 field_items=field_items,

453 recognized_items=self.recognized_bit_items,

454 required_items=self.required_bit_items,

455 )

456

457 description = field_items.get("description", "")

458 default_value = field_items.get("default_value", "0")

459

460 register.append_bit(name=field_name, description=description, default_value=default_value)

461

462 def _parse_bit_vector(

463 self, register: Register, field_name: str, field_items: dict[str, Any]

464 ) -> None:

465 self._check_field_items(

466 register_name=register.name,

467 field_name=field_name,

468 field_items=field_items,

469 recognized_items=self.recognized_bit_vector_items,

470 required_items=self.required_bit_vector_items,

471 )

472

473 width = field_items["width"]

474

475 description = field_items.get("description", "")

476 default_value = field_items.get("default_value", "0" * width)

477

478 register.append_bit_vector(

479 name=field_name, description=description, width=width, default_value=default_value

480 )

481

482 def _parse_enumeration(

483 self, register: Register, field_name: str, field_items: dict[str, Any]

484 ) -> None:

485 self._check_field_items(

486 register_name=register.name,

487 field_name=field_name,

488 field_items=field_items,

489 recognized_items=self.recognized_enumeration_items,

490 # Check that we have at least one element.

491 # This is checked also in the Enumeration class, which is needed if the user

492 # is working directly with the Python API.

493 # That is where we usually sanity check, to avoid duplication.

494 # However, this particular check is needed here also since the logic for default

495 # value below does not work if there are no elements.

496 required_items=self.required_enumeration_items,

497 )

498

499 description = field_items.get("description", "")

500 # We assert above that the enumeration has at least one element.

501 # Meaning that the result of this get can not be None.

502 elements: dict[str, str] = field_items.get("element")

503

504 # The default "default value" is the first declared enumeration element.

505 # Note that this works because dictionaries in Python are guaranteed ordered since

506 # Python 3.7.

507 default_value = field_items.get("default_value", next(iter(elements)))

508

509 register.append_enumeration(

510 name=field_name,

511 description=description,

512 elements=elements,

513 default_value=default_value,

514 )

515

516 def _parse_integer(

517 self, register: Register, field_name: str, field_items: dict[str, Any]

518 ) -> None:

519 self._check_field_items(

520 register_name=register.name,

521 field_name=field_name,

522 field_items=field_items,

523 recognized_items=self.recognized_integer_items,

524 required_items=self.required_integer_items,

525 )

526

527 max_value = field_items["max_value"]

528

529 description = field_items.get("description", "")

530 min_value = field_items.get("min_value", 0)

531 default_value = field_items.get("default_value", min_value)

532

533 register.append_integer(

534 name=field_name,

535 description=description,

536 min_value=min_value,

537 max_value=max_value,

538 default_value=default_value,

539 )

540

541

542def _convert_to_new_format( # noqa: C901

543 old_data: dict[str, Any],

544) -> dict[str, Any]:

545 """

546 Convert pre-6.0.0 format to the new format.

547 This is a semi-trash function that will be removed in the future.

548 """

549

550 def _get_register_dict(register_items: dict[str, Any]) -> dict[str, Any]:

551 register_dict = {}

552

553 for register_item_name, register_item_value in register_items.items():

554 if register_item_name in RegisterParser.default_register_items:

555 register_dict[register_item_name] = register_item_value

556

557 elif register_item_name in ["bit", "bit_vector", "enumeration", "integer"]:

558 for field_name, field_items in register_item_value.items():

559 field_dict = {"type": register_item_name}

560 field_dict.update(dict(field_items.items()))

561

562 register_dict[field_name] = field_dict

563

564 else:

565 raise ValueError(

566 f"Unknown item {register_item_name}. Looks like an error in the user data file."

567 )

568

569 return register_dict

570

571 result = {}

572

573 def _add_item(name: str, items: dict[str, Any]) -> None:

574 if name in result:

575 raise ValueError(f"Duplicate item {name}")

576

577 result[name] = items

578

579 if "register" in old_data:

580 for register_name, register_items in old_data["register"].items():

581 register_dict = _get_register_dict(register_items=register_items)

582 _add_item(name=register_name, items=register_dict)

583

584 if "register_array" in old_data:

585 for register_array_name, register_array_items in old_data["register_array"].items():

586 register_array_dict: dict[str, Any] = {"type": "register_array"}

587

588 for register_array_item_name, register_array_item_value in register_array_items.items():

589 if register_array_item_name in RegisterParser.default_register_array_items:

590 register_array_dict[register_array_item_name] = register_array_item_value

591

592 elif register_array_item_name == "register":

593 for register_name, register_items in register_array_item_value.items():

594 register_array_dict[register_name] = _get_register_dict(

595 register_items=register_items

596 )

597

598 else:

599 raise ValueError(

600 f"Unknown item {register_array_item_name}. "

601 "Looks like an error in the user data file."

602 )

603

604 _add_item(name=register_array_name, items=register_array_dict)

605

606 if "constant" in old_data:

607 for constant_name, constant_items in old_data["constant"].items():

608 constant_dict = {"type": "constant"}

609 constant_dict.update(dict(constant_items.items()))

610

611 _add_item(name=constant_name, items=constant_dict)

612

613 return result

614

615

616def _save_to_new_format(old_data: dict[str, Any], output_file: Path) -> None:

617 """

618 Save the old data to the new format.

619 """

620 new_data = _convert_to_new_format(old_data=old_data)

621

622 if output_file.suffix == ".toml":

623 with output_file.open("wb") as file_handle:

624 tomli_w.dump(new_data, file_handle, multiline_strings=True)

625

626 return

627

628 if output_file.suffix == ".json":

629 with output_file.open("w", encoding=DEFAULT_FILE_ENCODING) as file_handle:

630 json.dump(new_data, file_handle, indent=4)

631

632 return

633

634 if output_file.suffix == ".yaml":

635 with output_file.open("w", encoding=DEFAULT_FILE_ENCODING) as file_handle:

636 yaml.dump(new_data, file_handle)

637

638 return

639

640 raise ValueError(f"Unknown file format {output_file}")