Coverage for hdl_registers/parser/parser.py: 97%

1# --------------------------------------------------------------------------------------------------

4# This file is part of the hdl-registers project, an HDL register generator fast enough to run

5# in real time.

6# https://hdl-registers.com

7# https://github.com/hdl-registers/hdl-registers

8# --------------------------------------------------------------------------------------------------

10# Standard libraries

11import copy

12import json

13from pathlib import Path

14from typing import TYPE_CHECKING, Any, Optional

16# Third party libraries

17import tomli_w

18import yaml

19from tsfpga import DEFAULT_FILE_ENCODING

21# First party libraries

22from hdl_registers.about import WEBSITE_URL

23from hdl_registers.constant.bit_vector_constant import UnsignedVector

24from hdl_registers.register_list import RegisterList

25from hdl_registers.register_modes import REGISTER_MODES

27if TYPE_CHECKING:

28 # First party libraries

29 from hdl_registers.register import Register

30 from hdl_registers.register_mode import RegisterMode

33class RegisterParser:

34 # Attributes of the constant.

35 recognized_constant_items = {"type", "value", "description", "data_type"}

36 # Note that "type" being present is implied. We would not be parsing a constant unless we

37 # know it to be a "constant" type.

38 # So we save some CPU cycles by not checking for it.

39 required_constant_items = ["value"]

41 # Attributes of the register.

42 # Anything apart from these are names of fields.

43 default_register_items = {

44 "type",

45 "mode",

46 "description",

47 }

48 # While a 'mode' is required for a register, it may NOT be specified/changed in the data file

49 # for a default register.

50 # Hence this property is handled separately.

51 # And hence, registers have no required items.

53 # Attributes of the register array.

54 # Anything apart from these are names of registers.

55 default_register_array_items = {"type", "array_length", "description"}

56 # Note that "type" being present is implied.

57 # We would not be parsing a register array unless we know it to be a "register_array" type.

58 # So we save some CPU cycles by not checking for it.

59 required_register_array_items = ["array_length"]

61 # Attributes of the "bit" register field.

62 recognized_bit_items = {"type", "description", "default_value"}

63 # Note that "type" being present is implied.

64 # We would not be parsing a bit unless we know it to be a "bit" type.

65 # So we save some CPU cycles by not checking for it.

66 required_bit_items: list[str] = []

68 # Attributes of the "bit_vector" register field.

69 recognized_bit_vector_items = {"type", "description", "width", "default_value"}

70 # Note that "type" being present is implied.

71 # We would not be parsing a bit_vector unless we know it to be a "bit_vector" type.

72 # So we save some CPU cycles by not checking for it.

73 required_bit_vector_items = ["width"]

75 # Attributes of the "enumeration" register field.

76 recognized_enumeration_items = {"type", "description", "default_value", "element"}

77 # Note that "type" being present is implied.

78 # We would not be parsing a enumeration unless we know it to be a "enumeration" type.

79 # So we save some CPU cycles by not checking for it.

80 required_enumeration_items = ["element"]

82 # Attributes of the "integer" register field.

83 recognized_integer_items = {"type", "description", "min_value", "max_value", "default_value"}

84 # Note that "type" being present is implied.

85 # We would not be parsing a integer unless we know it to be a "integer" type.

86 # So we save some CPU cycles by not checking for it.

87 required_integer_items = ["max_value"]

89 def __init__(

90 self,

91 name: str,

92 source_definition_file: Path,

93 default_registers: Optional[list["Register"]] = None,

94 ):

95 """

96 Arguments:

97 name: The name of the register list.

98 source_definition_file: The source file that defined this register list.

99 Will be displayed in generated source code and documentation

100 for traceability.

101 default_registers: List of default registers.

102 Note that this list with :class:`.Register` objects will be deep copied, so you can

103 use the same list many times without worrying about mutability.

104 """

105 self._register_list = RegisterList(name=name, source_definition_file=source_definition_file)

106 self._source_definition_file = source_definition_file

107

108 self._default_register_names = []

109 if default_registers:

110 # Perform deep copy of the mutable register objects.

111 # Ignore a mypy error that seems buggy.

112 # We are assigning list[Register] to list[Register | RegisterArray]

113 # which should be absolutely fine, but mypy warns.

114 self._register_list.register_objects = copy.deepcopy(

115 default_registers # type: ignore[arg-type]

116 )

117 for register in default_registers:

118 self._default_register_names.append(register.name)

119

120 def parse(self, register_data: dict[str, Any]) -> RegisterList:

121 """

122 Parse the register data.

123

124 Arguments:

125 register_data: Register data as a dictionary.

126

127 Return:

128 The resulting register list.

129 """

130 for old_top_level_key_name in ["constant", "register", "register_array"]:

131 if old_top_level_key_name in register_data:

132 source_file = self._source_definition_file

133 output_file = (

134 source_file.parent.resolve()

135 / f"{source_file.stem}_version_6_format{source_file.suffix}"

136 )

137

138 print(

139 f"""

140ERROR: Parsing register data that appears to be in the old pre-6.0.0 format.

141ERROR: For more information, see: {WEBSITE_URL}/rst/basic_feature/basic_feature_register_modes.html

142ERROR: Your data will be automatically converted to the new format and saved to: {output_file}

143ERROR: Please inspect that file and update your data file to the new format.

144"""

145 )

146 _save_to_new_format(old_data=register_data, output_file=output_file)

147 raise ValueError("Found register data in old format. See message above.")

148

149 parser_methods = {

150 "constant": self._parse_constant,

151 "register": self._parse_plain_register,

152 "register_array": self._parse_register_array,

153 }

154

155 for top_level_name, top_level_items in register_data.items():

156 if not isinstance(top_level_items, dict):

157 message = (

158 f"Error while parsing {self._source_definition_file}: "

159 f'Got unknown top-level property "{top_level_name}".'

160 )

161 raise ValueError(message)

162

163 top_level_type = top_level_items.get("type", "register")

164

165 if top_level_type not in parser_methods:

166 valid_types_str = ", ".join(f'"{parser_key}"' for parser_key in parser_methods)

167 message = (

168 f'Error while parsing "{top_level_name}" in {self._source_definition_file}: '

169 f'Got unknown type "{top_level_type}". Expected one of {valid_types_str}.'

170 )

171 raise ValueError(message)

172

173 parser_methods[top_level_type](name=top_level_name, items=top_level_items)

174

175 return self._register_list

176

177 def _parse_constant(self, name: str, items: dict[str, Any]) -> None:

178 for item_name in self.required_constant_items:

179 if item_name not in items:

180 message = (

181 f'Error while parsing constant "{name}" in {self._source_definition_file}: '

182 f'Missing required property "{item_name}".'

183 )

184 raise ValueError(message)

185

186 for item_name in items.keys():

187 if item_name not in self.recognized_constant_items:

188 message = (

189 f'Error while parsing constant "{name}" in {self._source_definition_file}: '

190 f'Got unknown property "{item_name}".'

191 )

192 raise ValueError(message)

193

194 value = items["value"]

195 description = items.get("description", "")

196 data_type_str = items.get("data_type")

197

198 if data_type_str is not None:

199 if not isinstance(value, str):

200 raise ValueError(

201 f'Error while parsing constant "{name}" in '

202 f"{self._source_definition_file}: "

203 'May not set "data_type" for non-string constant.'

204 )

205

206 if data_type_str == "unsigned":

207 value = UnsignedVector(value)

208 else:

209 raise ValueError(

210 f'Error while parsing constant "{name}" in '

211 f"{self._source_definition_file}: "

212 f'Invalid data type "{data_type_str}".'

213 )

214

215 self._register_list.add_constant(name=name, value=value, description=description)

216

217 def _parse_plain_register(self, name: str, items: dict[str, Any]) -> None:

218 description = items.get("description", "")

219

220 if name in self._default_register_names:

221 # Default registers can be "updated" in the sense that the user can set a custom

222 # 'description' and add whatever fields they want in the current register list.

223 # They may not, however, change the 'mode' which is part of the default definition.

224 if "mode" in items:

225 message = (

226 f'Error while parsing register "{name}" in {self._source_definition_file}: '

227 'A "mode" may not be specified for a default register.'

228 )

229 raise ValueError(message)

230

231 register = self._register_list.get_register(register_name=name)

232 register.description = description

233

234 else:

235 # If it is a new register however, the 'mode' has to be specified.

236 if "mode" not in items:

237 message = (

238 f'Error while parsing register "{name}" in {self._source_definition_file}: '

239 f'Missing required property "mode".'

240 )

241 raise ValueError(message)

242

243 mode = self._get_mode(mode_name=items["mode"], register_name=name)

244

245 register = self._register_list.append_register(

246 name=name, mode=mode, description=description

247 )

248

249 self._parse_register_fields(register=register, register_items=items, register_array_note="")

250

251 def _get_mode(self, mode_name: str, register_name: str) -> "RegisterMode":

252 if mode_name in REGISTER_MODES:

253 return REGISTER_MODES[mode_name]

254

255 valid_modes_str = ", ".join(f'"{mode_key}"' for mode_key in REGISTER_MODES)

256 message = (

257 f'Error while parsing register "{register_name}" in {self._source_definition_file}: '

258 f'Got unknown mode "{mode_name}". Expected one of {valid_modes_str}.'

259 )

260 raise ValueError(message)

261

262 def _parse_register_fields(

263 self,

264 register_items: dict[str, Any],

265 register: "Register",

266 register_array_note: str,

267 ) -> None:

268 # Add any fields that are specified.

269 for item_name, item_value in register_items.items():

270 # Skip default items so we only get the fields.

271 if item_name in self.default_register_items:

272 continue

273

274 if not isinstance(item_value, dict):

275 message = (

276 f'Error while parsing register "{register.name}"{register_array_note} '

277 f"in {self._source_definition_file}: "

278 f'Got unknown property "{item_name}".'

279 )

280 raise ValueError(message)

281

282 if "type" not in item_value:

283 message = (

284 f'Error while parsing field "{item_name}" in register '

285 f'"{register.name}"{register_array_note} in {self._source_definition_file}: '

286 'Missing required property "type".'

287 )

288 raise ValueError(message)

289

290 field_type = item_value["type"]

291

292 parser_methods = {

293 "bit": self._parse_bit,

294 "bit_vector": self._parse_bit_vector,

295 "enumeration": self._parse_enumeration,

296 "integer": self._parse_integer,

297 }

298

299 if field_type not in parser_methods:

300 valid_types_str = ", ".join(f'"{parser_key}"' for parser_key in parser_methods)

301 message = (

302 f'Error while parsing field "{item_name}" in register '

303 f'"{register.name}"{register_array_note} in {self._source_definition_file}: '

304 f'Unknown field type "{field_type}". Expected one of {valid_types_str}.'

305 )

306 raise ValueError(message)

307

308 parser_methods[field_type](

309 register=register, field_name=item_name, field_items=item_value

310 )

311

312 def _parse_register_array(self, name: str, items: dict[str, Any]) -> None:

313 for required_property in self.required_register_array_items:

314 if required_property not in items:

315 message = (

316 f'Error while parsing register array "{name}" in '

317 f"{self._source_definition_file}: "

318 f'Missing required property "{required_property}".'

319 )

320 raise ValueError(message)

321

322 register_array_length = items["array_length"]

323 register_array_description = items.get("description", "")

324 register_array = self._register_list.append_register_array(

325 name=name, length=register_array_length, description=register_array_description

326 )

327

328 # Add all registers that are specified.

329 found_at_least_one_register = False

330 for item_name, item_value in items.items():

331 # Skip default items so we only get the registers.

332 if item_name in self.default_register_array_items:

333 continue

334

335 found_at_least_one_register = True

336

337 if not isinstance(item_value, dict):

338 message = (

339 f'Error while parsing register array "{name}" in '

340 f"{self._source_definition_file}: "

341 f'Got unknown property "{item_name}".'

342 )

343 raise ValueError(message)

344

345 item_type = item_value.get("type", "register")

346 if item_type != "register":

347 message = (

348 f'Error while parsing register "{item_name}" within array "{name}" in '

349 f"{self._source_definition_file}: "

350 f'Got unknown type "{item_type}". Expected "register".'

351 )

352 raise ValueError(message)

353

354 # A 'mode' is semi-required for plain registers, but always required for

355 # array registers.

356 if "mode" not in item_value:

357 raise ValueError(

358 f'Error while parsing register "{item_name}" within array "{name}" in '

359 f"{self._source_definition_file}: "

360 f'Missing required property "mode".'

361 )

362 register_mode = self._get_mode(mode_name=item_value["mode"], register_name=item_name)

363

364 register_description = item_value.get("description", "")

365

366 register = register_array.append_register(

367 name=item_name, mode=register_mode, description=register_description

368 )

369

370 self._parse_register_fields(

371 register_items=item_value,

372 register=register,

373 register_array_note=f' within array "{name}"',

374 )

375

376 if not found_at_least_one_register:

377 message = (

378 f'Error while parsing register array "{name}" in {self._source_definition_file}: '

379 "Array must contain at least one register."

380 )

381 raise ValueError(message)

382

383 def _check_field_items(

384 self,

385 register_name: str,

386 field_name: str,

387 field_items: dict[str, Any],

388 recognized_items: set[str],

389 required_items: list[str],

390 ) -> None:

391 """

392 Will raise exception if anything is wrong.

393 """

394 for item_name in required_items:

395 if item_name not in field_items:

396 message = (

397 f'Error while parsing field "{field_name}" in register "{register_name}" in '

398 f"{self._source_definition_file}: "

399 f'Missing required property "{item_name}".'

400 )

401 raise ValueError(message)

402

403 for item_name in field_items.keys():

404 if item_name not in recognized_items:

405 message = (

406 f'Error while parsing field "{field_name}" in register '

407 f'"{register_name}" in {self._source_definition_file}: '

408 f'Unknown property "{item_name}".'

409 )

410 raise ValueError(message)

411

412 def _parse_bit(

413 self, register: "Register", field_name: str, field_items: dict[str, Any]

414 ) -> None:

415 self._check_field_items(

416 register_name=register.name,

417 field_name=field_name,

418 field_items=field_items,

419 recognized_items=self.recognized_bit_items,

420 required_items=self.required_bit_items,

421 )

422

423 description = field_items.get("description", "")

424 default_value = field_items.get("default_value", "0")

425

426 register.append_bit(name=field_name, description=description, default_value=default_value)

427

428 def _parse_bit_vector(

429 self, register: "Register", field_name: str, field_items: dict[str, Any]

430 ) -> None:

431 self._check_field_items(

432 register_name=register.name,

433 field_name=field_name,

434 field_items=field_items,

435 recognized_items=self.recognized_bit_vector_items,

436 required_items=self.required_bit_vector_items,

437 )

438

439 width = field_items["width"]

440

441 description = field_items.get("description", "")

442 default_value = field_items.get("default_value", "0" * width)

443

444 register.append_bit_vector(

445 name=field_name, description=description, width=width, default_value=default_value

446 )

447

448 def _parse_enumeration(

449 self, register: "Register", field_name: str, field_items: dict[str, Any]

450 ) -> None:

451 self._check_field_items(

452 register_name=register.name,

453 field_name=field_name,

454 field_items=field_items,

455 recognized_items=self.recognized_enumeration_items,

456 # Check that we have at least one element.

457 # This is checked also in the Enumeration class, which is needed if the user

458 # is working directly with the Python API.

459 # That is where we usually sanity check, to avoid duplication.

460 # However, this particular check is needed here also since the logic for default

461 # value below does not work if there are no elements.

462 required_items=self.required_enumeration_items,

463 )

464

465 description = field_items.get("description", "")

466 # We assert above that the enumeration has at least one element.

467 # Meaning that the result of this get can not be None, as mypy thinks.

468 elements: dict[str, str] = field_items.get("element") # type: ignore[assignment]

469

470 # The default "default value" is the first declared enumeration element.

471 # Note that this works because dictionaries in Python are guaranteed ordered since

472 # Python 3.7.

473 default_value = field_items.get("default_value", list(elements)[0])

474

475 register.append_enumeration(

476 name=field_name,

477 description=description,

478 elements=elements,

479 default_value=default_value,

480 )

481

482 def _parse_integer(

483 self, register: "Register", field_name: str, field_items: dict[str, Any]

484 ) -> None:

485 self._check_field_items(

486 register_name=register.name,

487 field_name=field_name,

488 field_items=field_items,

489 recognized_items=self.recognized_integer_items,

490 required_items=self.required_integer_items,

491 )

492

493 max_value = field_items["max_value"]

494

495 description = field_items.get("description", "")

496 min_value = field_items.get("min_value", 0)

497 default_value = field_items.get("default_value", min_value)

498

499 register.append_integer(

500 name=field_name,

501 description=description,

502 min_value=min_value,

503 max_value=max_value,

504 default_value=default_value,

505 )

506

507

508def _convert_to_new_format( # pylint: disable=too-many-locals

509 old_data: dict[str, Any]

510) -> dict[str, Any]:

511 """

512 Convert pre-6.0.0 format to the new format.

513 """

514

515 def _get_register_dict(register_items: dict[str, Any]) -> dict[str, Any]:

516 register_dict = dict()

517

518 for register_item_name, register_item_value in register_items.items():

519 if register_item_name in RegisterParser.default_register_items:

520 register_dict[register_item_name] = register_item_value

521

522 elif register_item_name in ["bit", "bit_vector", "enumeration", "integer"]:

523 for field_name, field_items in register_item_value.items():

524 field_dict = dict(type=register_item_name)

525

526 for field_item_name, field_item_value in field_items.items():

527 field_dict[field_item_name] = field_item_value

528

529 register_dict[field_name] = field_dict

530

531 else:

532 raise ValueError(

533 f"Unknown item {register_item_name}. "

534 "Looks like an error in the user data file."

535 )

536

537 return register_dict

538

539 result = dict()

540

541 def _add_item(name: str, items: dict[str, Any]) -> None:

542 if name in result:

543 raise ValueError(f"Duplicate item {name}")

544

545 result[name] = items

546

547 if "register" in old_data:

548 for register_name, register_items in old_data["register"].items():

549 register_dict = _get_register_dict(register_items=register_items)

550 _add_item(name=register_name, items=register_dict)

551

552 if "register_array" in old_data:

553 for register_array_name, register_array_items in old_data["register_array"].items():

554 register_array_dict: dict[str, Any] = dict(type="register_array")

555

556 for register_array_item_name, register_array_item_value in register_array_items.items():

557 if register_array_item_name in RegisterParser.default_register_array_items:

558 register_array_dict[register_array_item_name] = register_array_item_value

559

560 elif register_array_item_name == "register":

561 for register_name, register_items in register_array_item_value.items():

562 register_array_dict[register_name] = _get_register_dict(

563 register_items=register_items

564 )

565

566 else:

567 raise ValueError(

568 f"Unknown item {register_array_item_name}. "

569 "Looks like an error in the user data file."

570 )

571

572 _add_item(name=register_array_name, items=register_array_dict)

573

574 if "constant" in old_data:

575 for constant_name, constant_items in old_data["constant"].items():

576 constant_dict = dict(type="constant")

577

578 for constant_item_name, constant_item_value in constant_items.items():

579 constant_dict[constant_item_name] = constant_item_value

580

581 _add_item(name=constant_name, items=constant_dict)

582

583 return result

584

585

586def _save_to_new_format(old_data: dict[str, Any], output_file: Path) -> None:

587 """

588 Save the old data to the new format.

589 """

590 new_data = _convert_to_new_format(old_data=old_data)

591

592 if output_file.suffix == ".toml":

593 with open(output_file, "wb") as file_handle:

594 tomli_w.dump(new_data, file_handle, multiline_strings=True)

595

596 return

597

598 if output_file.suffix == ".json":

599 with open(output_file, "w", encoding=DEFAULT_FILE_ENCODING) as file_handle:

600 json.dump(new_data, file_handle, indent=4)

601

602 return

603

604 if output_file.suffix == ".yaml":

605 with open(output_file, "w", encoding=DEFAULT_FILE_ENCODING) as file_handle:

606 yaml.dump(new_data, file_handle)

607

608 return

609

610 raise ValueError(f"Unknown file format {output_file}")