From 48adcd15798490b56443757b5aed9089ede1ef72 Mon Sep 17 00:00:00 2001 From: zvecr Date: Mon, 19 Oct 2026 13:09:18 +0100 Subject: [PATCH] Generalise version file search --- lib/python/qmk/cli/generate/api.py | 4 +- lib/python/qmk/community_modules.py | 18 +--- lib/python/qmk/keycodes.py | 20 +--- lib/python/qmk/tests/test_qmk_util.py | 134 ++++++++++++++++++++++++++ lib/python/qmk/util.py | 42 +++++++- 5 files changed, 179 insertions(+), 39 deletions(-) create mode 100644 lib/python/qmk/tests/test_qmk_util.py diff --git a/lib/python/qmk/cli/generate/api.py b/lib/python/qmk/cli/generate/api.py index ad6db504b0b..cdc4f947898 100755 --- a/lib/python/qmk/cli/generate/api.py +++ b/lib/python/qmk/cli/generate/api.py @@ -9,6 +9,7 @@ from milc import cli import qmk.path from qmk.datetime import current_datetime from qmk.info import info_json +from qmk.util import is_valid_bcd_string from qmk.json_schema import json_load from qmk.keymap import list_keymaps from qmk.keyboard import find_readme, list_keyboards, keyboard_alias_definitions @@ -25,8 +26,7 @@ def _list_constants(output_folder): ret = {} for file in (output_folder / 'constants').glob('**/*.json'): name, _, version = file.stem.rpartition('_') - parts = version.split('.') - if not name or len(parts) != 3 or not all(part.isascii() and part.isdigit() for part in parts): + if not name or not is_valid_bcd_string(version): continue if name not in ret: ret[name] = [] diff --git a/lib/python/qmk/community_modules.py b/lib/python/qmk/community_modules.py index 99ab25bf01f..d03246d91db 100644 --- a/lib/python/qmk/community_modules.py +++ b/lib/python/qmk/community_modules.py @@ -6,7 +6,7 @@ from functools import lru_cache from milc.attrdict import AttrDict from qmk.json_schema import json_load, validate, merge_ordered_dicts -from qmk.util import truthy +from qmk.util import truthy, list_version_files from qmk.constants import QMK_FIRMWARE, QMK_USERSPACE, HAS_QMK_USERSPACE from qmk.path import under_qmk_firmware, under_qmk_userspace @@ -22,23 +22,9 @@ class ModuleAPI(AttrDict): self[key] = value -def _list_module_files(): - versions = [] - module_definition_files = HOOKS_PATH.glob('*.hjson') - for file in module_definition_files: - version = file.stem - parts = version.split('.') - if len(parts) == 3 and all(part.isascii() and part.isdigit() for part in parts): - versions.append(version) - - versions.sort(key=lambda version: tuple(map(int, version.split('.')))) - - return list(map(lambda version: HOOKS_PATH / f'{version}.hjson', versions)) - - @lru_cache(maxsize=1) def module_api_list(): - module_definition_files = _list_module_files() + module_definition_files = list_version_files(HOOKS_PATH) module_definition_jsons = [json_load(f) for f in module_definition_files] module_definitions = merge_ordered_dicts(module_definition_jsons) latest_module_version = module_definition_files[-1].stem diff --git a/lib/python/qmk/keycodes.py b/lib/python/qmk/keycodes.py index b4e7a22abdd..e076848bfdf 100644 --- a/lib/python/qmk/keycodes.py +++ b/lib/python/qmk/keycodes.py @@ -1,24 +1,13 @@ from pathlib import Path from qmk.json_schema import merge_ordered_dicts, deep_update, json_load, validate +from qmk.util import is_valid_bcd_string, list_file_versions CONSTANTS_PATH = Path('data/constants/') KEYCODES_PATH = CONSTANTS_PATH / 'keycodes' EXTRAS_PATH = KEYCODES_PATH / 'extras' -def _find_versions(path, prefix): - ret = [] - for file in path.glob(f'{prefix}_*.hjson'): - version = file.stem[len(prefix) + 1:] - parts = version.split('.') - if len(parts) == 3 and all(part.isascii() and part.isdigit() for part in parts): - ret.append(version) - - ret.sort(key=lambda version: tuple(map(int, version.split('.'))), reverse=True) - return ret - - def _potential_search_versions(version, lang=None): versions = list_versions(lang) versions.reverse() @@ -108,7 +97,7 @@ def list_versions(lang=None): path = _search_path(lang) prefix = _search_prefix(lang) - return _find_versions(path, prefix) + return list_file_versions(path, prefix=f'{prefix}_', reverse=True) def list_languages(): @@ -116,9 +105,8 @@ def list_languages(): """ ret = set() for file in EXTRAS_PATH.glob('keycodes_*.hjson'): - name, _, version = file.stem.rpartition('_') - parts = version.split('.') - if len(parts) == 3 and all(part.isascii() and part.isdigit() for part in parts): + name, version = file.stem.rsplit('_', 1) + if is_valid_bcd_string(version): ret.add(name[len('keycodes_'):]) return ret diff --git a/lib/python/qmk/tests/test_qmk_util.py b/lib/python/qmk/tests/test_qmk_util.py new file mode 100644 index 00000000000..33858e79640 --- /dev/null +++ b/lib/python/qmk/tests/test_qmk_util.py @@ -0,0 +1,134 @@ +from pathlib import Path +from tempfile import TemporaryDirectory + +from qmk import util + + +def test_triplet_to_bcd_formats_version_components(): + assert util.triplet_to_bcd('1.2.3') == '0x01020003' + assert util.triplet_to_bcd('12.34.5678') == '0x12345678' + + +def test_triplet_to_bcd_accepts_suffix_after_version(): + assert util.triplet_to_bcd('1.2.3-4-gabcdef') == '0x01020003' + + +def test_triplet_to_bcd_returns_zero_for_non_matching_version(): + assert util.triplet_to_bcd('invalid') == '0x00000000' + assert util.triplet_to_bcd(None) == '0x00000000' + assert util.triplet_to_bcd('') == '0x00000000' + assert util.triplet_to_bcd('1.2') == '0x00000000' + # assert util.triplet_to_bcd('1.2.3.4') == '0x00000000' + assert util.triplet_to_bcd('1.-2.3') == '0x00000000' + assert util.triplet_to_bcd('\uff11.2.3') == '0x00000000' + assert util.triplet_to_bcd('1.2.a') == '0x00000000' + assert util.triplet_to_bcd('0.0.10000') == '0x00000000' + assert util.triplet_to_bcd('0.100.0') == '0x00000000' + assert util.triplet_to_bcd('100.0.0') == '0x00000000' + + +def test_is_valid_bcd_string(): + assert util.is_valid_bcd_string('1.2.3') + assert util.is_valid_bcd_string('12.34.5678') + + +def test_is_valid_bcd_string_returns_zero_for_non_matching_version(): + assert not util.is_valid_bcd_string(None) + assert not util.is_valid_bcd_string('') + assert not util.is_valid_bcd_string('1.2') + assert not util.is_valid_bcd_string('1.2.3.4') + assert not util.is_valid_bcd_string('1.-2.3') + assert not util.is_valid_bcd_string('\uff11.2.3') + assert not util.is_valid_bcd_string('1.2.a') + assert not util.is_valid_bcd_string('0.0.10000') + assert not util.is_valid_bcd_string('0.100.0') + assert not util.is_valid_bcd_string('100.0.0') + + +def test_list_versions_orders_multi_digit_versions(): + with TemporaryDirectory() as temporary: + path = Path(temporary) + for version in ('0.0.9', '0.0.10', '0.1.0'): + (path / f'{version}.hjson').touch() + assert util.list_file_versions(path) == ['0.0.9', '0.0.10', '0.1.0'] + + +def test_list_versions_can_reverse_order(): + with TemporaryDirectory() as temporary: + path = Path(temporary) + for version in ('0.0.9', '0.0.10', '0.1.0'): + (path / f'{version}.hjson').touch() + assert util.list_file_versions(path, reverse=True) == ['0.1.0', '0.0.10', '0.0.9'] + + +def test_list_versions_supports_custom_extension(): + with TemporaryDirectory() as temporary: + path = Path(temporary) + (path / '0.0.1.json').touch() + (path / '0.0.2.hjson').touch() + assert util.list_file_versions(path, ext='json') == ['0.0.1'] + + +def test_list_versions_ignores_non_version_files(): + with TemporaryDirectory() as temporary: + path = Path(temporary) + (path / '0.0.1.hjson').touch() + (path / 'invalid.hjson').touch() + assert util.list_file_versions(path) == ['0.0.1'] + + +def test_list_versions_ignores_suffixes(): + with TemporaryDirectory() as temporary: + path = Path(temporary) + (path / '0.0.1.hjson').touch() + (path / '0.0.1_quantum.hjson').touch() + assert util.list_file_versions(path) == ['0.0.1'] + + +def test_list_versions_ignores_too_large_bcd(): + with TemporaryDirectory() as temporary: + path = Path(temporary) + for version in ('99.99.9999', '0.0.10000', '0.100.0', '100.0.0'): + (path / f'{version}.hjson').touch() + assert util.list_file_versions(path) == ['99.99.9999'] + + +def test_list_versions_ignores_invalid_bcd(): + with TemporaryDirectory() as temporary: + path = Path(temporary) + for version in ('1.2', '1.2.3.4', '1.-2.3', '\uff11.2.3', '1.2.a'): + (path / f'{version}.hjson').touch() + assert util.list_file_versions(path) == [] + + +def test_list_versions_finds_only_with_prefix(): + with TemporaryDirectory() as temporary: + path = Path(temporary) + (path / 'keycodes_0.0.1.hjson').touch() + (path / 'keycodes_us_0.0.2.hjson').touch() + assert util.list_file_versions(path, prefix='keycodes_') == ['0.0.1'] + + +def test_list_version_files_orders_multi_digit_versions(): + with TemporaryDirectory() as temporary: + path = Path(temporary) + for version in ('0.0.9', '0.0.10', '0.1.0'): + (path / f'{version}.hjson').touch() + assert util.list_version_files(path) == [ + path / '0.0.9.hjson', + path / '0.0.10.hjson', + path / '0.1.0.hjson', + ] + + +def test_list_version_files_forwards_args(): + with TemporaryDirectory() as temporary: + path = Path(temporary) + (path / 'keycodes_0.0.9.json').touch() + (path / 'keycodes_0.0.10.json').touch() + (path / 'keycodes_0.1.0.hjson').touch() + (path / 'other_0.1.0.json').touch() + assert util.list_version_files(path, ext='json', prefix='keycodes_', reverse=True) == [ + path / 'keycodes_0.0.10.json', + path / 'keycodes_0.0.9.json', + ] diff --git a/lib/python/qmk/util.py b/lib/python/qmk/util.py index 6da684a577f..f44df09e89f 100644 --- a/lib/python/qmk/util.py +++ b/lib/python/qmk/util.py @@ -4,6 +4,7 @@ import contextlib import multiprocessing import sys import re +from pathlib import Path from milc import cli @@ -101,8 +102,39 @@ def parallel_map(*args, **kwargs): return list(map_fn(*args, **kwargs)) -def triplet_to_bcd(ver: str): - m = TRIPLET_PATTERN.match(ver) - if not m: - return '0x00000000' - return f'0x{int(m.group(1)):02d}{int(m.group(2)):02d}{int(m.group(3)):04d}' +def is_valid_bcd_string(ver: str) -> bool: + if ver: + parts = ver.split('.') + if len(parts) == 3 and all(part.isascii() and part.isdigit() for part in parts): + major, minor, patch = map(int, parts) + # Ensure that version is valid u32 bcd + if major <= 99 and minor <= 99 and patch <= 9999: + return True + return False + + +def triplet_to_bcd(ver: str) -> str: + if ver: + m = TRIPLET_PATTERN.match(ver) + if m and is_valid_bcd_string(m.expand(r'\1.\2.\3')): + return f'0x{int(m.group(1)):02d}{int(m.group(2)):02d}{int(m.group(3)):04d}' + return '0x00000000' + + +def _search_file_versions(parent: Path, ext: str = 'hjson', prefix: str = '', reverse: bool = False) -> dict[str, Path]: + versions = {} + found = parent.glob(f'{prefix}*.{ext}') + for file in found: + version = file.stem[len(prefix):] if prefix else file.stem + if is_valid_bcd_string(version): + versions[version] = file + + return dict(sorted(versions.items(), key=lambda item: tuple(map(int, item[0].split('.'))), reverse=reverse)) + + +def list_file_versions(parent: Path, ext: str = 'hjson', prefix: str = '', reverse: bool = False) -> list[str]: + return list(_search_file_versions(parent, ext=ext, prefix=prefix, reverse=reverse).keys()) + + +def list_version_files(parent: Path, ext: str = 'hjson', prefix: str = '', reverse: bool = False) -> list[Path]: + return list(_search_file_versions(parent, ext=ext, prefix=prefix, reverse=reverse).values())