From bef82729893bb3ecdb86deeed39e00737c92fc2b Mon Sep 17 00:00:00 2001 From: Aaron Shaw Date: Sat, 11 Mar 2023 16:25:20 +0000 Subject: [PATCH 1/2] feat: allow override of key location and remove old version support This is a quick and hacky fix to the issue described here https://github.com/NebraLtd/helium-syncrobit/issues/2#issuecomment-1464944930 The idea is to allow a device level environment variable `SWARM_KEY_URI_OVERRIDE` in balenaCloud to override the key location in the assigned variant without needing to define a new variant. This is useful for testing new devices or in instances like this Syncrobit one to provide a quick fix until we have implemented a "proper" fix to this issue. This additionally changes the following: - we hard code the gateway_mfr_rs version in the [test](https://github.com/NebraLtd/hm-pyhelper/blob/master/.github/workflows/publish-to-pypi-test.yml) and [production](https://github.com/NebraLtd/hm-pyhelper/blob/master/.github/workflows/publish-to-pypi.yml) python package build actions, so there is no need to support old versions of gateway-mfr-rs - due to removing the above, there is also no need to keep the KEY_STORAGE_BUS key in the hardware variants as this is not used anywhere else that I'm aware of **(we should double check this in config, diag, miner, packet forwarder and multiplexer containers just to be 100% sure)** - [Used as a fallback in diag](https://github.com/NebraLtd/hm-diag/blob/23c66980f75568a4cecb18c5375a08cd67bff550/hw_diag/utilities/hardware.py#L200) but not required Relates-to: https://github.com/NebraLtd/helium-syncrobit/issues/2 Relates-to: #221 Relates-to: #222 --- hm_pyhelper/hardware_definitions.py | 28 +---------- hm_pyhelper/miner_param.py | 35 ++++++-------- hm_pyhelper/tests/test_miner_param.py | 67 +++++++++++---------------- setup.py | 2 +- 4 files changed, 43 insertions(+), 89 deletions(-) diff --git a/hm_pyhelper/hardware_definitions.py b/hm_pyhelper/hardware_definitions.py index bede6ea..cab3137 100644 --- a/hm_pyhelper/hardware_definitions.py +++ b/hm_pyhelper/hardware_definitions.py @@ -26,7 +26,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['raspberrypi3-64'], 'SPIBUS': 'spidev1.2', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 38, 'MAC': 'eth0', @@ -48,7 +47,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['raspberrypi3-64'], 'SPIBUS': 'spidev1.2', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 38, 'MAC': 'eth0', @@ -65,14 +63,13 @@ def is_raspberry_pi() -> bool: }, # Nebra Outdoor Hotspot Gen2 - # Note:: assuming outdoor2 is equivalent to light miner 2 + # Note:: assuming outdoor2 is equivalent to indoor2 'nebra-outdoor2': { 'FRIENDLY': 'Nebra ROCK Pi 4 Outdoor', 'SUPPORTED_MODELS': ['Nebra ROCK Pi 4 Outdoor'], 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['rockpi-4b-rk3399'], 'SPIBUS': 'spidev32766.0', - 'KEY_STORAGE_BUS': '/dev/i2c-7', 'SWARM_KEY_URI': 'ecc://i2c-7:96?slot=0', 'RESET': 149, 'MAC': 'eth0', @@ -98,7 +95,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'armv6hf', 'BALENA_DEVICE_TYPE': ['raspberry-pi'], 'SPIBUS': 'spidev1.2', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 22, 'MAC': 'wlan0', @@ -120,7 +116,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'armv6hf', 'BALENA_DEVICE_TYPE': ['raspberry-pi'], 'SPIBUS': 'spidev1.2', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 22, 'MAC': 'wlan0', @@ -142,7 +137,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['radxa-zero'], 'SPIBUS': 'spidev0.0', - 'KEY_STORAGE_BUS': '/dev/i2c-3', 'SWARM_KEY_URI': 'ecc://i2c-3:96?slot=0', 'RESET': 415, 'MAC': 'wlan0', @@ -164,7 +158,7 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'armv6hf', 'BALENA_DEVICE_TYPE': ['raspberry-pi'], 'SPIBUS': 'spidev1.2', - 'KEY_STORAGE_BUS': '/dev/i2c-1', + 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 4, 'MAC': 'wlan0', 'STATUS': 26, @@ -185,7 +179,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'armv7hf', 'BALENA_DEVICE_TYPE': ['beaglebone-black'], 'SPIBUS': 'spidev1.0', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 60, 'MAC': 'eth0', @@ -207,7 +200,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'armv7hf', 'BALENA_DEVICE_TYPE': ['beaglebone-pocket'], 'SPIBUS': 'spidev1.2', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 60, 'MAC': 'wlan0', @@ -229,7 +221,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['rockpi-4b-rk3399'], 'SPIBUS': 'spidev32766.0', - 'KEY_STORAGE_BUS': '/dev/i2c-7', 'SWARM_KEY_URI': 'ecc://i2c-7:96?slot=0', 'RESET': 149, 'MAC': 'eth0', @@ -253,7 +244,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'armv7hf', 'BALENA_DEVICE_TYPE': ['asus-tinker-board'], 'SPIBUS': 'spidev2.0', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 167, 'MAC': 'eth0', @@ -275,7 +265,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['raspberrypi4-64'], 'SPIBUS': 'spidev0.0', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 25, 'MAC': 'wlan0', @@ -297,7 +286,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['raspberrypi3-64', 'raspberrypi4-64'], 'SPIBUS': 'spidev0.0', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 25, 'MAC': 'wlan0', @@ -319,7 +307,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['raspberrypi4-64'], 'SPIBUS': 'spidev0.0', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 17, 'MAC': 'wlan0', @@ -341,7 +328,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['raspberrypi4-64'], 'SPIBUS': 'spidev0.0', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 23, 'MAC': 'wlan0', @@ -363,7 +349,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['raspberrypi4-64'], 'SPIBUS': 'spidev0.0', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 17, 'MAC': 'wlan0', @@ -390,7 +375,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['raspberrypi4-64'], 'SPIBUS': 'spidev0.0', - 'KEY_STORAGE_BUS': '/dev/i2c-0', 'SWARM_KEY_URI': 'ecc://i2c-0:96?slot=0', 'RESET': 23, 'MAC': 'eth0', @@ -412,7 +396,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['raspberrypi4-64'], 'SPIBUS': 'spidev0.0', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 4, 'MAC': 'wlan0', @@ -441,7 +424,6 @@ def is_raspberry_pi() -> bool: 'raspberrypi0-2w-64', 'raspberrypi3-64', 'raspberrypi4-64'], 'SPIBUS': 'spidev0.0', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 22, 'MAC': 'eth0', @@ -465,7 +447,6 @@ def is_raspberry_pi() -> bool: 'raspberrypi0-2w-64', 'raspberrypi3-64', 'raspberrypi4-64'], 'SPIBUS': 'spidev0.0', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 17, 'MAC': 'eth0', @@ -487,7 +468,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['raspberrypicm4-ioboard'], 'SPIBUS': 'spidev0.0', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 17, 'MAC': 'wlan0', @@ -509,7 +489,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['raspberrypicm4-ioboard'], 'SPIBUS': 'spidev0.0', - 'KEY_STORAGE_BUS': '/dev/i2c-10', 'SWARM_KEY_URI': 'ecc://i2c-10:96?slot=0', 'RESET': 23, 'MAC': 'wlan0', @@ -531,7 +510,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['raspberrypicm4-ioboard'], 'SPIBUS': 'spidev0.0', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 17, 'MAC': 'wlan0', @@ -553,7 +531,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['raspberrypi4-64'], 'SPIBUS': 'spidev0.0', # There is a CSN1 pin which is connected to GPIO6 (HAT Pin 31) - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 22, 'MAC': 'eth0', @@ -575,7 +552,6 @@ def is_raspberry_pi() -> bool: 'CPU_ARCH': 'arm64', 'BALENA_DEVICE_TYPE': ['raspberrypicm4-ioboard'], 'SPIBUS': 'spidev0.0', - 'KEY_STORAGE_BUS': '/dev/i2c-1', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', 'RESET': 17, 'MAC': 'wlan0', diff --git a/hm_pyhelper/miner_param.py b/hm_pyhelper/miner_param.py index ce75104..e99b823 100644 --- a/hm_pyhelper/miner_param.py +++ b/hm_pyhelper/miner_param.py @@ -1,4 +1,5 @@ import os +import re import subprocess import json import platform @@ -26,7 +27,7 @@ @lock_ecc() -def run_gateway_mfr(sub_command: str, slot: int = 0) -> dict: +def run_gateway_mfr(sub_command: str, slot: int = False) -> dict: command = get_gateway_mfr_command(sub_command, slot=slot) try: @@ -110,39 +111,31 @@ def get_gateway_mfr_version() -> Version: raise GatewayMFRInvalidVersion(err_str).with_traceback(e.__traceback__) -def get_gateway_mfr_command(sub_command: str, slot: int = 0) -> list: +def get_gateway_mfr_command(sub_command: str, slot: int = False) -> list: gateway_mfr_path = get_gateway_mfr_path() command = [gateway_mfr_path] gateway_mfr_version = get_gateway_mfr_version() - if Version('0.1.1') < gateway_mfr_version < Version('0.2.0'): - try: - device_arg = [ - '--path', - get_variant_attribute(os.getenv('VARIANT'), 'KEY_STORAGE_BUS') - ] - command.extend(device_arg) - except (UnknownVariantException, UnknownVariantAttributeException) as e: - LOGGER.warning(str(e) + ' Omitting --path arg.') - - command.append(sub_command) - # In case of "key" command, append the slot number 0 at the end. - if sub_command == "key": - command.append("0") - - elif gateway_mfr_version >= Version('0.2.0'): + if gateway_mfr_version >= Version('0.2.0'): try: + if os.getenv('SWARM_KEY_URI_OVERRIDE'): + ecc_location = os.getenv('SWARM_KEY_URI_OVERRIDE') + else: + ecc_location = get_variant_attribute(os.getenv('VARIANT'), 'SWARM_KEY_URI') + device_arg = [ '--device', - get_variant_attribute(os.getenv('VARIANT'), 'SWARM_KEY_URI') + ecc_location ] command.extend(device_arg) except (UnknownVariantException, UnknownVariantAttributeException) as e: LOGGER.warning(str(e) + ' Omitting --device arg.') - slot_str = f'slot={slot}' - command[-1] = command[-1].replace('slot=0', slot_str) + if slot: + slot_str = f'slot={slot}' + slot_pattern = r'(slot=\d+)' + command[-1] = re.sub(slot_pattern, slot_str, command[-1]) if ' ' in sub_command: command += sub_command.split(' ') diff --git a/hm_pyhelper/tests/test_miner_param.py b/hm_pyhelper/tests/test_miner_param.py index a4e8d78..fbeb25b 100644 --- a/hm_pyhelper/tests/test_miner_param.py +++ b/hm_pyhelper/tests/test_miner_param.py @@ -8,7 +8,7 @@ GatewayMFRFileNotFoundException, UnsupportedGatewayMfrVersion from hm_pyhelper.lock_singleton import ResourceBusyError from hm_pyhelper.miner_param import retry_get_region, await_spi_available, \ - provision_key, run_gateway_mfr, \ + provision_key, run_gateway_mfr, get_gateway_mfr_path, \ did_gateway_mfr_test_result_include_miner_key_pass, \ get_mac_address, get_public_keys_rust, get_gateway_mfr_version, get_gateway_mfr_command @@ -119,44 +119,6 @@ def test_get_gateway_mfr_version_v021(self, mocked_subprocess_run): mocked_subprocess_run.assert_called_once_with( [ANY, '--version'], capture_output=True, check=True) - @patch('hm_pyhelper.miner_param.get_gateway_mfr_version', - return_value=Version('0.1.7')) - def test_get_gateway_mfr_command_v017(self, mocked_get_gateway_mfr_version): - actual_result = get_gateway_mfr_command('key') - expected_result = [ANY, '--path', '/dev/i2c-X', 'key', '0'] - self.assertListEqual(actual_result, expected_result) - mocked_get_gateway_mfr_version.assert_called_once() - - actual_result = get_gateway_mfr_command('info') - expected_result = [ANY, '--path', '/dev/i2c-X', 'info'] - self.assertListEqual(actual_result, expected_result) - - @patch.dict('os.environ', {"VARIANT": "NEBHNT-INVALID"}) - @patch('hm_pyhelper.miner_param.get_gateway_mfr_version', - return_value=Version('0.1.7')) - def test_get_gateway_mfr_command_v017_no_variant(self, mocked_get_gateway_mfr_version): - actual_result = get_gateway_mfr_command('key') - expected_result = [ANY, 'key', '0'] - self.assertListEqual(actual_result, expected_result) - mocked_get_gateway_mfr_version.assert_called_once() - - actual_result = get_gateway_mfr_command('info') - expected_result = [ANY, 'info'] - self.assertListEqual(actual_result, expected_result) - - @patch.dict('os.environ', {"VARIANT": "NEBHNT-NO-ECC-ADDRESS"}) - @patch('hm_pyhelper.miner_param.get_gateway_mfr_version', - return_value=Version('0.1.7')) - def test_get_gateway_mfr_command_v017_no_KEY_STORAGE_BUS(self, mocked_get_gateway_mfr_version): - actual_result = get_gateway_mfr_command('key') - expected_result = [ANY, 'key', '0'] - self.assertListEqual(actual_result, expected_result) - mocked_get_gateway_mfr_version.assert_called_once() - - actual_result = get_gateway_mfr_command('info') - expected_result = [ANY, 'info'] - self.assertListEqual(actual_result, expected_result) - @patch('hm_pyhelper.miner_param.get_gateway_mfr_version', return_value=Version('0.2.1')) def test_get_gateway_mfr_command_v021(self, mocked_get_gateway_mfr_version): @@ -165,8 +127,8 @@ def test_get_gateway_mfr_command_v021(self, mocked_get_gateway_mfr_version): self.assertListEqual(actual_result, expected_result) mocked_get_gateway_mfr_version.assert_called_once() - actual_result = get_gateway_mfr_command('info') - expected_result = [ANY, '--device', 'ecc://i2c-X:96?slot=0', 'info'] + actual_result = get_gateway_mfr_command('test', 9) + expected_result = [ANY, '--device', 'ecc://i2c-X:96?slot=9', 'test'] self.assertListEqual(actual_result, expected_result) @patch.dict('os.environ', {"VARIANT": "NEBHNT-INVALID"}) @@ -205,6 +167,7 @@ def test_get_gateway_mfr_command_v020(self, mocked_get_gateway_mfr_version): actual_result = get_gateway_mfr_command('info') expected_result = [ANY, '--device', 'ecc://i2c-X:96?slot=0', 'info'] + self.assertListEqual(actual_result, expected_result) @patch('hm_pyhelper.miner_param.get_gateway_mfr_version', return_value=Version('0.3.9')) @@ -216,6 +179,19 @@ def test_get_gateway_mfr_command_v021_upward(self, mocked_get_gateway_mfr_versio actual_result = get_gateway_mfr_command('info') expected_result = [ANY, '--device', 'ecc://i2c-X:96?slot=0', 'info'] + self.assertListEqual(actual_result, expected_result) + + @patch('hm_pyhelper.miner_param.get_gateway_mfr_version', + return_value=Version('0.3.9')) + def test_get_gateway_mfr_command_slot_force(self, mocked_get_gateway_mfr_version): + actual_result = get_gateway_mfr_command('key', 3) + expected_result = [ANY, '--device', 'ecc://i2c-X:96?slot=3', 'key'] + self.assertListEqual(actual_result, expected_result) + mocked_get_gateway_mfr_version.assert_called_once() + + actual_result = get_gateway_mfr_command('info', 15) + expected_result = [ANY, '--device', 'ecc://i2c-X:96?slot=15', 'info'] + self.assertListEqual(actual_result, expected_result) @patch('hm_pyhelper.miner_param.get_gateway_mfr_version', return_value=Version('0.0.0')) @@ -352,3 +328,12 @@ def test_is_spi_available(self, _): def test_error_mac_address(self): with pytest.raises(MinerFailedToFetchMacAddress): get_mac_address("test/path") + + @patch('platform.machine') + @patch('os.path.dirname') + def test_get_gateway_mfr_path(self, mock_dir, mock_platform): + mock_dir.return_value = "/test/this/works" + mock_platform.return_value = "aarch64" + actual_result = get_gateway_mfr_path() + expected_result = "/test/this/works/gateway_mfr_aarch64" + self.assertEqual(actual_result, expected_result) diff --git a/setup.py b/setup.py index faabd23..f4f2112 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name='hm_pyhelper', - version='0.13.54', + version='0.13.55', author="Nebra Ltd", author_email="support@nebra.com", description="Helium Python Helper", From 646693bbab92a9e4f9609aa152c0defc8563f7e5 Mon Sep 17 00:00:00 2001 From: Aaron Shaw Date: Mon, 13 Mar 2023 20:34:43 +0000 Subject: [PATCH 2/2] Update hardware_definitions.py --- hm_pyhelper/hardware_definitions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hm_pyhelper/hardware_definitions.py b/hm_pyhelper/hardware_definitions.py index cab3137..8f84c90 100644 --- a/hm_pyhelper/hardware_definitions.py +++ b/hm_pyhelper/hardware_definitions.py @@ -287,6 +287,7 @@ def is_raspberry_pi() -> bool: 'BALENA_DEVICE_TYPE': ['raspberrypi3-64', 'raspberrypi4-64'], 'SPIBUS': 'spidev0.0', 'SWARM_KEY_URI': 'ecc://i2c-1:96?slot=0', + 'ONBOARDING_KEY_URI': 'ecc://i2c-1:96?slot=15', 'RESET': 25, 'MAC': 'wlan0', 'STATUS': 20,