diff --git a/.github/workflows/master-slugs.yml b/.github/workflows/master-slugs.yml deleted file mode 100644 index 0c43d494a..000000000 --- a/.github/workflows/master-slugs.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Create Master Slug List on PR Merge -on: - pull_request: - types: - - closed - branches: - - master -jobs: - build: - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - steps: - - uses: actions/checkout@v3 - - name: Setup Python - uses: actions/setup-python@v3 - with: - python-version: 3.8 - - name: Install dependencies - run: pip install -r requirements.txt - - name: Regenerate Master Slug List - run: python3 tests/generate-slug-list.py - - name: Commit and Push Changes to Master - uses: EndBug/add-and-commit@v9 - with: - author_name: NetBox-Bot - author_email: info@netboxlabs.com - committer_name: NetBox-Bot - committer_email: info@netboxlabs.com - default_author: github_actions - message: "Regenerate master slug list after successful PR merge" - push: true diff --git a/requirements.txt b/requirements.txt index 6251a75a6..0142f2e90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,3 @@ pre-commit==3.3.3 pytest==7.4.0 PyYAML==6.0 yamllint==1.32.0 -gitpython==3.1.32 \ No newline at end of file diff --git a/tests/definitions_test.py b/tests/definitions_test.py index 98875763c..6d94fef32 100644 --- a/tests/definitions_test.py +++ b/tests/definitions_test.py @@ -1,19 +1,16 @@ -from test_configuration import COMPONENT_TYPES, IMAGE_FILETYPES, SCHEMAS, KNOWN_SLUGS, ROOT_DIR, USE_LOCAL_KNOWN_SLUGS, NETBOX_DT_LIBRARY_URL -import pickle_operations +from test_configuration import COMPONENT_TYPES, IMAGE_FILETYPES, SCHEMAS from yaml_loader import DecimalSafeLoader from device_types import DeviceType, ModuleType, verify_filename, validate_components import decimal import glob import json import os -import tempfile from urllib.request import urlopen import pytest import yaml from jsonschema import Draft4Validator, RefResolver from jsonschema.exceptions import ValidationError -from git import Repo def _get_definition_files(): """ @@ -35,29 +32,6 @@ def _get_definition_files(): return file_list -def _get_diff_from_upstream(): - file_list = [] - - repo = Repo(f"{os.path.dirname(os.path.abspath(__file__))}/../") - upstream = repo.remotes.upstream - upstream.fetch() - changes = repo.head.commit.diff(upstream.refs["master"].object.hexsha) - - for path, schema in SCHEMAS: - # Initialize the schema - with open(f"schema/{schema}") as schema_file: - schema = json.loads(schema_file.read(), parse_float=decimal.Decimal) - - # Validate that the schema exists - assert schema, f"Schema definition for {path} is empty!" - - for file in changes: - if file.b_path is not None: - if path in file.b_path: - file_list.append((file.b_path, schema)) - - return file_list - def _get_image_files(): """ Return a list of all image files within the specified path and manufacturer. @@ -87,19 +61,11 @@ def test_environment(): Run basic sanity checks on the environment to ensure tests are running correctly. """ # Validate that definition files exist - if definition_files: - pytest.skip("No changes to definition files found.") + assert definition_files, "No definition files found!" -definition_files = _get_diff_from_upstream() +definition_files = _get_definition_files() image_files = _get_image_files() -if USE_LOCAL_KNOWN_SLUGS: - KNOWN_SLUGS = pickle_operations.read_pickle_data(f'{ROOT_DIR}/tests/known-slugs.pickle') -else: - temp_dir = tempfile.TemporaryDirectory() - repo = Repo.clone_from(url=NETBOX_DT_LIBRARY_URL, to_path=temp_dir.name) - KNOWN_SLUGS = pickle_operations.read_pickle_data(f'{temp_dir.name}/tests/known-slugs.pickle') - @pytest.mark.parametrize(('file_path', 'schema'), definition_files) def test_definitions(file_path, schema): """ @@ -141,7 +107,7 @@ def test_definitions(file_path, schema): # Verify the slug is valid, only if the definition type is a Device if this_device.isDevice: - assert this_device.verify_slug(KNOWN_SLUGS), pytest.fail(this_device.failureMessage, False) + assert this_device.verify_slug(), pytest.fail(this_device.failureMessage, False) # Verify the filename is valid. Must either be the model or part_number. assert verify_filename(this_device), pytest.fail(this_device.failureMessage, False) diff --git a/tests/device_types.py b/tests/device_types.py index 7b58cd0e7..9a6c07a3c 100644 --- a/tests/device_types.py +++ b/tests/device_types.py @@ -1,3 +1,4 @@ +from test_configuration import KNOWN_SLUGS import os class DeviceType: @@ -40,18 +41,9 @@ class DeviceType: def get_filepath(self): return self.file_path - def verify_slug(self, KNOWN_SLUGS): + def verify_slug(self): # Verify the slug is unique, and not already known - known_slug_list_intersect = [(slug, file_path) for slug, file_path in KNOWN_SLUGS if slug == self.slug] - - if len(known_slug_list_intersect) == 0: - pass - elif len(known_slug_list_intersect) == 1: - if self.file_path not in known_slug_list_intersect[0][1]: - self.failureMessage = f'{self.file_path} has a duplicate slug: "{self.slug}"' - return False - return True - else: + if self.slug in KNOWN_SLUGS: self.failureMessage = f'{self.file_path} has a duplicate slug "{self.slug}"' return False @@ -66,7 +58,7 @@ class DeviceType: return False # Add the slug to the list of known slugs - KNOWN_SLUGS.add((self.slug, self.file_path)) + KNOWN_SLUGS.add(self.slug) return True def validate_power(self): diff --git a/tests/generate-slug-list.py b/tests/generate-slug-list.py deleted file mode 100644 index 7cf3f1d0e..000000000 --- a/tests/generate-slug-list.py +++ /dev/null @@ -1,90 +0,0 @@ -import os -import json -import glob -import yaml -import decimal -from yaml_loader import DecimalSafeLoader -from jsonschema import Draft4Validator, RefResolver -from jsonschema.exceptions import ValidationError -from test_configuration import SCHEMAS, KNOWN_SLUGS, ROOT_DIR -from urllib.request import urlopen -import pickle_operations - -def _get_device_type_files(): - """ - Return a list of all definition files within the specified path. - """ - file_list = [] - - for path, schema in SCHEMAS: - if path == 'device-types': - # Initialize the schema - with open(f"{ROOT_DIR}/schema/{schema}") as schema_file: - schema = json.loads(schema_file.read(), - parse_float=decimal.Decimal) - - # Validate that the schema exists - if not schema: - print(f"Schema definition for {path} is empty!") - exit(1) - - # Map each definition file to its schema as a tuple (file, schema) - for file in sorted(glob.glob(f"{path}/*/*", recursive=True)): - file_list.append((f'{file}', schema)) - - return file_list - -def _decimal_file_handler(uri): - """ - Handler to work with floating decimals that fail normal validation. - """ - with urlopen(uri) as url: - result = json.loads(url.read().decode("utf-8"), parse_float=decimal.Decimal) - return result - -def load_file(file_path, schema): - # Read file - try: - with open(file_path) as definition_file: - content = definition_file.read() - except Exception as exc: - return (False, f'Error opening "{file_path}". stderr: {exc}') - - # Check for trailing newline. YAML files must end with an emtpy newline. - if not content.endswith('\n'): - return (False, f'{file_path} is missing trailing newline') - - # Load YAML data from file - try: - definition = yaml.load(content, Loader=DecimalSafeLoader) - except Exception as exc: - return (False, f'Error during yaml.load "{file_path}". stderr: {exc}') - - # Validate YAML definition against the supplied schema - try: - resolver = RefResolver( - f"file://{os.getcwd()}/schema/devicetype.json", - schema, - handlers={"file": _decimal_file_handler}, - ) - # Validate definition against schema - Draft4Validator(schema, resolver=resolver).validate(definition) - except ValidationError as exc: - # Schema validation failure. Ensure you are following the proper format. - return (False, f'{file_path} failed validation: {exc}') - - return (True, definition) - -def _generate_known_slugs(): - all_files = _get_device_type_files() - - for file_path, schema in all_files: - definition_status, definition = load_file(file_path, schema) - if not definition_status: - print(definition) - exit(1) - - KNOWN_SLUGS.add((definition.get('slug'), file_path)) - -_generate_known_slugs() -pickle_operations.write_pickle_data(KNOWN_SLUGS, f'{ROOT_DIR}/tests/known-slugs.pickle') \ No newline at end of file diff --git a/tests/known-slugs.pickle b/tests/known-slugs.pickle deleted file mode 100644 index 801dc573a..000000000 Binary files a/tests/known-slugs.pickle and /dev/null differ diff --git a/tests/pickle_operations.py b/tests/pickle_operations.py deleted file mode 100644 index 193dd4da5..000000000 --- a/tests/pickle_operations.py +++ /dev/null @@ -1,14 +0,0 @@ -import pickle - -def write_pickle_data(data, file_path): - with open(file_path, 'wb') as pickle_file: - pickle.dump(data, pickle_file) - pickle_file.close() - - -def read_pickle_data(file_path): - with open(file_path, 'rb') as pickle_file: - data = pickle.load(pickle_file) - pickle_file.close() - - return data diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 5f90a0f44..68edcaf5d 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -1,5 +1,3 @@ -import os - SCHEMAS = ( ('device-types', 'devicetype.json'), ('module-types', 'moduletype.json'), @@ -21,10 +19,4 @@ COMPONENT_TYPES = ( 'module-bays', ) -ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname( __file__ ), '..')) - -KNOWN_SLUGS = set() - -USE_LOCAL_KNOWN_SLUGS = False - -NETBOX_DT_LIBRARY_URL = "https://github.com/netbox-community/devicetype-library.git" \ No newline at end of file +KNOWN_SLUGS = set() \ No newline at end of file