first commit

tests/models/cwm/test_configuration_cwm.py

@@ -0,0 +1,128 @@
+# Copyright 2025 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+import huggingface_hub
+
+from transformers.models.cwm import CwmConfig
+from transformers.testing_utils import require_torch
+
+from ...test_configuration_common import ConfigTester
+
+
+class CwmConfigTest(unittest.TestCase):
+    def test_default_config(self):
+        """Test default CWM configuration"""
+        config = CwmConfig()
+
+        # CWM defaults
+        self.assertEqual(config.sliding_window, 8192)
+        self.assertIsInstance(config.layer_types, list)
+
+        # Llama3 defaults
+        self.assertEqual(config.vocab_size, 128256)
+        self.assertIsNotNone(config.rope_parameters)
+        self.assertEqual(config.rope_parameters["rope_type"], "llama3")
+
+    def test_custom_sliding_window_config(self):
+        config = CwmConfig(sliding_window=4096)
+
+        self.assertEqual(config.sliding_window, 4096)
+
+    def test_custom_layer_types_config(self):
+        layer_types = ["full_attention", "sliding_attention", "sliding_attention", "full_attention"]
+        config = CwmConfig(num_hidden_layers=4, layer_types=layer_types)
+
+        self.assertEqual(config.layer_types, layer_types)
+        self.assertEqual(len(config.layer_types), config.num_hidden_layers)
+
+    def test_invalid_layer_types_length(self):
+        with self.assertRaises(huggingface_hub.errors.StrictDataclassClassValidationError):
+            CwmConfig(
+                num_hidden_layers=4,
+                layer_types=["full_attention", "sliding_attention"],  # Only 2 types for 4 layers
+            )
+
+    def test_invalid_layer_type_value(self):
+        with self.assertRaises(huggingface_hub.errors.StrictDataclassClassValidationError):
+            CwmConfig(num_hidden_layers=2, layer_types=["full_attention", "invalid_attention"])
+
+    def test_automatic_layer_types_generation(self):
+        # Test default pattern (every 4th layer uses full attention)
+        config = CwmConfig(num_hidden_layers=8)
+
+        expected_types = [
+            "full_attention",  # layer 0: 0 % 4 == 0
+            "sliding_attention",  # layer 1: 1 % 4 != 0
+            "sliding_attention",  # layer 2: 2 % 4 != 0
+            "sliding_attention",  # layer 3: 3 % 4 != 0
+            "full_attention",  # layer 4: 4 % 4 == 0
+            "sliding_attention",  # layer 5: 5 % 4 != 0
+            "sliding_attention",  # layer 6: 6 % 4 != 0
+            "sliding_attention",  # layer 7: 7 % 4 != 0
+        ]
+
+        self.assertEqual(config.layer_types, expected_types)
+
+    def test_rope_parameters_config(self):
+        custom_rope_parameters = {
+            "factor": 8.0,
+            "high_freq_factor": 2.0,
+            "low_freq_factor": 0.5,
+            "original_max_position_embeddings": 4096,
+            "rope_type": "llama3",
+            "rope_theta": 1_000_000.0,
+        }
+
+        config = CwmConfig(rope_parameters=custom_rope_parameters)
+
+        self.assertEqual(config.rope_parameters, custom_rope_parameters)
+
+    def test_config_serialization(self):
+        config = CwmConfig(
+            sliding_window=4096,
+            layer_types=["full_attention", "sliding_attention"] * 3,
+            num_hidden_layers=6,
+        )
+
+        config_dict = config.to_dict()
+        self.assertIn("sliding_window", config_dict)
+        self.assertIn("layer_types", config_dict)
+
+        new_config = CwmConfig.from_dict(config_dict)
+        self.assertEqual(new_config.sliding_window, config.sliding_window)
+        self.assertEqual(new_config.layer_types, config.layer_types)
+
+    def test_config_inheritance_from_llama(self):
+        config = CwmConfig()
+
+        # Llama config attributes
+        self.assertTrue(hasattr(config, "hidden_size"))
+        self.assertTrue(hasattr(config, "num_attention_heads"))
+        self.assertTrue(hasattr(config, "num_key_value_heads"))
+        self.assertTrue(hasattr(config, "intermediate_size"))
+        self.assertTrue(hasattr(config, "rope_parameters"))
+        self.assertTrue(hasattr(config, "attention_dropout"))
+
+
+@require_torch
+class CwmConfigTester(ConfigTester):
+    def __init__(self, parent, config_class=None, **kwargs):
+        super().__init__(parent, config_class=config_class, **kwargs)
+
+    def test_config(self):
+        config_class = CwmConfig
+        self.config_tester = ConfigTester(self, config_class=config_class)
+        self.config_tester.run_common_tests()