{"preflight_layer": 0, "seconds": 2.0485799312591553} {"preflight_layer": 1, "seconds": 4.110389947891235} {"preflight_layer": 2, "seconds": 5.677527189254761} {"preflight_layer": 3, "seconds": 6.754502058029175} {"preflight_layer": 4, "seconds": 8.340956926345825} {"preflight_layer": 5, "seconds": 10.697716236114502} {"preflight_layer": 6, "seconds": 12.576658964157104} {"preflight_layer": 7, "seconds": 14.14732575416565} Traceback (most recent call last): File "", line 198, in _run_module_as_main File "", line 88, in _run_code File "/opt/exp-model-parameter-similarity/functional/preflight.py", line 27, in if __name__=='__main__':main() ~~~~^^ File "/opt/exp-model-parameter-similarity/functional/preflight.py", line 18, in main h,_=lm.forward(ids[:1,:8],progress=progress) ~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/opt/exp-model-parameter-similarity/functional/model.py", line 215, in forward h = h + self.mlp(n,donor) ~~~~~~~~^^^^^^^^^ File "/opt/exp-model-parameter-similarity/functional/model.py", line 149, in mlp gate = silu(self.linear(h, p+'gate_proj')) ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ File "/opt/exp-model-parameter-similarity/functional/model.py", line 127, in linear self.guard() ~~~~~~~~~~^^ File "/opt/exp-model-parameter-similarity/functional/model.py", line 108, in guard raise MemoryError('Sustained host swap IO above 4 MiB/s in three observation windows') MemoryError: Sustained host swap IO above 4 MiB/s in three observation windows