{"layer": 0, "seconds": 3.669800043106079, "rss_kib": 117576, "process_swap_kib": 0, "available_kib": 975920} {"layer": 1, "seconds": 6.548107862472534, "rss_kib": 117440, "process_swap_kib": 0, "available_kib": 984012} {"layer": 2, "seconds": 9.072005033493042, "rss_kib": 115956, "process_swap_kib": 0, "available_kib": 925720} {"layer": 3, "seconds": 10.377433776855469, "rss_kib": 114984, "process_swap_kib": 0, "available_kib": 958016} {"layer": 4, "seconds": 11.7694993019104, "rss_kib": 108236, "process_swap_kib": 0, "available_kib": 1055012} {"layer": 5, "seconds": 13.289605379104614, "rss_kib": 108244, "process_swap_kib": 0, "available_kib": 1056400} {"layer": 6, "seconds": 15.41482949256897, "rss_kib": 84512, "process_swap_kib": 23736, "available_kib": 1072308} Traceback (most recent call last): File "", line 198, in _run_module_as_main File "", line 88, in _run_code File "/opt/exp-model-parameter-similarity/functional/preflight.py", line 39, in if __name__=='__main__':main() ~~~~^^ File "/opt/exp-model-parameter-similarity/functional/preflight.py", line 26, in main h,_=lm.forward(ids[:1,:8],progress=progress) ~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/opt/exp-model-parameter-similarity/functional/model.py", line 201, in forward self.guard() ~~~~~~~~~~^^ File "/opt/exp-model-parameter-similarity/functional/model.py", line 109, in guard raise MemoryError(f'Sustained host swap pressure plus constrained memory: available={available} KiB, process_swap={own_swap_kib} KiB') MemoryError: Sustained host swap pressure plus constrained memory: available=1071348 KiB, process_swap=23640 KiB