Fabian
74 lines
2.5 KiB
Python
74 lines
2.5 KiB
Python
"""Test if torch.cuda.synchronize() crashes after F.linear on gfx1010"""
|
|
import torch
|
|
import torch.nn.functional as F
|
|
import time, os, signal, sys
|
|
|
|
os.environ['HSA_OVERRIDE_GFX_VERSION'] = '10.1.0'
|
|
os.environ['HSA_ENABLE_SDMA'] = '0'
|
|
os.environ['TORCHDYNAMO_DISABLE'] = '1'
|
|
|
|
# Catch SIGSEGV
|
|
def segfault_handler(signum, frame):
|
|
print(f"\n*** SIGNAL {signum} (SIGSEGV) CAUGHT ***", flush=True)
|
|
sys.exit(1)
|
|
signal.signal(signal.SIGSEGV, segfault_handler)
|
|
|
|
print(f"Device: {torch.cuda.get_device_name(0)}", flush=True)
|
|
|
|
# Simulate exactly what the model does:
|
|
# For each layer: transfer from CPU → F.linear → sync → next layer
|
|
shapes = [
|
|
("CBW505", (1024, 256)),
|
|
("CBW506", (256, 1024)),
|
|
("CBW507", (3840, 2560)),
|
|
("CBW508", (3840, 64)),
|
|
("CBW509", (11520, 3840)), # This is where ComfyUI crashes
|
|
("CBW510", (3840, 3840)),
|
|
("CBW511", (3840, 3840)),
|
|
("CBW512", (11520, 3840)),
|
|
("CBW513", (3840, 11520)),
|
|
]
|
|
|
|
# Start with latent-like input
|
|
x = torch.randn(1, 1024, 256, dtype=torch.float16, device='cuda:0')
|
|
torch.cuda.synchronize()
|
|
print("Initial input on GPU", flush=True)
|
|
|
|
for i, (name, (out_feat, in_feat)) in enumerate(shapes):
|
|
print(f"\n--- {name}: Linear({in_feat} -> {out_feat}) ---", flush=True)
|
|
|
|
# Step 1: Pre-sync (like v20 patch)
|
|
print(f" pre-sync...", flush=True)
|
|
torch.cuda.synchronize()
|
|
print(f" pre-sync OK", flush=True)
|
|
|
|
# Step 2: CPU dequant simulation → transfer to GPU
|
|
w_cpu = torch.randn(out_feat, in_feat, dtype=torch.float16)
|
|
b_cpu = torch.randn(out_feat, dtype=torch.float16)
|
|
print(f" CPU weights created", flush=True)
|
|
|
|
w_gpu = w_cpu.to('cuda:0', non_blocking=False)
|
|
b_gpu = b_cpu.to('cuda:0', non_blocking=False)
|
|
print(f" transferred to GPU", flush=True)
|
|
|
|
# Step 3: Adjust input dimension
|
|
if x.shape[-1] != in_feat:
|
|
x = torch.randn(1, 1024, in_feat, dtype=torch.float16, device='cuda:0')
|
|
torch.cuda.synchronize()
|
|
print(f" input resized to [{1}, {1024}, {in_feat}]", flush=True)
|
|
|
|
# Step 4: F.linear (dispatches GPU kernel)
|
|
print(f" F.linear starting...", flush=True)
|
|
x = F.linear(x, w_gpu, b_gpu)
|
|
print(f" F.linear dispatched, out={list(x.shape)}", flush=True)
|
|
|
|
# Step 5: Post-sync (forces GPU to finish and report errors)
|
|
print(f" post-sync...", flush=True)
|
|
torch.cuda.synchronize()
|
|
print(f" post-sync OK", flush=True)
|
|
|
|
del w_cpu, b_cpu, w_gpu, b_gpu
|
|
print(f" {name} DONE", flush=True)
|
|
|
|
print("\n=== ALL PASSED ===")
|