@ -98,6 +98,34 @@ class ComparisonRunner:
if not os . path . exists ( self . comparison_dir ) :
os . makedirs ( self . comparison_dir )
print ( " PYTHON: Attempting to load ' traced_resnet50.pth ' ... " )
try :
self . models [ ' ResNet ' ] = torch . jit . load ( ' traced_resnet50.pth ' , map_location = self . device )
print ( " PYTHON: Successfully loaded ' traced_resnet50.pth ' . " )
self . models [ ' ResNet ' ] . eval ( )
print ( " PYTHON: ResNet JIT model set to eval(). " )
except Exception as e :
print ( f " PYTHON: CRITICAL ERROR loading ' traced_resnet50.pth ' : {e} " )
self . models [ ' ResNet ' ] = None # Ensure it's None if loading failed
# Print sums of ResNet.bn1 running_mean and running_var from state_dict
print ( " PYTHON: Attempting to access ResNet state_dict (if model loaded)... " )
if self . models . get ( ' ResNet ' ) :
try :
resnet_state_dict = self . models [ ' ResNet ' ] . state_dict ( )
print ( " PYTHON ResNet state_dict keys: " , list ( resnet_state_dict . keys ( ) ) ) # PRINT ALL KEYS
py_bn1_running_mean = resnet_state_dict . get ( ' bn1.running_mean ' )
py_bn1_running_var = resnet_state_dict . get ( ' bn1.running_var ' )
if py_bn1_running_mean is not None and py_bn1_running_var is not None :
print ( f " PYTHON ResNet.bn1 running_mean sum (from state_dict): {py_bn1_running_mean.sum().item():.10f} " )
print ( f " PYTHON ResNet.bn1 running_var sum (from state_dict): {py_bn1_running_var.sum().item():.10f} " )
else :
print ( " PYTHON: ResNet.bn1 running_mean or running_var is None in state_dict. " )
except Exception as e :
print ( f " PYTHON: Error accessing ResNet.bn1 state_dict: {e} " )
# Load other models if necessary (e.g., BBRegressor, Classifier)
def load_python_models ( self ) :
print ( " DEBUG: ComparisonRunner.load_python_models() ENTERED " ) # DEBUG PRINT
""" Initialize Python models """
@ -301,6 +329,93 @@ class ComparisonRunner:
print ( " Skipping layer1.0 parameter comparison: ResNet model or its layer1 not found/empty. " )
print ( " --- END CURRENTLY USED layer1.0 PARAMS COMPARISON --- \n " ) # Corrected to \n
# --- START WEIGHT COMPARISON FOR layer1.1 and layer1.2 ---
for block_idx_in_layer1 in [ 1 , 2 ] : # For layer1.1 and layer1.2
print ( f " \n --- COMPARING CURRENTLY USED layer1.{block_idx_in_layer1} PARAMS (Python vs C++) --- " )
layer1_block_prefix = f " layer1.{block_idx_in_layer1}. "
# Components within a standard bottleneck block (no downsample for these)
block_components = {
" conv1 " : [ " weight " ] ,
" bn1 " : [ " weight " , " bias " , " running_mean " , " running_var " , " num_batches_tracked " ] ,
" conv2 " : [ " weight " ] ,
" bn2 " : [ " weight " , " bias " , " running_mean " , " running_var " , " num_batches_tracked " ] ,
" conv3 " : [ " weight " ] ,
" bn3 " : [ " weight " , " bias " , " running_mean " , " running_var " , " num_batches_tracked " ] ,
}
if self . models . get ( ' ResNet ' ) and hasattr ( self . models [ ' ResNet ' ] , ' layer1 ' ) and len ( self . models [ ' ResNet ' ] . layer1 ) > block_idx_in_layer1 :
py_layer1_block_module = self . models [ ' ResNet ' ] . layer1 [ block_idx_in_layer1 ]
for comp_name , param_list in block_components . items ( ) :
py_comp_module = py_layer1_block_module
try :
# No nested modules like 'downsample' for these blocks
py_comp_module = getattr ( py_comp_module , comp_name )
except AttributeError :
print ( f " Python ResNet model ' s layer1.{block_idx_in_layer1} does not have component {comp_name}. Skipping. " )
continue
for p_name in param_list :
py_param_tensor_name = f " {layer1_block_prefix}{comp_name}.{p_name} "
# C++ saves files like layer1_0_bn1_weight.pt or layer1_1_bn1_weight.pt
cpp_param_filename = f " {layer1_block_prefix.replace( ' . ' , ' _ ' )}{comp_name.replace( ' . ' , ' _ ' )}_{p_name}.pt "
py_param_tensor = None
if hasattr ( py_comp_module , p_name ) :
param_tensor_val = getattr ( py_comp_module , p_name )
if param_tensor_val is not None :
py_param_tensor = param_tensor_val . detach ( ) . cpu ( )
print ( f " Python ResNet {py_param_tensor_name} shape: {py_param_tensor.shape} " )
else :
print ( f " Python ResNet {py_param_tensor_name} is None. " )
elif p_name == " num_batches_tracked " and isinstance ( py_comp_module , torch . nn . BatchNorm2d ) :
# PyTorch stores num_batches_tracked in _buffers, not as a direct attribute usually
if py_comp_module . num_batches_tracked is not None :
py_param_tensor = py_comp_module . num_batches_tracked . detach ( ) . cpu ( )
print ( f " Python ResNet {py_param_tensor_name} (from buffer) shape: {py_param_tensor.shape} " )
else :
print ( f " Python ResNet {py_param_tensor_name} (from buffer) is None. " )
else :
print ( f " Python ResNet module {comp_name} does not have param/buffer {p_name}. " )
cpp_param_path = os . path . join ( self . root_dir , " exported_weights/backbone_regenerated " , cpp_param_filename )
cpp_param_tensor = None
if os . path . exists ( cpp_param_path ) :
try :
cpp_param_tensor = torch . load ( cpp_param_path , map_location = ' cpu ' , weights_only = False )
except Exception as e :
print ( f " Error loading C++ {cpp_param_filename} from {cpp_param_path}: {e} " )
else :
print ( f " Warning: C++ {cpp_param_filename} file not found: {cpp_param_path} " )
print ( f " Comparison for {py_param_tensor_name} vs {cpp_param_filename}: " )
if py_param_tensor is not None and cpp_param_tensor is not None :
if isinstance ( py_param_tensor , torch . Tensor ) and isinstance ( cpp_param_tensor , torch . Tensor ) :
# Ensure tensors are float for allclose if one is int (e.g. num_batches_tracked)
py_param_tensor_float = py_param_tensor . float ( )
cpp_param_tensor_float = cpp_param_tensor . float ( )
all_close = torch . allclose ( py_param_tensor_float , cpp_param_tensor_float )
print ( f " torch.allclose: {all_close} " )
if not all_close :
abs_diff = torch . abs ( py_param_tensor_float - cpp_param_tensor_float )
mae = torch . mean ( abs_diff ) . item ( )
max_abs_err = torch . max ( abs_diff ) . item ( )
print ( f " MAE (Weight/Buffer): {mae:.4e} " )
print ( f " Max Abs Err (Weight/Buffer): {max_abs_err:.4e} " )
# Also print L2 norms for context
l2_py = torch . linalg . norm ( py_param_tensor_float . flatten ( ) ) . item ( )
l2_cpp = torch . linalg . norm ( cpp_param_tensor_float . flatten ( ) ) . item ( )
print ( f " L2 Norm Python: {l2_py:.4e} " )
print ( f " L2 Norm C++: {l2_cpp:.4e} " )
else :
print ( f " Skipping comparison due to type mismatch after loading for {py_param_tensor_name}. " )
else :
print ( f " Skipping comparison because one or both tensors could not be obtained for {py_param_tensor_name}. " )
else :
print ( f " Skipping layer1.{block_idx_in_layer1} parameter comparison: ResNet model or its layer1 not found/long enough. " )
print ( f " --- END CURRENTLY USED layer1.{block_idx_in_layer1} PARAMS COMPARISON --- \n " )
# --- END WEIGHT COMPARISON FOR layer1.1 and layer1.2 ---
# --- END TEMPORARY WEIGHT COMPARISON --- # This marker is now after layer1.0 checks
print ( " \n --- Types at END of load_python_models: --- " )
@ -524,14 +639,32 @@ class ComparisonRunner:
if ' Classifier ' in self . models : print ( f " self.models[ ' Classifier ' ] type: {type(self.models[ ' Classifier ' ])} " )
if ' BBRegressor ' in self . models : print ( f " self.models[ ' BBRegressor ' ] type: {type(self.models[ ' BBRegressor ' ])} " )
print ( " \n Comparing ResNet outputs... " )
py_input_common_dir = os . path . join ( self . root_dir , ' test ' , ' input_samples ' , ' common ' )
cpp_output_resnet_dir = os . path . join ( self . cpp_output_dir , ' resnet ' )
# Ensure self.py_resnet_output_dir is defined, e.g., in __init__ or where other py output dirs are
if not hasattr ( self , ' py_resnet_output_dir ' ) or not self . py_resnet_output_dir :
self . py_resnet_output_dir = Path ( self . python_output_dir ) / ' resnet '
self . py_resnet_output_dir . mkdir ( parents = True , exist_ok = True )
# Convert to Path objects for exists() check, though os.path.exists also works with strings
# Define Path objects for directory check s
py_input_common_dir_path = Path ( py_input_common_dir )
cpp_output_resnet_dir_path = Path ( cpp_output_resnet_dir )
comparison_configs = [
( " ResNet Conv1 Output (Pre-BN) " , " _conv1_output_py.pt " , " _conv1_output.pt " , self . py_resnet_output_dir , cpp_output_resnet_dir ) ,
( " ResNet Conv1 " , " _conv1_output.pt " , " _conv1_output.pt " , self . py_resnet_output_dir , cpp_output_resnet_dir ) , # Assumes Py also saved conv1 output if it was meant to be same as C++ pre-bn
( " ResNet BN1 " , " _bn1_output.pt " , " _bn1_output.pt " , self . py_resnet_output_dir , cpp_output_resnet_dir ) ,
( " ResNet ReLU1 " , " _relu1_output.pt " , " _relu1_output.pt " , self . py_resnet_output_dir , cpp_output_resnet_dir ) ,
( " ResNet MaxPool " , " _maxpool_output.pt " , " _maxpool_output.pt " , self . py_resnet_output_dir , cpp_output_resnet_dir ) ,
( " ResNet Layer1.0 Block Output " , " _layer1_0_block_output.pt " , " _layer1_0_block_output.pt " , self . py_resnet_output_dir , cpp_output_resnet_dir ) ,
( " ResNet Layer1.0 Shortcut Output " , " _layer1_0_shortcut_output.pt " , " _layer1_0_shortcut_output.pt " , self . py_resnet_output_dir , cpp_output_resnet_dir ) ,
( " ResNet Layer1 " , " _layer1_output.pt " , " _layer1_output.pt " , self . py_resnet_output_dir , cpp_output_resnet_dir ) ,
( " ResNet Layer2 " , " _layer2_output.pt " , " _layer2_output.pt " , self . py_resnet_output_dir , cpp_output_resnet_dir ) ,
( " ResNet Layer3 " , " _layer3_output.pt " , " _layer3_output.pt " , self . py_resnet_output_dir , cpp_output_resnet_dir ) ,
( " ResNet Layer4 " , " _layer4_output.pt " , " _layer4_output.pt " , self . py_resnet_output_dir , cpp_output_resnet_dir ) ,
( " ResNet Features " , " _features_output.pt " , " _features_output.pt " , self . py_resnet_output_dir , cpp_output_resnet_dir )
]
if not py_input_common_dir_path . exists ( ) or not cpp_output_resnet_dir_path . exists ( ) :
print ( f " ResNet input ({py_input_common_dir_path}) or C++ ResNet output dir ({cpp_output_resnet_dir_path}) not found. Skipping ResNet comparison. " )
# Populate NaN for all expected ResNet comparisons if dirs are missing
@ -584,14 +717,45 @@ class ComparisonRunner:
with torch . no_grad ( ) :
py_model_resnet = self . models . get ( ' ResNet ' )
if py_model_resnet :
# Original GPU path for all Python ResNet layers
current_features = preprocessed_py_image_for_conv1 # Start with preprocessed image
current_features = preprocessed_py_image_for_conv1
py_conv1_out = py_model_resnet . conv1 ( current_features )
py_bn1_out = py_model_resnet . bn1 ( py_conv1_out . clone ( ) )
py_relu1_out = py_model_resnet . relu ( py_bn1_out . clone ( ) )
py_maxpool_out = py_model_resnet . maxpool ( py_relu1_out . clone ( ) )
x_for_py_layer1_input = py_maxpool_out # This is the input to layer1 block
# Ensure self.py_resnet_output_dir is defined and is a Path object
if not hasattr ( self , ' py_resnet_output_dir ' ) or not self . py_resnet_output_dir :
self . py_resnet_output_dir = Path ( self . python_output_dir ) / ' resnet '
self . py_resnet_output_dir . mkdir ( parents = True , exist_ok = True )
py_conv1_out_path = self . py_resnet_output_dir / f ' sample_{i}_conv1_output_py.pt '
torch . save ( py_conv1_out . cpu ( ) , str ( py_conv1_out_path ) )
# --- BN1 on CPU for debugging (Python) ---
py_bn1_out = py_model_resnet . bn1 ( py_conv1_out ) # Original line
py_relu1_out = py_model_resnet . relu ( py_bn1_out )
py_maxpool_out = py_model_resnet . maxpool ( py_relu1_out )
x_for_py_layer1_input = py_maxpool_out
# Output of the first bottleneck block in layer1
py_layer1_0_block_out_tensor = None # Initialize to avoid ref before assignment if try fails
if hasattr ( py_model_resnet , ' layer1 ' ) and len ( py_model_resnet . layer1 ) > 0 :
try :
py_layer1_0_block_out_tensor = py_model_resnet . layer1 [ 0 ] ( x_for_py_layer1_input ) # REMOVED .clone() for consistency with best Layer1.0 result
# Ensure cpp_resnet_sample_dir is defined, if not, use a fallback or define it earlier
# Assuming cpp_resnet_sample_dir is defined like: cpp_resnet_sample_dir = Path(self.cpp_output_dir) / 'resnet'
# Which should be: cpp_resnet_dir = Path(self.cpp_output_dir) / 'resnet' # as per usage elsewhere
# And then: cpp_resnet_sample_dir = cpp_resnet_dir # if sample specific subdirs are not used for this
# For safety, let's use the already established cpp_output_resnet_dir path from later in the code
# cpp_output_resnet_dir = os.path.join(self.cpp_output_dir, 'resnet')
# Need to ensure cpp_output_resnet_dir is a Path object if used with /
# From later code: cpp_output_resnet_dir_path = Path(self.cpp_output_dir) / 'resnet'
current_cpp_resnet_dir = Path ( self . cpp_output_dir ) / ' resnet ' # Define it based on existing patterns
current_cpp_resnet_dir . mkdir ( parents = True , exist_ok = True ) # Ensure directory exists
py_layer1_0_block_save_path = current_cpp_resnet_dir / f ' sample_{i}_layer1_0_block_output.pt '
torch . save ( py_layer1_0_block_out_tensor . cpu ( ) , str ( py_layer1_0_block_save_path ) )
# print(f"DEBUG: Saved Python layer1[0] block output for sample {i} to {py_layer1_0_block_save_path}")
except Exception as e_block :
print ( f " ERROR: Failed to get/save Python layer1[0] block output for sample {i}: {e_block} " )
# Shortcut for layer1.0 (if exists)
if hasattr ( py_model_resnet , ' layer1 ' ) and len ( py_model_resnet . layer1 ) > 0 and \
@ -646,6 +810,7 @@ class ComparisonRunner:
cpp_conv1_path = os . path . join ( cpp_output_resnet_dir , f ' sample_{i}_conv1_output.pt ' ) # ADDED
cpp_bn1_path = os . path . join ( cpp_output_resnet_dir , f ' sample_{i}_bn1_output.pt ' ) # ADDED
cpp_relu1_path = os . path . join ( cpp_output_resnet_dir , f ' sample_{i}_relu1_output.pt ' ) # ADDED
cpp_layer1_0_block_output_path = os . path . join ( cpp_output_resnet_dir , f ' sample_{i}_layer1_0_block_output.pt ' ) # ADDED
cpp_layer1_out = self . load_cpp_tensor ( cpp_layer1_path , self . device )
cpp_layer2_out = self . load_cpp_tensor ( cpp_layer2_path , self . device )
@ -657,17 +822,33 @@ class ComparisonRunner:
cpp_conv1_out = self . load_cpp_tensor ( cpp_conv1_path , self . device ) # ADDED
cpp_bn1_out = self . load_cpp_tensor ( cpp_bn1_path , self . device ) # ADDED
cpp_relu1_out = self . load_cpp_tensor ( cpp_relu1_path , self . device ) # ADDED
cpp_layer1_0_block_output_tensor = self . load_cpp_tensor ( cpp_layer1_0_block_output_path , self . device ) # ADDED
# Load the Python pre-BN conv1 output that was saved earlier
py_conv1_out_pre_bn_tensor = None
# Ensure self.py_resnet_output_dir is defined (it should be if the save operation worked)
if hasattr ( self , ' py_resnet_output_dir ' ) and self . py_resnet_output_dir :
py_conv1_out_pre_bn_path = self . py_resnet_output_dir / f ' sample_{i}_conv1_output_py.pt '
if py_conv1_out_pre_bn_path . exists ( ) :
try :
py_conv1_out_pre_bn_tensor = torch . load ( str ( py_conv1_out_pre_bn_path ) , map_location = self . device )
except Exception as e_load_py_conv1 :
print ( f " Error loading Python conv1_output_py (pre-BN) for sample {i}: {e_load_py_conv1} " )
else :
print ( f " Warning: self.py_resnet_output_dir not defined, cannot load py_conv1_output_py.pt for sample {i} " )
self . _compare_tensor_data ( py_conv1_out , cpp_conv1_out , " ResNet Conv1 " , i , current_errors ) # REVERTED to py_conv1_out
self . _compare_tensor_data ( py_bn1_out , cpp_bn1_out , " ResNet BN1 " , i , current_errors )
self . _compare_tensor_data ( py_relu1_out , cpp_relu1_out , " ResNet ReLU1 " , i , current_errors )
# Comparisons
self . _compare_tensor_data ( py_conv1_out_pre_bn_tensor , cpp_conv1_out , " ResNet Conv1 Output (Pre-BN) " , i , current_errors )
self . _compare_tensor_data ( py_conv1_out , cpp_conv1_out , " ResNet Conv1 " , i , current_errors )
self . _compare_tensor_data ( py_bn1_out , cpp_bn1_out , " ResNet BN1 " , i , current_errors )
self . _compare_tensor_data ( py_relu1_out , cpp_relu1_out , " ResNet ReLU1 " , i , current_errors )
self . _compare_tensor_data ( py_maxpool_out , cpp_maxpool_out , " ResNet MaxPool " , i , current_errors )
self . _compare_tensor_data ( py_layer1_out , cpp_layer1_out , " ResNet Layer1 " , i , current_errors )
self . _compare_tensor_data ( py_layer1_0_shortcut_out , cpp_layer1_0_shortcut_out , " ResNet Layer1.0 Shortcut " , i , current_errors )
self . _compare_tensor_data ( py_layer2_out , cpp_layer2_out , " ResNet Layer2 " , i , current_errors )
self . _compare_tensor_data ( py_layer3_out , cpp_layer3_out , " ResNet Layer3 " , i , current_errors )
self . _compare_tensor_data ( py_layer4_out , cpp_layer4_out , " ResNet Layer4 " , i , current_errors )
self . _compare_tensor_data ( py_features_out , cpp_features_out , " ResNet Features " , i , current_errors )
self . _compare_tensor_data ( py_layer1_0_shortcut_out , cpp_layer1_0_shortcut_out , " ResNet Layer1.0 Shortcut " , i , current_errors )
if current_errors : self . all_comparison_stats [ f " ResNet_Sample_{i} " ] = current_errors
mht
2 months ago