From a00505e835ee7f0c7c31a77b38d445bfbcc77845 Mon Sep 17 00:00:00 2001
From: alexander-darwiche
 <59787022+alexander-darwiche@users.noreply.github.com>
Date: Tue, 12 Nov 2024 15:02:51 -0500
Subject: [PATCH 1/5] Adding in new convolutional weight standardization

---
 src/resnet.py | 23 ++++++++++++++++++++---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/resnet.py b/src/resnet.py
index 4a755bb..848e42f 100644
--- a/src/resnet.py
+++ b/src/resnet.py
@@ -36,6 +36,23 @@
 
 __all__ = ['ResNet', 'resnet20']
 
+class Conv2d(nn.Conv2d):
+
+    def __init__(self, in_channels, out_channels, kernel_size, stride=1,
+                 padding=0, dilation=1, groups=1, bias=True):
+        super(Conv2d, self).__init__(in_channels, out_channels, kernel_size, stride,
+                 padding, dilation, groups, bias)
+
+    def forward(self, x):
+        weight = self.weight
+        weight_mean = weight.mean(dim=1, keepdim=True).mean(dim=2,
+                                  keepdim=True).mean(dim=3, keepdim=True)
+        weight = weight - weight_mean
+        std = weight.view(weight.size(0), -1).std(dim=1).view(-1, 1, 1, 1) + 1e-5
+        weight = weight / std.expand_as(weight)
+        return F.conv2d(x, weight, self.bias, self.stride,
+                        self.padding, self.dilation, self.groups)
+
 def _weights_init(m):
     classname = m.__class__.__name__
     #print(classname)
@@ -59,9 +76,9 @@ class BasicBlock(nn.Module):
 
     def __init__(self, in_planes, planes, stride=1, option='A'):
         super(BasicBlock, self).__init__()
-        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
+        self.conv1 = Conv2d(nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False))
         self.bn1 = nn.BatchNorm2d(planes)
-        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
+        self.conv2 = Conv2d(nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False))
         self.bn2 = nn.BatchNorm2d(planes)
 
         self.shortcut = nn.Sequential()
@@ -73,7 +90,7 @@ def __init__(self, in_planes, planes, stride=1, option='A'):
                 self.shortcut = ShortcutLayer(stride, in_planes, planes)
             elif option == 'B':
                 self.shortcut = nn.Sequential(
-                     nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False),
+                     Conv2d(nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False)),
                      nn.BatchNorm2d(self.expansion * planes)
                 )
 

From bccc80c348c2470d279060e26b0f95ee315359ed Mon Sep 17 00:00:00 2001
From: alexander-darwiche
 <59787022+alexander-darwiche@users.noreply.github.com>
Date: Tue, 12 Nov 2024 15:10:31 -0500
Subject: [PATCH 2/5] Removing wrappers on Conv2d

---
 src/resnet.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/resnet.py b/src/resnet.py
index 848e42f..5143577 100644
--- a/src/resnet.py
+++ b/src/resnet.py
@@ -76,9 +76,9 @@ class BasicBlock(nn.Module):
 
     def __init__(self, in_planes, planes, stride=1, option='A'):
         super(BasicBlock, self).__init__()
-        self.conv1 = Conv2d(nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False))
+        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
         self.bn1 = nn.BatchNorm2d(planes)
-        self.conv2 = Conv2d(nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False))
+        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
         self.bn2 = nn.BatchNorm2d(planes)
 
         self.shortcut = nn.Sequential()
@@ -90,7 +90,7 @@ def __init__(self, in_planes, planes, stride=1, option='A'):
                 self.shortcut = ShortcutLayer(stride, in_planes, planes)
             elif option == 'B':
                 self.shortcut = nn.Sequential(
-                     Conv2d(nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False)),
+                     nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False),
                      nn.BatchNorm2d(self.expansion * planes)
                 )
 

From 265eb1231d1492fce2c522fe3734db84163e5406 Mon Sep 17 00:00:00 2001
From: alexander-darwiche
 <59787022+alexander-darwiche@users.noreply.github.com>
Date: Tue, 12 Nov 2024 15:31:38 -0500
Subject: [PATCH 3/5] test print statement

---
 src/resnet.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/resnet.py b/src/resnet.py
index 5143577..8fc1ec0 100644
--- a/src/resnet.py
+++ b/src/resnet.py
@@ -42,6 +42,7 @@ def __init__(self, in_channels, out_channels, kernel_size, stride=1,
                  padding=0, dilation=1, groups=1, bias=True):
         super(Conv2d, self).__init__(in_channels, out_channels, kernel_size, stride,
                  padding, dilation, groups, bias)
+        print('redefining conv2d')
 
     def forward(self, x):
         weight = self.weight

From 084c9e789e4c21662f105ecf99602ad48cd39061 Mon Sep 17 00:00:00 2001
From: alexander-darwiche
 <59787022+alexander-darwiche@users.noreply.github.com>
Date: Tue, 12 Nov 2024 15:45:25 -0500
Subject: [PATCH 4/5] Swapping nn.conv2d to conv2d

---
 src/resnet.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/resnet.py b/src/resnet.py
index 8fc1ec0..6763cfb 100644
--- a/src/resnet.py
+++ b/src/resnet.py
@@ -77,9 +77,9 @@ class BasicBlock(nn.Module):
 
     def __init__(self, in_planes, planes, stride=1, option='A'):
         super(BasicBlock, self).__init__()
-        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
+        self.conv1 = Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
         self.bn1 = nn.BatchNorm2d(planes)
-        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
+        self.conv2 = Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
         self.bn2 = nn.BatchNorm2d(planes)
 
         self.shortcut = nn.Sequential()
@@ -91,7 +91,7 @@ def __init__(self, in_planes, planes, stride=1, option='A'):
                 self.shortcut = ShortcutLayer(stride, in_planes, planes)
             elif option == 'B':
                 self.shortcut = nn.Sequential(
-                     nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False),
+                     Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False),
                      nn.BatchNorm2d(self.expansion * planes)
                 )
 
@@ -108,7 +108,7 @@ def __init__(self, block, num_blocks, num_classes=10):
         super(ResNet, self).__init__()
         self.in_planes = 16
 
-        self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)
+        self.conv1 = Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False)
         self.bn1 = nn.BatchNorm2d(16)
         self.layer1 = self._make_layer(block, 16, num_blocks[0], stride=1)
         self.layer2 = self._make_layer(block, 32, num_blocks[1], stride=2)

From d58baf3acdee9afcf3882e3e72cd878f75c8e5d1 Mon Sep 17 00:00:00 2001
From: alexander-darwiche
 <59787022+alexander-darwiche@users.noreply.github.com>
Date: Tue, 12 Nov 2024 16:03:35 -0500
Subject: [PATCH 5/5] remove debugging print statement

---
 src/resnet.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/resnet.py b/src/resnet.py
index 6763cfb..9e3911c 100644
--- a/src/resnet.py
+++ b/src/resnet.py
@@ -42,7 +42,6 @@ def __init__(self, in_channels, out_channels, kernel_size, stride=1,
                  padding=0, dilation=1, groups=1, bias=True):
         super(Conv2d, self).__init__(in_channels, out_channels, kernel_size, stride,
                  padding, dilation, groups, bias)
-        print('redefining conv2d')
 
     def forward(self, x):
         weight = self.weight